From cf28b2794b23604c1b45285f139415397998ac6d Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 20 Nov 2019 13:20:11 -0500 Subject: [PATCH 001/270] created skeleton for greyscale model --- common/ScaLBL.h | 5 + cpu/Greyscale.cpp | 278 ++++++++++++++ gpu/Greyscale.cu | 311 ++++++++++++++++ models/GreyscaleModel.cpp | 568 +++++++++++++++++++++++++++++ models/GreyscaleModel.h | 81 ++++ tests/CMakeLists.txt | 1 + tests/lbpm_greyscale_simulator.cpp | 64 ++++ 7 files changed, 1308 insertions(+) create mode 100644 cpu/Greyscale.cpp create mode 100644 gpu/Greyscale.cu create mode 100644 models/GreyscaleModel.cpp create mode 100644 models/GreyscaleModel.h create mode 100644 tests/lbpm_greyscale_simulator.cpp diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a50ab7ed..efca3be8 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -55,6 +55,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); +// GREYSCALE MODEL +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp new file mode 100644 index 00000000..a800413d --- /dev/null +++ b/cpu/Greyscale.cpp @@ -0,0 +1,278 @@ +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; + // conserved momemnts + double rho,ux,uy,uz,uu; + // non-conserved moments + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + + for (int n=start; n 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + // q=7 + nr7 = neighborList[n+6*Np]; + f7 = dist[nr7]; + + // q = 8 + nr8 = neighborList[n+7*Np]; + f8 = dist[nr8]; + + // q=9 + nr9 = neighborList[n+8*Np]; + f9 = dist[nr9]; + + // q = 10 + nr10 = neighborList[n+9*Np]; + f10 = dist[nr10]; + + // q=11 + nr11 = neighborList[n+10*Np]; + f11 = dist[nr11]; + + // q=12 + nr12 = neighborList[n+11*Np]; + f12 = dist[nr12]; + + // q=13 + nr13 = neighborList[n+12*Np]; + f13 = dist[nr13]; + + // q=14 + nr14 = neighborList[n+13*Np]; + f14 = dist[nr14]; + + // q=15 + nr15 = neighborList[n+14*Np]; + f15 = dist[nr15]; + + // q=16 + nr16 = neighborList[n+15*Np]; + f16 = dist[nr16]; + + // q=17 + //fq = dist[18*Np+n]; + nr17 = neighborList[n+16*Np]; + f17 = dist[nr17]; + + // q=18 + nr18 = neighborList[n+17*Np]; + f18 = dist[nr18]; + + rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; + uu = 1.5*(ux*ux+uy*uy+uz*uz); + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx; + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx; + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy; + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy; + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz; + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz; + + // q = 7 + dist[nr8] = f7*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy); + + // q = 8 + dist[nr7] = f8*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy); + + // q = 9 + dist[nr10] = f9*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy); + + // q = 10 + dist[nr9] = f10*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy); + + // q = 11 + dist[nr12] = f11*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz); + + // q = 12 + dist[nr11] = f12*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) - 0.08333333333*(Fx+Fz); + + // q = 13 + dist[nr14] = f13*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz); + + // q= 14 + dist[nr13] = f14*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz); + + // q = 15 + dist[nr16] = f15*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz); + + // q = 16 + dist[nr15] = f16*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz); + + // q = 17 + dist[nr18] = f17*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz); + + // q = 18 + dist[nr17] = f18*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz); + + } +} \ No newline at end of file diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu new file mode 100644 index 00000000..04b5e979 --- /dev/null +++ b/gpu/Greyscale.cu @@ -0,0 +1,311 @@ +#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; + // conserved momemnts + double rho,ux,uy,uz,uu; + // non-conserved moments + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + // q=7 + nr7 = neighborList[n+6*Np]; + f7 = dist[nr7]; + + // q = 8 + nr8 = neighborList[n+7*Np]; + f8 = dist[nr8]; + + // q=9 + nr9 = neighborList[n+8*Np]; + f9 = dist[nr9]; + + // q = 10 + nr10 = neighborList[n+9*Np]; + f10 = dist[nr10]; + + // q=11 + nr11 = neighborList[n+10*Np]; + f11 = dist[nr11]; + + // q=12 + nr12 = neighborList[n+11*Np]; + f12 = dist[nr12]; + + // q=13 + nr13 = neighborList[n+12*Np]; + f13 = dist[nr13]; + + // q=14 + nr14 = neighborList[n+13*Np]; + f14 = dist[nr14]; + + // q=15 + nr15 = neighborList[n+14*Np]; + f15 = dist[nr15]; + + // q=16 + nr16 = neighborList[n+15*Np]; + f16 = dist[nr16]; + + // q=17 + //fq = dist[18*Np+n]; + nr17 = neighborList[n+16*Np]; + f17 = dist[nr17]; + + // q=18 + nr18 = neighborList[n+17*Np]; + f18 = dist[nr18]; + + rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18; + uu = 1.5*(ux*ux+uy*uy+uz*uz); + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx; + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx; + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy; + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy; + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz; + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz; + + // q = 7 + dist[nr8] = f7*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy); + + // q = 8 + dist[nr7] = f8*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy); + + // q = 9 + dist[nr10] = f9*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy); + + // q = 10 + dist[nr9] = f10*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy); + + // q = 11 + dist[nr12] = f11*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz); + + // q = 12 + dist[nr11] = f12*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) - 0.08333333333*(Fx+Fz); + + // q = 13 + dist[nr14] = f13*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz); + + // q= 14 + dist[nr13] = f14*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz); + + // q = 15 + dist[nr16] = f15*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz); + + // q = 16 + dist[nr15] = f16*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz); + + // q = 17 + dist[nr18] = f17*(1.0-rlx) + + rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz); + + // q = 18 + dist[nr17] = f18*(1.0-rlx) + + rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz); + } + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + + dvc_ScaLBL_D3Q19_AAeven_Greyscale<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale: %s \n",cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp new file mode 100644 index 00000000..980d15b5 --- /dev/null +++ b/models/GreyscaleModel.cpp @@ -0,0 +1,568 @@ +/* +color lattice boltzmann model + */ +#include "models/GreyscaleModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include +#include + +ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + SignDist.resize(Nx,Ny,Nz); SignDist.fill(0); + +} +ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ + +} + +void ScaLBL_GreyscaleModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + greyscale_db = db->getDatabase( "Greyscale" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tau = 1.0; + tolerance = 0.01; + Fx = Fy = Fz = 0.0; + Restart=false; + din=dout=1.0; + flux=0.0; + + // Color Model parameters + if (greyscale_db->keyExists( "timestepMax" )){ + timestepMax = greyscale_db->getScalar( "timestepMax" ); + } + if (greyscale_db->keyExists( "tau" )){ + tau = greyscale_db->getScalar( "tauA" ); + } + if (greyscale_db->keyExists( "F" )){ + Fx = greyscale_db->getVector( "F" )[0]; + Fy = greyscale_db->getVector( "F" )[1]; + Fz = greyscale_db->getVector( "F" )[2]; + } + if (greyscale_db->keyExists( "Restart" )){ + Restart = greyscale_db->getScalar( "Restart" ); + } + if (greyscale_db->keyExists( "din" )){ + din = greyscale_db->getScalar( "din" ); + } + if (greyscale_db->keyExists( "dout" )){ + dout = greyscale_db->getScalar( "dout" ); + } + if (greyscale_db->keyExists( "flux" )){ + flux = greyscale_db->getScalar( "flux" ); + } + if (greyscale_db->keyExists( "tolerance" )){ + tolerance = greyscale_db->getScalar( "tolerance" ); + } + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } +} + +void ScaLBL_GreyscaleModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_GreyscaleModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + int count = 0; + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kgetVector( "ComponentLabels" ); + auto PorosityList = greyscale_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + + NLABELS=LabelList.size(); + if (NLABELS != PorosityList.size()){ + ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + // fluid labels are reserved / negative labels are immobile + if (VALUE == 1) POROSITY=1.0; + else if (VALUE == 2) POROSITY=1.0; + else if (VALUE < 1) POROSITY = 0.0; + int idx = Map(i,j,k); + if (!(idx < 0)) + Porosity[idx] = POROSITY; + } + } + } + + if (NLABELS != PermeabilityList.size()){ + ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); + } + for (int k=1;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + // fluid labels are reserved / negative labels are immobile + if (VALUE == 1) PERMEABILITY=1.0; + else if (VALUE == 2) PERMEABILITY=1.0; + else if (VALUE < 1) PERMEABILITY = 0.0; + int idx = Map(i,j,k); + if (!(idx < 0)) + Permeability[idx] = PERMEABILITY; + } + } + } + + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm, label_count[idx]); + + if (rank==0){ + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + int n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n"); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + int n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n"); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; + + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + // initialize phi based on PhaseLabel (include solid component labels) + double *Poros, *Perm; + Poros = new double[Np]; + Perm = new double[Np]; + AssignComponentLabels(Poros,Perm); + ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); +} + +/******************************************************** + * AssignComponentLabels * + ********************************************************/ + +void ScaLBL_GreyscaleModel::Initialize(){ + + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + /* + * This function initializes model + */ + if (Restart == true){ + if (rank==0){ + printf("Reading restart file! \n"); + } + + // Read in the restart file to CPU buffers + int *TmpMap; + TmpMap = new int[Np]; + + double *cDist; + cDist = new double[19*Np]; + ScaLBL_CopyToHost(TmpMap, dvcMap, Np*sizeof(int)); + + ifstream File(LocalRestartFile,ios::binary); + int idx; + double value; + for (int n=0; n analysis_db; + timestep=0; + double rlx = 1.0/tau; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + + if (timestep%1000==0){ + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int k=1; k 0){ + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; + } + } + } + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + + error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); + flow_rate_previous = flow_rate; + + //if (rank==0) printf("Computing Minkowski functionals \n"); + Morphology.ComputeScalar(SignDist,0.f); + //Morphology.PrintAll(); + double mu = (tau-0.5)/3.f; + double Vs = Morphology.V(); + double As = Morphology.A(); + double Hs = Morphology.H(); + double Xs = Morphology.X(); + Vs=sumReduce( Dm->Comm, Vs); + As=sumReduce( Dm->Comm, As); + Hs=sumReduce( Dm->Comm, Hs); + Xs=sumReduce( Dm->Comm, Xs); + double h = Dm->voxel_length; + double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + if (rank==0) { + printf(" %f\n",absperm); + FILE * log_file = fopen("Permeability.csv","a"); + fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, + h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); + fclose(log_file); + } + } + } + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + + +void ScaLBL_GreyscaleModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout +/* ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); + + FILE *OUTFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + + * + */ + +} diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h new file mode 100644 index 00000000..37ddf28f --- /dev/null +++ b/models/GreyscaleModel.h @@ -0,0 +1,81 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "common/Database.h" +#include "common/ScaLBL.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_GreyscaleModel{ +public: + ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + double tau; + double tolerance; + double Fx,Fy,Fz,flux; + double din,dout; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr greyscale_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + IntArray Map; + DoubleArray SignDist; + signed char *id; + int *NeighborList; + int *dvcMap; + double *fq; + double *Permeability; + double *Porosity; + double *Velocity; + double *Pressure; + +private: + MPI_Comm comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + void AssignComponentLabels(double *Porosity, double *Permeablity); + +}; + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8d600321..8b14a9dc 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,6 +3,7 @@ #ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator ) ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp new file mode 100644 index 00000000..9ab7c385 --- /dev/null +++ b/tests/lbpm_greyscale_simulator.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "models/GreyscaleModel.h" +//#define WRITE_SURFACES + +/* + * Simulator for two-phase flow in porous media + * James E. McClure 2013-2014 + */ + +using namespace std; + + +int main(int argc, char **argv) +{ + //***************************************** + // ***** MPI STUFF **************** + //***************************************** + // Initialize MPI + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { + // parallel domain size (# of sub-domains) + int nprocx,nprocy,nprocz; + int iproc,jproc,kproc; + + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Greyscale Single Phase Permeability Calculation \n"); + printf("********************************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + + ScaLBL_MRTModel MRT(rank,nprocs,comm); + auto filename = argv[1]; + MRT.ReadParams(filename); + MRT.SetDomain(); // this reads in the domain + MRT.ReadInput(); + MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables + MRT.Initialize(); // initializing the model will set initial conditions for variables + MRT.Run(); + MRT.VelocityField(); + } + // **************************************************** + MPI_Barrier(comm); + MPI_Finalize(); + // **************************************************** +} From 2abcf030286f148af2bcbea34d5bf67708b01f92 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 21 Nov 2019 13:01:24 -0500 Subject: [PATCH 002/270] greyscale update --- tests/lbpm_greyscale_simulator.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index 9ab7c385..0744a214 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -47,15 +47,14 @@ int main(int argc, char **argv) MPI_Barrier(comm); - ScaLBL_MRTModel MRT(rank,nprocs,comm); + ScaLBL_GreyscaleModel Greyscale(rank,nprocs,comm); auto filename = argv[1]; - MRT.ReadParams(filename); - MRT.SetDomain(); // this reads in the domain - MRT.ReadInput(); - MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables - MRT.Initialize(); // initializing the model will set initial conditions for variables - MRT.Run(); - MRT.VelocityField(); + Greyscale.ReadParams(filename); + Greyscale.SetDomain(); // this reads in the domain + Greyscale.ReadInput(); + Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables + Greyscale.Initialize(); // initializing the model will set initial conditions for variables + Greyscale.Run(); } // **************************************************** MPI_Barrier(comm); From 86beafab8acb435a76b03381e8631d909b55c6fe Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 21 Nov 2019 13:43:32 -0500 Subject: [PATCH 003/270] save the work for cpu version --- common/ScaLBL.h | 6 +- cpu/Greyscale.cpp | 277 ++++++++++++++++++++--------- models/GreyscaleModel.cpp | 166 ++++++++++++++--- models/GreyscaleModel.h | 11 +- tests/lbpm_greyscale_simulator.cpp | 16 +- 5 files changed, 356 insertions(+), 120 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index efca3be8..ecb1ffed 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -56,9 +56,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); // GREYSCALE MODEL -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity); -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index a800413d..fa9a1f49 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1,9 +1,19 @@ -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ +#include + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity){ int n; // conserved momemnts - double rho,ux,uy,uz,uu; + double rho,vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu = (1.0/rlx-0.5)/3.0;//kinematic viscosity for (int n=start; n ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { - SignDist.resize(Nx,Ny,Nz); SignDist.fill(0); + SignDist.resize(Nx,Ny,Nz); + SignDist.fill(0); } ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ @@ -35,13 +35,17 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ Restart=false; din=dout=1.0; flux=0.0; + dp = 10.0; //unit of 'dp': voxel // Color Model parameters if (greyscale_db->keyExists( "timestepMax" )){ timestepMax = greyscale_db->getScalar( "timestepMax" ); } if (greyscale_db->keyExists( "tau" )){ - tau = greyscale_db->getScalar( "tauA" ); + tau = greyscale_db->getScalar( "tau" ); + } + if (greyscale_db->keyExists( "dp" )){ + dp = greyscale_db->getScalar( "dp" ); } if (greyscale_db->keyExists( "F" )){ Fx = greyscale_db->getVector( "F" )[0]; @@ -80,6 +84,12 @@ void ScaLBL_GreyscaleModel::SetDomain(){ Ly = Dm->Ly; Lz = Dm->Lz; N = Nx*Ny*Nz; + + SignDist.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way MPI_Barrier(comm); @@ -140,6 +150,9 @@ void ScaLBL_GreyscaleModel::ReadInput(){ if (rank == 0) cout << "Domain set." << endl; } +/******************************************************** + * AssignComponentLabels * + ********************************************************/ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Permeablity) { size_t NLABELS=0; @@ -182,8 +195,14 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm else if (VALUE == 2) POROSITY=1.0; else if (VALUE < 1) POROSITY = 0.0; int idx = Map(i,j,k); - if (!(idx < 0)) - Porosity[idx] = POROSITY; + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Porosity[idx] = POROSITY; + } + } } } } @@ -205,13 +224,21 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm //Mask->id[n] = 0; // set mask to zero since this is an immobile component } } - // fluid labels are reserved / negative labels are immobile + // Permeability of fluid labels are reserved + // NOTE: the voxel permeability of apparent pore nodes should be infinity + // TODO: Need to revise the PERMEABILITY of nodes whose VALUE=1 and 2 if (VALUE == 1) PERMEABILITY=1.0; else if (VALUE == 2) PERMEABILITY=1.0; else if (VALUE < 1) PERMEABILITY = 0.0; int idx = Map(i,j,k); - if (!(idx < 0)) - Permeability[idx] = PERMEABILITY; + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Permeability[idx] = PERMEABILITY; + } + } } } } @@ -229,7 +256,7 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm POROSITY=PorosityList[idx]; PERMEABILITY=PermeabilityList[idx]; double volume_fraction = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs); - printf(" label=%d, porosity=%f, permeability=%f, volume fraction==%f\n",VALUE,POROSITY,PERMEABILITY,volume_fraction); + printf(" label=%d, porosity=%.3g, permeability=%.3g, volume fraction==%.3g\n",VALUE,POROSITY,PERMEABILITY,volume_fraction); } } @@ -324,9 +351,6 @@ void ScaLBL_GreyscaleModel::Create(){ ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); } -/******************************************************** - * AssignComponentLabels * - ********************************************************/ void ScaLBL_GreyscaleModel::Initialize(){ @@ -387,10 +411,6 @@ void ScaLBL_GreyscaleModel::Run(){ //......................................... Minkowski Morphology(Mask); - DoubleArray Velocity_x(Nx,Ny,Nz); - DoubleArray Velocity_y(Nx,Ny,Nz); - DoubleArray Velocity_z(Nx,Ny,Nz); - DoubleArray Pressure(Nx,Ny,Nz); //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); @@ -403,21 +423,21 @@ void ScaLBL_GreyscaleModel::Run(){ //************************************************************************/ timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ if (timestep%1000==0){ - ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); @@ -509,6 +529,106 @@ void ScaLBL_GreyscaleModel::Run(){ // ************************************************************************ } +void ScaLBL_GreyscaleModel::VelocityField(){ + +/* Minkowski Morphology(Mask); + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); + + memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); + Morphology.Initialize(); + Morphology.UpdateMeshValues(); + Morphology.ComputeLocal(); + Morphology.Reduce(); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int n=0; nLastExterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + + for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double mu = (tau-0.5)/3.f; + if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); + if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, + Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); + */ + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + Array& SignData = visData[0].vars[0]->data; + Array& VelxData = visData[0].vars[1]->data; + Array& VelyData = visData[0].vars[2]->data; + Array& VelzData = visData[0].vars[3]->data; + + ASSERT(visData[0].vars[0]->name=="SignDist"); + ASSERT(visData[0].vars[1]->name=="Velocity_x"); + ASSERT(visData[0].vars[2]->name=="Velocity_y"); + ASSERT(visData[0].vars[3]->name=="Velocity_z"); + + fillData.copy(SignDist,SignData); + fillData.copy(Velocity_x,VelxData); + fillData.copy(Velocity_y,VelyData); + fillData.copy(Velocity_z,VelzData); + + IO::writeData( timestep, visData, Dm->Comm ); + +} void ScaLBL_GreyscaleModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 37ddf28f..9b970a65 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -30,6 +30,7 @@ public: void Initialize(); void Run(); void WriteDebug(); + void VelocityField(); bool Restart,pBC; int timestep,timestepMax; @@ -38,6 +39,7 @@ public: double tolerance; double Fx,Fy,Fz,flux; double din,dout; + double dp;//solid particle diameter, unit in voxel int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -54,16 +56,19 @@ public: std::shared_ptr analysis_db; std::shared_ptr vis_db; - IntArray Map; - DoubleArray SignDist; signed char *id; int *NeighborList; int *dvcMap; double *fq; - double *Permeability; + double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; double *Pressure; + IntArray Map; + DoubleArray SignDist; + DoubleArray Velocity_x; + DoubleArray Velocity_y; + DoubleArray Velocity_z; private: MPI_Comm comm; diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index 9ab7c385..e15797e3 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -47,15 +47,15 @@ int main(int argc, char **argv) MPI_Barrier(comm); - ScaLBL_MRTModel MRT(rank,nprocs,comm); + ScaLBL_GreyscaleModel GreyscaleModel(rank,nprocs,comm); auto filename = argv[1]; - MRT.ReadParams(filename); - MRT.SetDomain(); // this reads in the domain - MRT.ReadInput(); - MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables - MRT.Initialize(); // initializing the model will set initial conditions for variables - MRT.Run(); - MRT.VelocityField(); + GreyscaleModel.ReadParams(filename); + GreyscaleModel.SetDomain(); // this reads in the domain + GreyscaleModel.ReadInput(); + GreyscaleModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleModel.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleModel.Run(); + GreyscaleModel.VelocityField(); } // **************************************************** MPI_Barrier(comm); From a4b0f3e26eafa27d5de3b81008e99f060ce86a49 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 21 Nov 2019 14:05:58 -0500 Subject: [PATCH 004/270] gpu version of greyscale LBM is also updated --- gpu/Greyscale.cu | 287 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 198 insertions(+), 89 deletions(-) diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 04b5e979..3365c6f9 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -3,12 +3,20 @@ #define NBLOCKS 1024 #define NTHREADS 256 -__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity){ int n; // conserved momemnts - double rho,ux,uy,uz,uu; + double rho,vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu = (1.0/rlx-0.5)/3.0;//kinematic viscosity int S = Np/NBLOCKS/NTHREADS + 1; for (int s=0; s>>(dist,start,finish,Np,rlx,Fx,Fy,Fz); + dvc_ScaLBL_D3Q19_AAeven_Greyscale<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -301,11 +410,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis } } -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ - dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz); +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity){ + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale: %s \n",cudaGetErrorString(err)); } -} \ No newline at end of file +} From cf5a284f6dd7ef2ac3b03128c11d2eb960651c03 Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 3 Dec 2019 13:01:37 -0500 Subject: [PATCH 005/270] adding subphase functionality --- example/Workflow/HelperFunctions.R | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/example/Workflow/HelperFunctions.R b/example/Workflow/HelperFunctions.R index a86ee6fe..497cb262 100644 --- a/example/Workflow/HelperFunctions.R +++ b/example/Workflow/HelperFunctions.R @@ -7,11 +7,26 @@ ReadSubphase<-function(PATH){ S<-read.csv(FILE,head=TRUE,sep=" ") S$Vw<-S$Vwc+S$Vwd S$Vn<-S$Vnc+S$Vnd + S$Aw<-S$Awc+S$Awd + S$An<-S$Anc+S$And + S$Hw<-S$Hwc+S$Hwd + S$Hn<-S$Hnc+S$Hnd + S$Xw<-S$Xwc+S$Xwd + S$Xn<-S$Xnc+S$Xnd + S$Sw<-S$Vw/(S$Vn+S$Vw) + S$pw<-(S$pwc*S$Vwc+S$pwd*S$Vwd) / (S$Vwc+S$Vwd) + S$pn<-(S$pnc*S$Vnc+S$pnd*S$Vnd) / (S$Vnc+S$Vnd) + S$Qwx<-S$Vw*(S$Pwc_x+S$Pwd_x)/(S$Mwc+S$Mwd) S$Qnx<-S$Vn*(S$Pnc_x+S$Pnd_x)/(S$Mnc+S$Mnd) - S$Krn<-S$nun*S$Qnx/S$Fx - S$Krw<-S$nuw*S$Qwx/S$Fx + S$Qwy<-S$Vw*(S$Pwc_y+S$Pwd_y)/(S$Mwc+S$Mwd) + S$Qny<-S$Vn*(S$Pnc_y+S$Pnd_y)/(S$Mnc+S$Mnd) + S$Qwz<-S$Vw*(S$Pwc_z+S$Pwd_z)/(S$Mwc+S$Mwd) + S$Qnz<-S$Vn*(S$Pnc_z+S$Pnd_z)/(S$Mnc+S$Mnd) + + S$Krn<-S$nun*S$Qnz/S$Fz + S$Krw<-S$nuw*S$Qwz/S$Fz S$Case<-PATH return(S) } From a67d3f8b6900e032c02d2b879f0c666aab86c0fd Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 9 Dec 2019 14:44:58 -0500 Subject: [PATCH 006/270] calculate the medium porosity if read domain from Filename --- common/Domain.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/common/Domain.cpp b/common/Domain.cpp index 0fa8545a..82bcaee2 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -585,6 +585,50 @@ void Domain::Decomp(std::string Filename) MPI_Recv(id,N,MPI_CHAR,0,15,Comm,MPI_STATUS_IGNORE); } MPI_Barrier(Comm); + + // Compute the porosity + double sum; + double sum_local=0.0; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + //......................................................... + // If external boundary conditions are applied remove solid + if (BoundaryCondition > 0 && kproc() == 0){ + if (inlet_layers_z < 4) inlet_layers_z=4; + for (int k=0; k 0 && kproc() == nprocz-1){ + if (outlet_layers_z < 4) outlet_layers_z=4; + for (int k=Nz-outlet_layers_z; k 0){ + sum_local+=1.0; + } + } + } + } + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm); + porosity = sum*iVol_global; + if (rank()==0) printf("Media porosity = %f \n",porosity); + //......................................................... } void Domain::AggregateLabels(char *FILENAME){ From 9c48b3de7070734e6ba3b637a369e634787bd321 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 9 Dec 2019 15:17:27 -0500 Subject: [PATCH 007/270] enable single phase abs-perm simulator to read medium from Filename --- common/Domain.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ models/MRTModel.cpp | 13 ++++++++----- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 0fa8545a..82bcaee2 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -585,6 +585,50 @@ void Domain::Decomp(std::string Filename) MPI_Recv(id,N,MPI_CHAR,0,15,Comm,MPI_STATUS_IGNORE); } MPI_Barrier(Comm); + + // Compute the porosity + double sum; + double sum_local=0.0; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + //......................................................... + // If external boundary conditions are applied remove solid + if (BoundaryCondition > 0 && kproc() == 0){ + if (inlet_layers_z < 4) inlet_layers_z=4; + for (int k=0; k 0 && kproc() == nprocz-1){ + if (outlet_layers_z < 4) outlet_layers_z=4; + for (int k=Nz-outlet_layers_z; k 0){ + sum_local+=1.0; + } + } + } + } + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm); + porosity = sum*iVol_global; + if (rank()==0) printf("Media porosity = %f \n",porosity); + //......................................................... } void Domain::AggregateLabels(char *FILENAME){ diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 04fe937d..e2984b2a 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -93,16 +93,19 @@ void ScaLBL_MRTModel::SetDomain(){ } void ScaLBL_MRTModel::ReadInput(){ - int rank=Dm->rank(); - size_t readID; - //....................................................................... - //....................................................................... - Mask->ReadIDs(); sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + Mask->ReadIDs(); + } + // Generate the signed distance map // Initialize the domain and communication Array id_solid(Nx,Ny,Nz); From bbd2a6e34a81d1349ffbc4ac4a729164bd7f2133 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 9 Dec 2019 15:56:44 -0500 Subject: [PATCH 008/270] update output (*.out and Permeability.csv) for Greyscale --- models/GreyscaleModel.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index cf66d6f4..5af10205 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -498,13 +498,24 @@ void ScaLBL_GreyscaleModel::Run(){ Xs=sumReduce( Dm->Comm, Xs); double h = Dm->voxel_length; double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; - if (rank==0) { - printf(" %f\n",absperm); - FILE * log_file = fopen("Permeability.csv","a"); - fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, - h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); - fclose(log_file); - } + + if (rank==0){ + printf(" AbsPerm = %.5g [micron^2]\n",absperm); + bool WriteHeader=false; + FILE * log_file = fopen("Permeability.csv","r"); + if (log_file != NULL) + fclose(log_file); + else + WriteHeader=true; + log_file = fopen("Permeability.csv","a"); + if (WriteHeader) + fprintf(log_file,"timesteps Fx Fy Fz mu Vs As Hs Xs vax vay vaz absperm \n", + timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); + + fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, + h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); + fclose(log_file); + } } } PROFILE_STOP("Loop"); From 5f85b767d6f5fd01463506d832d44f6be02b2ba5 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 9 Dec 2019 22:30:36 -0500 Subject: [PATCH 009/270] add debugging for greyscale lbm --- models/GreyscaleModel.cpp | 56 +++++++++++++++--------------- tests/lbpm_greyscale_simulator.cpp | 1 + 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 5af10205..2ec3b85e 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -643,34 +643,36 @@ void ScaLBL_GreyscaleModel::VelocityField(){ void ScaLBL_GreyscaleModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout -/* ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); + DoubleArray PhaseField(Nx,Ny,Nz); - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,OUTFILE); - fclose(OUTFILE); + //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - FILE *AFILE; - sprintf(LocalRankFilename,"A.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - FILE *BFILE; - sprintf(LocalRankFilename,"B.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); - - ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); - FILE *PFILE; - sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); - PFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PFILE); - fclose(PFILE); +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); +// +// ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); +// FILE *PFILE; +// sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); +// PFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PFILE); +// fclose(PFILE); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); FILE *VELX_FILE; @@ -693,7 +695,5 @@ void ScaLBL_GreyscaleModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); - * - */ } diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index 9f910a32..61322d6d 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -55,6 +55,7 @@ int main(int argc, char **argv) Greyscale.Initialize(); // initializing the model will set initial conditions for variables Greyscale.Run(); Greyscale.VelocityField(); + Greyscale.WriteDebug(); } // **************************************************** MPI_Barrier(comm); From cddcfa0188d932f755981e92ede2d71c5394d267 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 10 Dec 2019 16:54:42 -0500 Subject: [PATCH 010/270] fix the bug and now have a workable greyscale BGK model in both CPU and GPU --- cpu/Greyscale.cpp | 54 +++++++++++++++++---------------------- gpu/Greyscale.cu | 52 +++++++++++++++++-------------------- models/GreyscaleModel.cpp | 14 +++++++++- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index fa9a1f49..48e61a56 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1,6 +1,6 @@ #include -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity){ int n; // conserved momemnts @@ -14,6 +14,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis double perm;//voxel permeability double c0, c1; //Guo's model parameters double mu = (1.0/rlx-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) for (int n=start; nRegularLayout(Map,&Porosity[0],PhaseField); + FILE *POROS_FILE; + sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); + POROS_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,POROS_FILE); + fclose(POROS_FILE); + ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); + FILE *PERM_FILE; + sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); + PERM_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PERM_FILE); + fclose(PERM_FILE); } From 38f97c2848aa484fe887313d2c054ae78208f594 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 13 Jan 2020 22:50:37 -0500 Subject: [PATCH 011/270] save the work --- gpu/Greyscale.cu | 468 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 18bfba58..fdb0a462 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -394,6 +394,474 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist } } +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity, double Den){ + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + // conserved momemnts + double rho,jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu = (1.0/rlx-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); From 3b23fca11864c313da091bd375000dfb70d39a88 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 14 Jan 2020 12:00:22 -0500 Subject: [PATCH 012/270] change specifier of printf to correct the output for very large image --- common/Domain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 82bcaee2..48bfed15 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -391,7 +391,7 @@ void Domain::Decomp(std::string Filename) for (int idx=0; idx Date: Tue, 14 Jan 2020 12:01:33 -0500 Subject: [PATCH 013/270] change specifier of printf to correct the output for very large image --- common/Domain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 82bcaee2..48bfed15 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -391,7 +391,7 @@ void Domain::Decomp(std::string Filename) for (int idx=0; idx Date: Fri, 17 Jan 2020 18:46:28 -0500 Subject: [PATCH 014/270] save the work, CPU versions seem to work, but need non-unity porosity test --- common/Domain.cpp | 2 +- common/ScaLBL.h | 7 + cpu/Greyscale.cpp | 1070 +++++++++++++++++++++++++++++++++++++ models/GreyscaleModel.cpp | 20 +- models/GreyscaleModel.h | 1 + 5 files changed, 1093 insertions(+), 7 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 82bcaee2..48bfed15 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -391,7 +391,7 @@ void Domain::Decomp(std::string Filename) for (int idx=0; idx 10Np => odd part of dist) + fq = dist[nread]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nread]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nread = neighborList[n+5*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/Den+0.5*porosity*Gx; + vy = jy/Den+0.5*porosity*Gy; + vz = jz/Den+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = Den*(-porosity*mu/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = Den*(-porosity*mu/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = Den*(-porosity*mu/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=Den*Gx; + Fy=Den*Gy; + Fz=Den*Gz; + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + + //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(jx*jx+jy*jy+jz*jz)/Den/porosity + 57*pressure*porosity) - m1); +// m2 = m2 + rlx_setA*((12*Den - 5.5*(jx*jx+jy*jy+jz*jz)/Den/porosity-27*pressure*porosity) - m2); +// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); +// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); +// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/Den/porosity) - m9); +// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/Den/porosity)- m10); +// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/Den/porosity) - m11); +// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/Den/porosity)- m12); +// m13 = m13 + rlx_setA*((jx*jy/Den/porosity) - m13); +// m14 = m14 + rlx_setA*((jy*jz/Den/porosity) - m14); +// m15 = m15 + rlx_setA*((jx*jz/Den/porosity) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8); + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + jx+=0.5*Fx;//There is no collision for momentum, but they must be updated subject to the body force + jy+=0.5*Fy; + jz+=0.5*Fz; + //.................inverse transformation...................................................... + // q=0 + //fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2 + + 0.3333333333333333*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + dist[n] = fq; + + // q = 1 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+Np]; + dist[nread] = fq; + + // q=2 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n]; + dist[nread] = fq; + + // q = 3 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+3*Np]; + dist[nread] = fq; + + // q = 4 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+2*Np]; + dist[nread] = fq; + + // q = 5 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + nread = neighborList[n+5*Np]; + dist[nread] = fq; + + // q = 6 + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + nread = neighborList[n+4*Np]; + dist[nread] = fq; + + // q = 7 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + + Fz*(0. - (3.*uz)/porosity)); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + + Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + + Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + + Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + + Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + + Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + + Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + + Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) + +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + + Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + } +} + diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 2ce6ff5e..1cdae815 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -8,7 +8,7 @@ color lattice boltzmann model #include ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); @@ -30,6 +30,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ // set defaults timestepMax = 100000; tau = 1.0; + Den = 1.0;//constant density tolerance = 0.01; Fx = Fy = Fz = 0.0; Restart=false; @@ -37,13 +38,16 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ flux=0.0; dp = 10.0; //unit of 'dp': voxel - // Color Model parameters + // Greyscale Model parameters if (greyscale_db->keyExists( "timestepMax" )){ timestepMax = greyscale_db->getScalar( "timestepMax" ); } if (greyscale_db->keyExists( "tau" )){ tau = greyscale_db->getScalar( "tau" ); } + if (greyscale_db->keyExists( "Den" )){ + Den = greyscale_db->getScalar( "Den" ); + } if (greyscale_db->keyExists( "dp" )){ dp = greyscale_db->getScalar( "dp" ); } @@ -423,15 +427,19 @@ void ScaLBL_GreyscaleModel::Run(){ //************************************************************************/ timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 9b970a65..ac939aed 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -36,6 +36,7 @@ public: int timestep,timestepMax; int BoundaryCondition; double tau; + double Den;//constant density double tolerance; double Fx,Fy,Fz,flux; double din,dout; From 7e4e91a06beb1ee75633e31e0e25e3fe1e032e45 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 18 Jan 2020 18:43:20 -0500 Subject: [PATCH 015/270] The CPU version of incompressible MRT greyscale model is available now --- cpu/Greyscale.cpp | 298 +++++++++++++++------------------------------- 1 file changed, 93 insertions(+), 205 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index 11a8eb5c..95cf516b 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -724,169 +724,113 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(jx*jx+jy*jy+jz*jz)/Den/porosity + 57*pressure*porosity) - m1); -// m2 = m2 + rlx_setA*((12*Den - 5.5*(jx*jx+jy*jy+jz*jz)/Den/porosity-27*pressure*porosity) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/Den/porosity) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/Den/porosity)- m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/Den/porosity) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/Den/porosity)- m12); -// m13 = m13 + rlx_setA*((jx*jy/Den/porosity) - m13); -// m14 = m14 + rlx_setA*((jy*jz/Den/porosity) - m14); -// m15 = m15 + rlx_setA*((jx*jz/Den/porosity) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) + + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) + + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) + + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) + + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) + + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) + + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) + + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) + + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); //....................................................................................................... - jx+=0.5*Fx;//There is no collision for momentum, but they must be updated subject to the body force - jy+=0.5*Fy; - jz+=0.5*Fz; //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2 - + 0.3333333333333333*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); dist[1*Np+n] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); dist[2*Np+n] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[3*Np+n] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[4*Np+n] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[5*Np+n] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[6*Np+n] = fq; // q = 7 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); dist[7*Np+n] = fq; // q = 8 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); dist[8*Np+n] = fq; // q = 9 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); dist[9*Np+n] = fq; // q = 10 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); dist[10*Np+n] = fq; // q = 11 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); dist[11*Np+n] = fq; // q = 12 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); dist[12*Np+n] = fq; // q = 13 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); dist[13*Np+n] = fq; // q= 14 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); dist[14*Np+n] = fq; // q = 15 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); dist[15*Np+n] = fq; // q = 16 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); dist[16*Np+n] = fq; // q = 17 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); dist[17*Np+n] = fq; // q = 18 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); dist[18*Np+n] = fq; //........................................................................ @@ -1258,186 +1202,130 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(jx*jx+jy*jy+jz*jz)/Den/porosity + 57*pressure*porosity) - m1); -// m2 = m2 + rlx_setA*((12*Den - 5.5*(jx*jx+jy*jy+jz*jz)/Den/porosity-27*pressure*porosity) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/Den/porosity) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/Den/porosity)- m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/Den/porosity) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/Den/porosity)- m12); -// m13 = m13 + rlx_setA*((jx*jy/Den/porosity) - m13); -// m14 = m14 + rlx_setA*((jy*jz/Den/porosity) - m14); -// m15 = m15 + rlx_setA*((jx*jz/Den/porosity) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) + + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) + + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) + + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) + + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) + + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) + + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) + + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) + + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); //....................................................................................................... - - jx+=0.5*Fx;//There is no collision for momentum, but they must be updated subject to the body force - jy+=0.5*Fy; - jz+=0.5*Fz; + //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2 - + 0.3333333333333333*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); nread = neighborList[n+Np]; dist[nread] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); nread = neighborList[n]; dist[nread] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+3*Np]; dist[nread] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+2*Np]; dist[nread] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+5*Np]; dist[nread] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - +0.05555555555555555*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+4*Np]; dist[nread] = fq; // q = 7 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 - //fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - +0.027777777777777776*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); nread = neighborList[n+16*Np]; dist[nread] = fq; //........................................................................ From 060bee7b4495324841b51d205e701fd05fb152fe Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 18 Jan 2020 22:52:04 -0500 Subject: [PATCH 016/270] GPU version of incompressible greysacle MRT model is ready --- cpu/Greyscale.cpp | 2 - gpu/Greyscale.cu | 1313 +++++++++++++++++++++++++++++++-------------- 2 files changed, 920 insertions(+), 395 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index 95cf516b..f2be769e 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -391,7 +391,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int double jx,jy,jz; // non-conserved moments double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double m3,m5,m7; double fq; //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) @@ -851,7 +850,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis double jx,jy,jz; // non-conserved moments double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double m3,m5,m7; double fq; //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index fdb0a462..5b8273fe 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -396,14 +396,17 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity, double Den){ + int n; double vx,vy,vz,v_mag; double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model // conserved momemnts - double rho,jx,jy,jz; + double jx,jy,jz; // non-conserved moments double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double m3,m5,m7; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) double porosity; double perm;//voxel permeability @@ -413,6 +416,20 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, double rlx_setA = rlx; double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; for (int s=0; s 10Np => odd part of dist) + fq = dist[nread]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nread]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nread = neighborList[n+5*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/Den+0.5*porosity*Gx; + vy = jy/Den+0.5*porosity*Gy; + vz = jz/Den+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = Den*(-porosity*mu/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = Den*(-porosity*mu/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = Den*(-porosity*mu/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=Den*Gx; + Fy=Den*Gy; + Fz=Den*Gz; + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) + + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) + + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) + + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) + + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) + + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) + + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) + + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) + + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + nread = neighborList[n+Np]; + dist[nread] = fq; + + // q=2 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + nread = neighborList[n]; + dist[nread] = fq; + + // q = 3 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + nread = neighborList[n+3*Np]; + dist[nread] = fq; + + // q = 4 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + nread = neighborList[n+2*Np]; + dist[nread] = fq; + + // q = 5 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + nread = neighborList[n+5*Np]; + dist[nread] = fq; + + // q = 6 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + nread = neighborList[n+4*Np]; + dist[nread] = fq; + + // q = 7 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + + } + } +} + extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity){ @@ -873,6 +1378,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis } extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity){ + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); cudaError_t err = cudaGetLastError(); @@ -880,3 +1386,24 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale: %s \n",cudaGetErrorString(err)); } } + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den){ + + dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den){ + + dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); + } +} + From 783d7ff7b2cf9027ab9f26d48fb5e5595c18fa81 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 20 Jan 2020 13:17:03 -0500 Subject: [PATCH 017/270] remove the reserved labels (i.e. Label=1,2) which store some pre-defined voxel perm values that may accidentally overwrite use-defined labels --- models/GreyscaleModel.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 1cdae815..ac44c6d3 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -194,10 +194,6 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm //Mask->id[n] = 0; // set mask to zero since this is an immobile component } } - // fluid labels are reserved / negative labels are immobile - if (VALUE == 1) POROSITY=1.0; - else if (VALUE == 2) POROSITY=1.0; - else if (VALUE < 1) POROSITY = 0.0; int idx = Map(i,j,k); if (!(idx < 0)){ if (POROSITY<=0.0){ @@ -228,19 +224,13 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm //Mask->id[n] = 0; // set mask to zero since this is an immobile component } } - // Permeability of fluid labels are reserved - // NOTE: the voxel permeability of apparent pore nodes should be infinity - // TODO: Need to revise the PERMEABILITY of nodes whose VALUE=1 and 2 - if (VALUE == 1) PERMEABILITY=1.0; - else if (VALUE == 2) PERMEABILITY=1.0; - else if (VALUE < 1) PERMEABILITY = 0.0; int idx = Map(i,j,k); if (!(idx < 0)){ if (PERMEABILITY<=0.0){ ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); } else{ - Permeability[idx] = PERMEABILITY; + Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; } } } @@ -254,13 +244,15 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm for (int idx=0; idxComm, label_count[idx]); if (rank==0){ + printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); printf("Component labels: %lu \n",NLABELS); for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); } } From fb33408a956bffbca4c20f8b6fc77f527686e1ee Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 20 Jan 2020 19:29:03 -0500 Subject: [PATCH 018/270] 1.disable debug write-out; 2. add a weighted porosity --- models/GreyscaleModel.cpp | 17 +++++++++++++---- models/GreyscaleModel.h | 1 + tests/lbpm_greyscale_simulator.cpp | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index ac44c6d3..b1f6603e 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -8,7 +8,7 @@ color lattice boltzmann model #include ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); @@ -243,6 +243,13 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm for (int idx=0; idxComm, label_count[idx]); + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); printf("Component labels: %lu \n",NLABELS); @@ -251,11 +258,12 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm POROSITY=PorosityList[idx]; PERMEABILITY=PermeabilityList[idx]; double volume_fraction = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs); - printf(" label=%d, porosity=%.3g, permeability=%.3g [um^2] (=%.3g [voxel^2]), volume fraction=%.3g\n", + printf(" label=%d: porosity=%.3g, permeability=%.3g [um^2] (=%.3g [voxel^2]), volume fraction=%.3g\n", VALUE,POROSITY,PERMEABILITY,PERMEABILITY/Dm->voxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); } - } @@ -497,7 +505,8 @@ void ScaLBL_GreyscaleModel::Run(){ Hs=sumReduce( Dm->Comm, Hs); Xs=sumReduce( Dm->Comm, Xs); double h = Dm->voxel_length; - double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; if (rank==0){ printf(" AbsPerm = %.5g [micron^2]\n",absperm); diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index ac939aed..b427218b 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -41,6 +41,7 @@ public: double Fx,Fy,Fz,flux; double din,dout; double dp;//solid particle diameter, unit in voxel + double GreyPorosity; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index 61322d6d..ef253cd7 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -55,7 +55,7 @@ int main(int argc, char **argv) Greyscale.Initialize(); // initializing the model will set initial conditions for variables Greyscale.Run(); Greyscale.VelocityField(); - Greyscale.WriteDebug(); + //Greyscale.WriteDebug(); } // **************************************************** MPI_Barrier(comm); From e3afe1eba80700d65b2b00e8ab9b65354ff201a5 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 21 Jan 2020 14:31:33 -0500 Subject: [PATCH 019/270] add a restart utitlity to the greyscale simulator --- models/GreyscaleModel.cpp | 95 ++++++++++++++++++++---------- tests/lbpm_greyscale_simulator.cpp | 2 +- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index b1f6603e..06077780 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -1,5 +1,5 @@ /* -color lattice boltzmann model +Greyscale lattice boltzmann model */ #include "models/GreyscaleModel.h" #include "analysis/distance.h" @@ -7,6 +7,12 @@ color lattice boltzmann model #include #include +template +void DeleteArray( const TYPE *p ) +{ + delete [] p; +} + ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) @@ -117,6 +123,7 @@ void ScaLBL_GreyscaleModel::ReadInput(){ Mask->Decomp(Filename); } else{ + if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); Mask->ReadIDs(); } for (int i=0; iid[i]; // save what was read @@ -357,39 +364,23 @@ void ScaLBL_GreyscaleModel::Create(){ void ScaLBL_GreyscaleModel::Initialize(){ - if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); - /* - * This function initializes model - */ + if (Restart == true){ if (rank==0){ - printf("Reading restart file! \n"); + printf("Initializing distributions from Restart! \n"); } - // Read in the restart file to CPU buffers - int *TmpMap; - TmpMap = new int[Np]; - - double *cDist; - cDist = new double[19*Np]; - ScaLBL_CopyToHost(TmpMap, dvcMap, Np*sizeof(int)); - - ifstream File(LocalRestartFile,ios::binary); - int idx; - double value; - for (int n=0; n cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + FILE *File; + File=fopen(LocalRestartFile,"rb"); + fread(cfq.get(),sizeof(double),19*Np,File); + fclose(File); + // Copy the restart data to the GPU - ScaLBL_CopyToDevice(fq,cDist,19*Np*sizeof(double)); + ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -400,6 +391,21 @@ void ScaLBL_GreyscaleModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int visualization_interval = 1000; + int restart_interval = 10000; // number of timesteps in between in saving distributions for restart + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "visualization_interval" )){ + visualization_interval = analysis_db->getScalar( "visualization_interval" ); + } + if (analysis_db->keyExists( "restart_interval" )){ + restart_interval = analysis_db->getScalar( "restart_interval" ); + } + if (greyscale_db->keyExists( "timestep" )){ + timestep = greyscale_db->getScalar( "timestep" ); + } if (rank==0){ printf("********************************************************\n"); @@ -418,8 +424,7 @@ void ScaLBL_GreyscaleModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - //std::shared_ptr analysis_db; - timestep=0; + auto current_db = db->cloneDatabase(); double rlx = 1.0/tau; double error = 1.0; double flow_rate_previous = 0.0; @@ -443,7 +448,7 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ - if (timestep%1000==0){ + if (timestep%analysis_interval==0){ //ScaLBL_D3Q19_Momentum(fq,Velocity, Np); //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); @@ -518,7 +523,7 @@ void ScaLBL_GreyscaleModel::Run(){ WriteHeader=true; log_file = fopen("Permeability.csv","a"); if (WriteHeader) - fprintf(log_file,"timesteps Fx Fy Fz mu Vs As Hs Xs vax vay vaz absperm \n", + fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, @@ -526,7 +531,35 @@ void ScaLBL_GreyscaleModel::Run(){ fclose(log_file); } } + + if (timestep%visualization_interval==0){ + VelocityField(); + } + + if (timestep%restart_interval==0){ + //Use rank=0 write out Restart.db + if (rank==0) { + greyscale_db->putScalar("timestep",timestep); + greyscale_db->putScalar( "Restart", true ); + current_db->putDatabase("Greyscale", greyscale_db); + std::ofstream OutStream("Restart.db"); + current_db->print(OutStream, ""); + OutStream.close(); + + } + //Write out Restart data. + std::shared_ptr cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU + + FILE *RESTARTFILE; + RESTARTFILE=fopen(LocalRestartFile,"wb"); + fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); + fclose(RESTARTFILE); + MPI_Barrier(comm); + } } + PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_greyscale_simulator",1); //************************************************************************ diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index ef253cd7..b7ed442e 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -54,7 +54,7 @@ int main(int argc, char **argv) Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables Greyscale.Initialize(); // initializing the model will set initial conditions for variables Greyscale.Run(); - Greyscale.VelocityField(); + //Greyscale.VelocityField(); //Greyscale.WriteDebug(); } // **************************************************** From 0372b9d1e8379ba0e2c3e4f14aeb6eece9946c77 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 21 Jan 2020 23:24:10 -0500 Subject: [PATCH 020/270] save the work, update how flow_rate is computed in greyscale simulator --- models/GreyscaleModel.cpp | 36 ++++++++++++++++++++---------------- models/GreyscaleModel.h | 1 + 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 06077780..36f853b1 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -99,6 +99,7 @@ void ScaLBL_GreyscaleModel::SetDomain(){ Velocity_x.resize(Nx,Ny,Nz); Velocity_y.resize(Nx,Ny,Nz); Velocity_z.resize(Nx,Ny,Nz); + PorosityMap.resize(Nx,Ny,Nz); id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way @@ -449,37 +450,40 @@ void ScaLBL_GreyscaleModel::Run(){ //************************************************************************/ if (timestep%analysis_interval==0){ - //ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); double count_loc=0; double count; double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - vax_loc = vay_loc = vaz_loc = 0.f; + double px_loc,py_loc,pz_loc; + double px,py,pz; + double mass_loc,mass_glb; + + px_loc = py_loc = pz_loc = 0.f; + mass_loc = 0.f; for (int k=1; k 0){ - vax_loc += Velocity_x(i,j,k); - vay_loc += Velocity_y(i,j,k); - vaz_loc += Velocity_z(i,j,k); - count_loc+=1.0; + px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); + py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); + pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); + mass_loc += Den*PorosityMap(i,j,k); } } } } - MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - vax /= count; - vay /= count; - vaz /= count; + vax = px/mass_glb; + vay = py/mass_glb; + vaz = pz/mass_glb; double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); double dir_x = Fx/force_mag; @@ -492,7 +496,7 @@ void ScaLBL_GreyscaleModel::Run(){ dir_z = 1.0; force_mag = 1.0; } - double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); flow_rate_previous = flow_rate; diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index b427218b..d1399053 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -71,6 +71,7 @@ public: DoubleArray Velocity_x; DoubleArray Velocity_y; DoubleArray Velocity_z; + DoubleArray PorosityMap; private: MPI_Comm comm; From 2a66e63672084b687677c891dc4b44001f92188b Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 29 Jan 2020 17:14:48 -0500 Subject: [PATCH 021/270] add pressure BC for abs-perm simulator; need validation test for this --- models/GreyscaleModel.cpp | 62 +++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 36f853b1..0499951f 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -44,7 +44,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ flux=0.0; dp = 10.0; //unit of 'dp': voxel - // Greyscale Model parameters + // ---------------------- Greyscale Model parameters -----------------------// if (greyscale_db->keyExists( "timestepMax" )){ timestepMax = greyscale_db->getScalar( "timestepMax" ); } @@ -77,10 +77,14 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ if (greyscale_db->keyExists( "tolerance" )){ tolerance = greyscale_db->getScalar( "tolerance" ); } + // ------------------------------------------------------------------------// + + //------------------------ Other Domain parameters ------------------------// BoundaryCondition = 0; if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } + // ------------------------------------------------------------------------// } void ScaLBL_GreyscaleModel::SetDomain(){ @@ -366,6 +370,9 @@ void ScaLBL_GreyscaleModel::Create(){ void ScaLBL_GreyscaleModel::Initialize(){ if (rank==0) printf ("Initializing distributions \n"); + //TODO: for BGK, you need to consider voxel porosity + // for IMRT, the whole set of feq is different + // if in the future you have different collison mode, need to write two set of initialization functions ScaLBL_D3Q19_Init(fq, Np); if (Restart == true){ @@ -431,21 +438,36 @@ void ScaLBL_GreyscaleModel::Run(){ double flow_rate_previous = 0.0; while (timestep < timestepMax && error > tolerance) { //************************************************************************/ + // *************ODD TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - //ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - //ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); + //ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -461,12 +483,28 @@ void ScaLBL_GreyscaleModel::Run(){ double px_loc,py_loc,pz_loc; double px,py,pz; double mass_loc,mass_glb; + + //parameters for domain average + int64_t i,j,k,n,imin,jmin,kmin,kmax; + // If external boundary conditions are set, do not average over the inlet and outlet + kmin=1; kmax=Nz-1; + //In case user forgets to specify the inlet/outlet buffer layers for BC>0 + if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; + if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; + + imin=jmin=1; + // If inlet/outlet layers exist use these as default + //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; + //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; + if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer + if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; + px_loc = py_loc = pz_loc = 0.f; mass_loc = 0.f; - for (int k=1; k 0){ px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); From 6e7cb832546f5064551019cc713e8f7c63ee9203 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 29 Jan 2020 23:49:36 -0500 Subject: [PATCH 022/270] add pressure to output data --- common/ScaLBL.h | 8 ++++---- cpu/Greyscale.cpp | 20 ++++++++++++++++---- gpu/Greyscale.cu | 36 ++++++++++++++++++++++++------------ models/GreyscaleModel.cpp | 35 ++++++++++++++++++++++++++--------- models/GreyscaleModel.h | 3 ++- 5 files changed, 72 insertions(+), 30 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index d2495e3f..04cfbd97 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -57,16 +57,16 @@ extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int star // GREYSCALE MODEL extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, - double *Poros,double *Perm, double *Velocity); + double *Poros,double *Perm, double *Velocity,double *Pressure); extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, - double *Poros,double *Perm, double *Velocity); + double *Poros,double *Perm, double *Velocity,double *Pressure); extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, - double *Poros,double *Perm, double *Velocity,double Den); + double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, - double *Poros,double *Perm, double *Velocity,double Den); + double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); // MRT MODEL diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index f2be769e..d1bde7f2 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1,11 +1,12 @@ #include extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity){ + double *Poros,double *Perm, double *Velocity, double *Pressure){ int n; // conserved momemnts double rho,vx,vy,vz,v_mag; double ux,uy,uz,u_mag; + double pressure; //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; @@ -48,6 +49,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + pressure = rho/porosity/3.0; vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; @@ -159,15 +161,18 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity){ + double *Poros,double *Perm, double *Velocity,double *Pressure){ int n; // conserved momemnts double rho,vx,vy,vz,v_mag; double ux,uy,uz,u_mag; + double pressure; //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; @@ -266,6 +271,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + pressure = rho/porosity/3.0; vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; @@ -377,12 +383,14 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity, double Den){ + double *Poros,double *Perm, double *Velocity, double Den,double *Pressure){ int n; double vx,vy,vz,v_mag; double ux,uy,uz,u_mag; @@ -837,11 +845,13 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity, double Den){ + double *Poros,double *Perm, double *Velocity, double Den,double *Pressure){ int n, nread; double vx,vy,vz,v_mag; double ux,uy,uz,u_mag; @@ -1332,6 +1342,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 5b8273fe..12ef6f17 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -4,11 +4,12 @@ #define NTHREADS 256 __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity){ + double *Poros,double *Perm, double *Velocity, double *Pressure){ int n; // conserved momemnts double rho,vx,vy,vz,v_mag; double ux,uy,uz,u_mag; + double pressure; //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; @@ -56,6 +57,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int f if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + pressure = rho/porosity/3.0; vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; @@ -167,17 +169,20 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int f Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } } __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity){ + double *Poros,double *Perm, double *Velocity, double *Pressure){ int n; // conserved momemnts double rho,vx,vy,vz,v_mag; double ux,uy,uz,u_mag; + double pressure; //double uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; @@ -279,6 +284,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + pressure = rho/porosity/3.0; vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; @@ -390,12 +396,14 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } } __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity, double Den){ + double *Poros,double *Perm, double *Velocity, double Den, double *Pressure){ int n; double vx,vy,vz,v_mag; @@ -857,6 +865,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } @@ -864,7 +874,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double Den){ + double *Poros,double *Perm, double *Velocity,double Den, double *Pressure){ int n, nread; double vx,vy,vz,v_mag; @@ -1361,15 +1371,17 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; } } } -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity){ +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double *Pressure){ - dvc_ScaLBL_D3Q19_AAeven_Greyscale<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); + dvc_ScaLBL_D3Q19_AAeven_Greyscale<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1377,9 +1389,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis } } -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity){ +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double *Pressure){ - dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity); + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1387,9 +1399,9 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in } } -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den){ +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ - dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den); + dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1397,9 +1409,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int } } -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den){ +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ - dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den); + dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 0499951f..018af5ec 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -104,6 +104,7 @@ void ScaLBL_GreyscaleModel::SetDomain(){ Velocity_y.resize(Nx,Ny,Nz); Velocity_z.resize(Nx,Ny,Nz); PorosityMap.resize(Nx,Ny,Nz); + Pressure.resize(Nx,Ny,Nz); id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way @@ -320,7 +321,7 @@ void ScaLBL_GreyscaleModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures @@ -441,8 +442,8 @@ void ScaLBL_GreyscaleModel::Run(){ // *************ODD TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - //ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs @@ -450,15 +451,15 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - //ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - //ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + //ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs @@ -466,8 +467,8 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity); - //ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den); + //ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -476,6 +477,7 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); + //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); double count_loc=0; double count; @@ -678,6 +680,7 @@ void ScaLBL_GreyscaleModel::VelocityField(){ auto VyVar = std::make_shared(); auto VzVar = std::make_shared(); auto SignDistVar = std::make_shared(); + auto PressureVar = std::make_shared(); IO::initialize("","silo","false"); // Create the MeshDataStruct @@ -706,20 +709,34 @@ void ScaLBL_GreyscaleModel::VelocityField(){ VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); visData[0].vars.push_back(VzVar); + PressureVar->name = "Pressure"; + PressureVar->type = IO::VariableType::VolumeVariable; + PressureVar->dim = 1; + PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PressureVar); + Array& SignData = visData[0].vars[0]->data; Array& VelxData = visData[0].vars[1]->data; Array& VelyData = visData[0].vars[2]->data; Array& VelzData = visData[0].vars[3]->data; + Array& PressureData = visData[0].vars[4]->data; ASSERT(visData[0].vars[0]->name=="SignDist"); ASSERT(visData[0].vars[1]->name=="Velocity_x"); ASSERT(visData[0].vars[2]->name=="Velocity_y"); ASSERT(visData[0].vars[3]->name=="Velocity_z"); + ASSERT(visData[0].vars[4]->name=="Pressure"); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + fillData.copy(SignDist,SignData); fillData.copy(Velocity_x,VelxData); fillData.copy(Velocity_y,VelyData); fillData.copy(Velocity_z,VelzData); + fillData.copy(Pressure,PressureData); IO::writeData( timestep, visData, Dm->Comm ); diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index d1399053..a7a5f528 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -65,13 +65,14 @@ public: double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; - double *Pressure; + double *Pressure_dvc; IntArray Map; DoubleArray SignDist; DoubleArray Velocity_x; DoubleArray Velocity_y; DoubleArray Velocity_z; DoubleArray PorosityMap; + DoubleArray Pressure; private: MPI_Comm comm; From 25df1e0f3522ffcd1ad51e3de90892e2894ac51c Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 30 Jan 2020 13:23:27 -0500 Subject: [PATCH 023/270] add a few print-out to make the program output more verbose --- common/Domain.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 48bfed15..1be64859 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -434,7 +434,7 @@ void Domain::Decomp(std::string Filename) } if (inlet_layers_z > 0){ - printf("Checkerboard pattern at z inlet for %i layers \n",inlet_layers_z); + printf("Checkerboard pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); // use checkerboard pattern for (int k = zStart; k < zStart+inlet_layers_z; k++){ for (int j = 0; j 0){ - printf("Checkerboard pattern at z outlet for %i layers \n",outlet_layers_z); + printf("Checkerboard pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); // use checkerboard pattern for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ for (int j = 0; j 0 && kproc() == 0){ - if (inlet_layers_z < 4) inlet_layers_z=4; + if (inlet_layers_z < 4){ + inlet_layers_z=4; + if(RANK==0){ + printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n"); + printf(" the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i",inlet_layers_z-1,inlet_layers_phase); + } + } for (int k=0; k 0 && kproc() == nprocz-1){ - if (outlet_layers_z < 4) outlet_layers_z=4; + if (outlet_layers_z < 4){ + outlet_layers_z=4; + if(RANK==0){ + printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n"); + printf(" the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i",outlet_layers_z-1,outlet_layers_phase); + } + } for (int k=Nz-outlet_layers_z; k Date: Thu, 30 Jan 2020 17:57:56 -0500 Subject: [PATCH 024/270] fix printf bug --- common/Domain.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 1be64859..1028a0ef 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -597,8 +597,7 @@ void Domain::Decomp(std::string Filename) if (inlet_layers_z < 4){ inlet_layers_z=4; if(RANK==0){ - printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n"); - printf(" the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i",inlet_layers_z-1,inlet_layers_phase); + printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase); } } for (int k=0; k 0 && kproc() == nprocz-1){ if (outlet_layers_z < 4){ outlet_layers_z=4; - if(RANK==0){ - printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n"); - printf(" the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i",outlet_layers_z-1,outlet_layers_phase); + if(RANK==nprocs-1){ + printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase); } } for (int k=Nz-outlet_layers_z; k Date: Fri, 31 Jan 2020 15:15:26 -0500 Subject: [PATCH 025/270] Some updates:(1)add different fq initialization for BGK and IMRT;(2)user can choose collision model --- common/ScaLBL.h | 4 +++ cpu/D3Q19.cpp | 27 ++++++++++++++ gpu/D3Q19.cu | 40 ++++++++++++++++++++- models/GreyscaleModel.cpp | 76 +++++++++++++++++++++++++++++++++------ models/GreyscaleModel.h | 1 + 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 04cfbd97..007fda34 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -46,6 +46,7 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np); + extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np); extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np); @@ -56,6 +57,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); // GREYSCALE MODEL + +extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den); + extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double *Pressure); diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 2af59883..244bb3d2 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -84,6 +84,33 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np) } } + +extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den) +{ + int n; + for (n=0; n>>(dist, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ + dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); } } diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 018af5ec..5f8e4e36 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -43,6 +43,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel + CollisionType = 1; //1: IMRT; 2: BGK // ---------------------- Greyscale Model parameters -----------------------// if (greyscale_db->keyExists( "timestepMax" )){ @@ -77,6 +78,10 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ if (greyscale_db->keyExists( "tolerance" )){ tolerance = greyscale_db->getScalar( "tolerance" ); } + auto collision = greyscale_db->getWithDefault( "collision", "IMRT" ); + if (collision == "BGK"){ + CollisionType=2; + } // ------------------------------------------------------------------------// //------------------------ Other Domain parameters ------------------------// @@ -374,7 +379,20 @@ void ScaLBL_GreyscaleModel::Initialize(){ //TODO: for BGK, you need to consider voxel porosity // for IMRT, the whole set of feq is different // if in the future you have different collison mode, need to write two set of initialization functions - ScaLBL_D3Q19_Init(fq, Np); + if (CollisionType==1){ + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + if (rank==0) printf("Collision model: Incompressible MRT.\n"); + } + else if (CollisionType==2){ + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf("Collision model: BGK.\n"); + } + else{ + if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + CollisionType=1; + greyscale_db->putScalar( "collision", "IMRT" ); + } if (Restart == true){ if (rank==0){ @@ -442,8 +460,17 @@ void ScaLBL_GreyscaleModel::Run(){ // *************ODD TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs @@ -451,25 +478,52 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - //ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - //ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs if (BoundaryCondition == 3){ ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - //ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ if (timestep%analysis_interval==0){ diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index a7a5f528..792e87ea 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -35,6 +35,7 @@ public: bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; + int CollisionType; double tau; double Den;//constant density double tolerance; From ea8fceda8c40f7e904c6999aba617d2e20a78451 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 31 Jan 2020 15:42:27 -0500 Subject: [PATCH 026/270] revert to the old velocity averaging method as it is more accurate --- models/GreyscaleModel.cpp | 63 ++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 5f8e4e36..4b803272 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -530,15 +530,16 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); + //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); double count_loc=0; double count; double vax,vay,vaz; - double px_loc,py_loc,pz_loc; - double px,py,pz; - double mass_loc,mass_glb; + double vax_loc,vay_loc,vaz_loc; + //double px_loc,py_loc,pz_loc; + //double px,py,pz; + //double mass_loc,mass_glb; //parameters for domain average int64_t i,j,k,n,imin,jmin,kmin,kmax; @@ -555,30 +556,51 @@ void ScaLBL_GreyscaleModel::Run(){ if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; - - px_loc = py_loc = pz_loc = 0.f; - mass_loc = 0.f; +// px_loc = py_loc = pz_loc = 0.f; +// mass_loc = 0.f; +// for (int k=kmin; k 0){ +// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +// mass_loc += Den*PorosityMap(i,j,k); +// } +// } +// } +// } +// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// +// vax = px/mass_glb; +// vay = py/mass_glb; +// vaz = pz/mass_glb; + + vax_loc = vay_loc = vaz_loc = 0.f; for (int k=kmin; k 0){ - px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); - py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); - pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); - mass_loc += Den*PorosityMap(i,j,k); + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; } } } } - MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - - vax = px/mass_glb; - vay = py/mass_glb; - vaz = pz/mass_glb; + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + vax /= count; + vay /= count; + vaz /= count; + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); double dir_x = Fx/force_mag; double dir_y = Fy/force_mag; @@ -590,7 +612,8 @@ void ScaLBL_GreyscaleModel::Run(){ dir_z = 1.0; force_mag = 1.0; } - double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; + //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); flow_rate_previous = flow_rate; From 50e4b5a9baf3a2767c1b69e8ef70a5a4377dbd48 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 1 Feb 2020 14:04:39 -0500 Subject: [PATCH 027/270] add the greyscale effective viscosity back, but by default it is set equal to the normal viscosity --- common/ScaLBL.h | 8 ++--- cpu/Greyscale.cpp | 48 ++++++++++++++--------------- gpu/Greyscale.cu | 64 +++++++++++++++++++-------------------- models/GreyscaleModel.cpp | 29 ++++++++++-------- models/GreyscaleModel.h | 1 + 5 files changed, 77 insertions(+), 73 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 007fda34..447a9b14 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -60,16 +60,16 @@ extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int star extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den); -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz, +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index d1bde7f2..16fad1e0 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1,6 +1,6 @@ #include -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double Gx, double Gy, double Gz, +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity, double *Pressure){ int n; // conserved momemnts @@ -14,7 +14,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis double porosity; double perm;//voxel permeability double c0, c1; //Guo's model parameters - double mu = (1.0/rlx-0.5)/3.0;//kinematic viscosity + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) for (int n=start; n>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + dvc_ScaLBL_D3Q19_AAeven_Greyscale<<>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1389,9 +1389,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis } } -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double *Pressure){ +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double *Pressure){ - dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1399,9 +1399,9 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in } } -extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ - dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); + dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1409,9 +1409,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int } } -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ - dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); + dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 4b803272..79b7a9c7 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -14,7 +14,7 @@ void DeleteArray( const TYPE *p ) } ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); @@ -36,6 +36,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ // set defaults timestepMax = 100000; tau = 1.0; + tau_eff = tau; Den = 1.0;//constant density tolerance = 0.01; Fx = Fy = Fz = 0.0; @@ -52,6 +53,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ if (greyscale_db->keyExists( "tau" )){ tau = greyscale_db->getScalar( "tau" ); } + tau_eff = greyscale_db->getWithDefault( "tau_eff", tau ); if (greyscale_db->keyExists( "Den" )){ Den = greyscale_db->getScalar( "Den" ); } @@ -453,6 +455,7 @@ void ScaLBL_GreyscaleModel::Run(){ PROFILE_START("Loop"); auto current_db = db->cloneDatabase(); double rlx = 1.0/tau; + double rlx_eff = 1.0/tau_eff; double error = 1.0; double flow_rate_previous = 0.0; while (timestep < timestepMax && error > tolerance) { @@ -462,13 +465,13 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL switch (CollisionType){ case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; } ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE @@ -480,13 +483,13 @@ void ScaLBL_GreyscaleModel::Run(){ } switch (CollisionType){ case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -496,13 +499,13 @@ void ScaLBL_GreyscaleModel::Run(){ ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL switch (CollisionType){ case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; } ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE @@ -514,13 +517,13 @@ void ScaLBL_GreyscaleModel::Run(){ } switch (CollisionType){ case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 792e87ea..c670239f 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -37,6 +37,7 @@ public: int BoundaryCondition; int CollisionType; double tau; + double tau_eff; double Den;//constant density double tolerance; double Fx,Fy,Fz,flux; From 793d294aa33550aa7cefc063f321306bea2c512b Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 1 Feb 2020 17:03:42 -0500 Subject: [PATCH 028/270] CPU version update: remove the higher-order terms in body force --- cpu/Greyscale.cpp | 422 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 320 insertions(+), 102 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index 16fad1e0..b4b017c8 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -69,94 +69,173 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finis Fz=Gz; } + //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// +// // q=0 +// dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 1 +// dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q=2 +// dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 3 +// dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 4 +// dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 5 +// dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); +// +// // q = 6 +// dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); +// +// // q = 7 +// dist[7*Np+n] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 8 +// dist[8*Np+n] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 9 +// dist[9*Np+n] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 10 +// dist[10*Np+n] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 11 +// dist[11*Np+n] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); +// +// // q = 12 +// dist[12*Np+n] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + +// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 13 +// dist[13*Np+n] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + +// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q= 14 +// dist[14*Np+n] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); +// +// // q = 15 +// dist[15*Np+n] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); +// +// // q = 16 +// dist[16*Np+n] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + +// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 17 +// dist[17*Np+n] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + +// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 18 +// dist[18*Np+n] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + //----------------------------------------------------------------------------------------------------------------------------------------// + + //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// // q=0 - dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); // q = 1 dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); // q=2 dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); // q = 3 dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); // q = 4 dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); // q = 5 dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); // q = 6 dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); // q = 7 dist[7*Np+n] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); // q = 8 dist[8*Np+n] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); // q = 9 dist[9*Np+n] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); // q = 10 dist[10*Np+n] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); // q = 11 dist[11*Np+n] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); // q = 12 dist[12*Np+n] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); // q = 13 dist[13*Np+n] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); // q= 14 dist[14*Np+n] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); // q = 15 dist[15*Np+n] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); // q = 16 dist[16*Np+n] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); // q = 17 dist[17*Np+n] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); // q = 18 dist[18*Np+n] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); + //-------------------------------------------------------------------------------------------------------------------------------------------// + //Update velocity on device Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; @@ -291,93 +370,176 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, in Fz=Gz; } + //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// +// // q=0 +// dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 1 +// dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q=2 +// dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 3 +// dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 4 +// dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 5 +// dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); +// +// // q = 6 +// dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); +// +// // q = 7 +// dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 8 +// dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 9 +// dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 10 +// dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 11 +// dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); +// +// // q = 12 +// dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + +// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 13 +// dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + +// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q= 14 +// dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); +// +// // q = 15 +// dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); +// +// // q = 16 +// dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + +// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 17 +// dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + +// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 18 +// dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + //----------------------------------------------------------------------------------------------------------------------------------------// + + + + //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// // q=0 - dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); // q = 1 dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); // q=2 dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); // q = 3 dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); // q = 4 dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); // q = 5 dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); // q = 6 dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); // q = 7 dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); // q = 8 dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); // q = 9 dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); // q = 10 dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); // q = 11 dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); // q = 12 dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); // q = 13 dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); // q= 14 dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); // q = 15 dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); // q = 16 dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); // q = 17 dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); // q = 18 dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); + //-------------------------------------------------------------------------------------------------------------------------------------------// + + //Update velocity on device Velocity[0*Np+n] = ux; @@ -730,11 +892,45 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int //Calculate pressure for Incompressible-MRT model pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + //-------------------- IMRT collison where body force has higher-order terms -------------// +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) - + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -744,25 +940,19 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int jz = jz + Fz; m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) - + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) - + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) - + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) - + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) - + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) - + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) - + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); //....................................................................................................... + //.................inverse transformation...................................................... // q=0 fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; @@ -1209,11 +1399,45 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis //Calculate pressure for Incompressible-MRT model pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + //-------------------- IMRT collison where body force has higher-order terms -------------// +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) - + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -1223,25 +1447,19 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis jz = jz + Fz; m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) - + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) - + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) - + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) - + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) - + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) - + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) - + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); //....................................................................................................... - + + //.................inverse transformation...................................................... // q=0 fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; From 46b8c1de7fb0253c3a14414a5b1036f0a3f4b972 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 1 Feb 2020 17:22:13 -0500 Subject: [PATCH 029/270] GPU version update: remove higher-order terms in body force --- gpu/Greyscale.cu | 414 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 313 insertions(+), 101 deletions(-) diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index d3fd52ab..0a9a63e0 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -77,93 +77,173 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int f Fz=Gz; } + //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// +// // q=0 +// dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 1 +// dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q=2 +// dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 3 +// dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 4 +// dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 5 +// dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); +// +// // q = 6 +// dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); +// +// // q = 7 +// dist[7*Np+n] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 8 +// dist[8*Np+n] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 9 +// dist[9*Np+n] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 10 +// dist[10*Np+n] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 11 +// dist[11*Np+n] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); +// +// // q = 12 +// dist[12*Np+n] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + +// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 13 +// dist[13*Np+n] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + +// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q= 14 +// dist[14*Np+n] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); +// +// // q = 15 +// dist[15*Np+n] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); +// +// // q = 16 +// dist[16*Np+n] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + +// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 17 +// dist[17*Np+n] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + +// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 18 +// dist[18*Np+n] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + //----------------------------------------------------------------------------------------------------------------------------------------// + + + //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// // q=0 - dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + dist[n] = f0*(1.0-rlx)+ rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); // q = 1 dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); // q=2 dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); // q = 3 dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); // q = 4 dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); // q = 5 dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); // q = 6 dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); // q = 7 dist[7*Np+n] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); // q = 8 dist[8*Np+n] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); // q = 9 dist[9*Np+n] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); // q = 10 dist[10*Np+n] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); // q = 11 dist[11*Np+n] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); // q = 12 dist[12*Np+n] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); // q = 13 dist[13*Np+n] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); // q= 14 dist[14*Np+n] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); // q = 15 dist[15*Np+n] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); // q = 16 dist[16*Np+n] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); // q = 17 dist[17*Np+n] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); // q = 18 dist[18*Np+n] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); + //-------------------------------------------------------------------------------------------------------------------------------------------// //Update velocity on device Velocity[0*Np+n] = ux; @@ -304,93 +384,174 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist Fz=Gz; } + //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// +// // q=0 +// dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 1 +// dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q=2 +// dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 3 +// dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 4 +// dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 5 +// dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); +// +// // q = 6 +// dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); +// +// // q = 7 +// dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 8 +// dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 9 +// dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 10 +// dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 11 +// dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); +// +// // q = 12 +// dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + +// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 13 +// dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + +// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q= 14 +// dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); +// +// // q = 15 +// dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); +// +// // q = 16 +// dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + +// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 17 +// dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + +// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 18 +// dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + //----------------------------------------------------------------------------------------------------------------------------------------// + + + //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// // q=0 - dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); // q = 1 dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); // q=2 dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); // q = 3 dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); // q = 4 dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); // q = 5 dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); // q = 6 dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); // q = 7 dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); // q = 8 dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); // q = 9 dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); // q = 10 dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - Fz*(0. - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); // q = 11 dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); // q = 12 dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); // q = 13 dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); // q= 14 dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); // q = 15 dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); // q = 16 dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); // q = 17 dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); // q = 18 dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); + //-------------------------------------------------------------------------------------------------------------------------------------------// + //Update velocity on device Velocity[0*Np+n] = ux; @@ -750,11 +911,43 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, //Calculate pressure for Incompressible-MRT model pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) - + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -764,20 +957,13 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, jz = jz + Fz; m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) - + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) - + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) - + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) - + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) - + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) - + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) - + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); @@ -1238,11 +1424,43 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double //Calculate pressure for Incompressible-MRT model pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) - + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -1252,25 +1470,19 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double jz = jz + Fz; m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) - + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) - + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) - + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) - + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) - + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) - + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) - + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); //....................................................................................................... - + + //.................inverse transformation...................................................... // q=0 fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; From b73208b4718604c4e95a0593bd83531d9b9b9971 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Mon, 3 Feb 2020 14:05:23 -0500 Subject: [PATCH 030/270] fix water seed --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 5a9c56d4..3b58fff1 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -937,7 +937,7 @@ void ScaLBL_ColorModel::Run(){ else if (USE_SEED){ delta_volume = volA*Dm->Volume - initial_volume; CURRENT_MORPH_TIMESTEPS += analysis_interval; - //double massChange = SeedPhaseField(seed_water); + double massChange = SeedPhaseField(seed_water); if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", seed_water, delta_volume, delta_volume_target); } else if (USE_MORPHOPEN_OIL){ From c426aa7d1db9fd637f72308d88ed5b89158aeb83 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Mon, 3 Feb 2020 15:13:45 -0500 Subject: [PATCH 031/270] remove deprecated pressure BC routines --- common/ScaLBL.h | 5 -- cpu/D3Q19.cpp | 139 ------------------------------------------------ 2 files changed, 144 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index d7f012d1..610fce5d 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -101,11 +101,6 @@ extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double // BOUNDARY CONDITION ROUTINES -//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, -// int Nx, int Ny, int Nz); -//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, -// int Nx, int Ny, int Nz, int outlet); - extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np); extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np); diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index be081528..564eb96d 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -680,145 +680,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, } } -extern "C" void ScaLBL_D3Q19_Pressure_BC_z(int *list, double *dist, double din, int count, int Np) -{ - int n; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double ux,uy,uz; - double Cxz,Cyz; - - for (int idx=0; idx Date: Mon, 3 Feb 2020 17:22:13 -0500 Subject: [PATCH 032/270] add a weighting factor to the water seeding method --- models/ColorModel.cpp | 133 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 23 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index df4afab9..bcffa9df 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1202,35 +1202,26 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ double mass_loss =0.f; double count =0.f; double *Aq_tmp, *Bq_tmp; + double *Vel_tmp; - Aq_tmp = new double [7*Np]; - Bq_tmp = new double [7*Np]; + Aq_tmp = new double [7*Np]; + Bq_tmp = new double [7*Np]; + Vel_tmp = new double [3*Np]; ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Vel_tmp, Velocity, 7*Np*sizeof(double)); -/* for (int k=1; kgnb.Px+Averages->gwb.Px)/(Averages->gnb.M+Averages->gwb.M); + double vy_glb = (Averages->gnb.Py+Averages->gwb.Py)/(Averages->gnb.M+Averages->gwb.M); + double vz_glb = (Averages->gnb.Pz+Averages->gwb.Pz)/(Averages->gnb.M+Averages->gwb.M); + double v_mag_glb = sqrt(vx_glb*vx_glb+vy_glb*vy_glb+vz_glb*vz_glb); - if (Averages->SDs(i,j,k) < 0.f){ - // skip - } - else if (phase(i,j,k) > 0.f ){ - phase(i,j,k) -= random_value*seed_water_in_oil; - mass_loss += random_value*seed_water_in_oil; - count++; - } - else { - - } - } - } - } - */ for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ - double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); + double weight = (v_mag_localFirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ - double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); + double weight = (v_mag_localSDs(i,j,k) < 0.f){ +// // skip +// } +// else if (phase(i,j,k) > 0.f ){ +// phase(i,j,k) -= random_value*seed_water_in_oil; +// mass_loss += random_value*seed_water_in_oil; +// count++; +// } +// else { +// +// } +// } +// } +// } +// */ +// for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ +// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; +// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; +// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; +// double phase_id = (dA - dB) / (dA + dB); +// if (phase_id > 0.0){ +// Aq_tmp[n] -= 0.3333333333333333*random_value; +// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; +// +// Bq_tmp[n] += 0.3333333333333333*random_value; +// Bq_tmp[n+Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; +// } +// mass_loss += random_value*seed_water_in_oil; +// } +// +// for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ +// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; +// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; +// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; +// double phase_id = (dA - dB) / (dA + dB); +// if (phase_id > 0.0){ +// Aq_tmp[n] -= 0.3333333333333333*random_value; +// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; +// +// Bq_tmp[n] += 0.3333333333333333*random_value; +// Bq_tmp[n+Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; +// } +// mass_loss += random_value*seed_water_in_oil; +// } +// +// count = Dm->Comm.sumReduce( count ); +// mass_loss = Dm->Comm.sumReduce( mass_loss ); +// if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); +// +// // Need to initialize Aq, Bq, Den, Phi directly +// //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); +// +// return(mass_loss); +//} + double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); From a372d604503739f3e1564c7bec4bc683cfbb189a Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 4 Feb 2020 13:58:06 -0500 Subject: [PATCH 033/270] resolve some minor issues after the MPI backend updates --- models/GreyscaleModel.cpp | 24 +++++++++++++++--------- models/GreyscaleModel.h | 2 +- tests/lbpm_greyscale_simulator.cpp | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 79b7a9c7..11d92c80 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -261,7 +261,7 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm // Set Dm to match Mask for (int i=0; iid[i] = Mask->id[i]; - for (int idx=0; idxComm, label_count[idx]); + for (int idx=0; idxComm.sumReduce(label_count[idx]); //Initialize a weighted porosity after considering grey voxels GreyPorosity=0.0; @@ -595,11 +595,16 @@ void ScaLBL_GreyscaleModel::Run(){ } } } - MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + vax = Mask->Comm.sumReduce( vax_loc ); + vay = Mask->Comm.sumReduce( vay_loc ); + vaz = Mask->Comm.sumReduce( vaz_loc ); + count = Mask->Comm.sumReduce( count_loc ); + vax /= count; vay /= count; vaz /= count; @@ -629,10 +634,11 @@ void ScaLBL_GreyscaleModel::Run(){ double As = Morphology.A(); double Hs = Morphology.H(); double Xs = Morphology.X(); - Vs=sumReduce( Dm->Comm, Vs); - As=sumReduce( Dm->Comm, As); - Hs=sumReduce( Dm->Comm, Hs); - Xs=sumReduce( Dm->Comm, Xs); + Vs = Dm->Comm.sumReduce( Vs); + As = Dm->Comm.sumReduce( As); + Hs = Dm->Comm.sumReduce( Hs); + Xs = Dm->Comm.sumReduce( Xs); + double h = Dm->voxel_length; //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index c670239f..a99925b1 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -10,7 +10,7 @@ Implementation of color lattice boltzmann model #include #include "common/Communication.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "common/Database.h" #include "common/ScaLBL.h" #include "ProfilerApp.h" diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index b7ed442e..a54b6fc4 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -8,7 +8,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" -#include "common/MPI_Helpers.h" +#include "common/MPI.h" #include "models/GreyscaleModel.h" //#define WRITE_SURFACES From 6d4e68d8b8b870519862a07584e3cfcb3808b554 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 4 Feb 2020 14:02:49 -0500 Subject: [PATCH 034/270] set morphological target from kr --- models/ColorModel.cpp | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index bcffa9df..b8578f4e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -524,6 +524,24 @@ void ScaLBL_ColorModel::Run(){ int RESCALE_FORCE_COUNT = 0; int RESCALE_FORCE_MAX = 0; + /* history for morphological algoirthm */ + double KRA_MORPH_FACTOR=0.8; + double volA_prev = 0.0; + double log_krA_prev = 1.0; + double log_krA_target = 1.0; + double log_krA = 0.0; + double slope_krA_volume = 0.0; + if (color_db->keyExists( "vol_A_previous" )){ + volA_prev = color_db->getScalar( "vol_A_previous" ); + } + if (color_db->keyExists( "log_krA_previous" )){ + log_krA_prev = color_db->getScalar( "log_krA_previous" ); + } + if (color_db->keyExists( "krA_morph_factor" )){ + KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); + } + + /* defaults for simulation protocols */ auto protocol = color_db->getWithDefault( "protocol", "none" ); if (protocol == "image sequence"){ // Get the list of images @@ -811,7 +829,17 @@ void ScaLBL_ColorModel::Run(){ if ( isSteady ){ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + /** morphological target based on relative permeability for A **/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + log_krA = log(krA_TMP); + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); + log_krA_prev = log_krA; + volA_prev = volA; + printf(" ",log_krA, log_krA_target, vol_A, ); + /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); @@ -1279,7 +1307,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ // Need to initialize Aq, Bq, Den, Phi directly //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); return(mass_loss); } From 7569c9bf7315c11dcf5814fe7e8aad1c5bf09c00 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 12 Feb 2020 14:19:16 -0500 Subject: [PATCH 035/270] support for grid file in MRT model --- models/MRTModel.cpp | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 2ddba403..60847e54 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -3,6 +3,7 @@ */ #include "models/MRTModel.h" #include "analysis/distance.h" +#include "common/ReadMicroCT.h" ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), @@ -98,15 +99,29 @@ void ScaLBL_MRTModel::ReadInput(){ sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - if (domain_db->keyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask->Decomp(Filename); - } - else{ - Mask->ReadIDs(); - } + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, comm ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else{ + Mask->ReadIDs(); + } - // Generate the signed distance map + // Generate the signed distance map // Initialize the domain and communication Array id_solid(Nx,Ny,Nz); // Solve for the position of the solid phase From 46c407695620df3b45f3eabba073576b643c453f Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 17 Feb 2020 12:06:58 -0500 Subject: [PATCH 036/270] fix some typo --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b8578f4e..36c40224 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -838,7 +838,7 @@ void ScaLBL_ColorModel::Run(){ delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); log_krA_prev = log_krA; volA_prev = volA; - printf(" ",log_krA, log_krA_target, vol_A, ); + printf(" ",log_krA, log_krA_target, volA); /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); From 586bc09f842efac51a399a5842e5469399f8a4eb Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 21 Feb 2020 11:11:59 -0500 Subject: [PATCH 037/270] fix print bug --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b8578f4e..097d53e4 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -838,7 +838,7 @@ void ScaLBL_ColorModel::Run(){ delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); log_krA_prev = log_krA; volA_prev = volA; - printf(" ",log_krA, log_krA_target, vol_A, ); + printf(" log(kr)=%f, TARGET log(kr)=%f, volume=%f \n",log_krA, log_krA_target, vol_A, ); /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); From a42a0c84408d652d382cf8b5c66910a89718d44c Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 21 Feb 2020 11:16:26 -0500 Subject: [PATCH 038/270] fix print bug --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 097d53e4..1f695bed 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -838,7 +838,7 @@ void ScaLBL_ColorModel::Run(){ delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); log_krA_prev = log_krA; volA_prev = volA; - printf(" log(kr)=%f, TARGET log(kr)=%f, volume=%f \n",log_krA, log_krA_target, vol_A, ); + printf(" log(kr)=%f, TARGET log(kr)=%f, volume=%f \n",log_krA, log_krA_target, vol_A); /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); From b99d32ef0c120e9fa7d9399d7a68b4c4a5b85779 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 21 Feb 2020 11:28:18 -0500 Subject: [PATCH 039/270] fix print --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 82680a1c..8293e09f 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -838,7 +838,7 @@ void ScaLBL_ColorModel::Run(){ delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); log_krA_prev = log_krA; volA_prev = volA; - printf(" log(kr)=%f, TARGET log(kr)=%f, volume=%f \n",log_krA, log_krA_target, volA); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); From 81a25b99977f03f729fffd20d9d8938ca455eeb6 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 21 Feb 2020 11:43:58 -0500 Subject: [PATCH 040/270] try for better Ca target --- models/ColorModel.cpp | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 8293e09f..2c773c8e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -521,8 +521,6 @@ void ScaLBL_ColorModel::Run(){ double NOISE_THRESHOLD = 0.0; double BUMP_RATE = 2.0; bool USE_BUMP_RATE = false; - int RESCALE_FORCE_COUNT = 0; - int RESCALE_FORCE_MAX = 0; /* history for morphological algoirthm */ double KRA_MORPH_FACTOR=0.8; @@ -801,6 +799,20 @@ void ScaLBL_ColorModel::Run(){ double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); + if (SET_CAPILLARY_NUMBER && CURRENT_STEADY_TIMESTEPS%MIN_STEADY_TIMESTEPS < analysis_interval ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + if ( morph_timesteps > morph_interval ){ bool isSteady = false; @@ -808,23 +820,6 @@ void ScaLBL_ColorModel::Run(){ isSteady = true; if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; - - if (SET_CAPILLARY_NUMBER && RESCALE_FORCE_COUNT < RESCALE_FORCE_MAX){ - RESCALE_FORCE_COUNT++; - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } if ( isSteady ){ MORPH_ADAPT = true; @@ -913,7 +908,6 @@ void ScaLBL_ColorModel::Run(){ Fx *= capillary_number / Ca; Fy *= capillary_number / Ca; Fz *= capillary_number / Ca; - RESCALE_FORCE_COUNT = 1; if (force_mag > 1e-3){ Fx *= 1e-3/force_mag; // impose ceiling for stability Fy *= 1e-3/force_mag; @@ -933,6 +927,7 @@ void ScaLBL_ColorModel::Run(){ Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); color_db->putVector("F",{Fx,Fy,Fz}); } + CURRENT_STEADY_TIMESTEPS = 0; } else{ From 1694f4530ccaeb93aa484ad8883a320c0c714b4c Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 21 Feb 2020 21:22:54 -0500 Subject: [PATCH 041/270] comment out the variable rescale_force_count that was deprecated --- models/ColorModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 2c773c8e..3fef03d1 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -583,9 +583,9 @@ void ScaLBL_ColorModel::Run(){ SET_CAPILLARY_NUMBER=true; //RESCALE_FORCE_MAX = 1; } - if (analysis_db->keyExists( "rescale_force_count" )){ - RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); - } +// if (analysis_db->keyExists( "rescale_force_count" )){ +// RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); +// } if (color_db->keyExists( "timestep" )){ timestep = color_db->getScalar( "timestep" ); } From fa4a20ddf0c8e156d40dae2e3818a54ee54186de Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 24 Feb 2020 17:24:11 -0500 Subject: [PATCH 042/270] GPU standard MRT absperm simulator: move the body force execution from normal space to moment space --- gpu/D3Q19.cu | 231 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 181 insertions(+), 50 deletions(-) diff --git a/gpu/D3Q19.cu b/gpu/D3Q19.cu index ccd125b2..62212a17 100644 --- a/gpu/D3Q19.cu +++ b/gpu/D3Q19.cu @@ -786,11 +786,36 @@ dvc_ScaLBL_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); +// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); +// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); +// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); +// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); +// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); +// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); +// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); +// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); +// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); + + //Body force is executed in the moment space + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) + + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -809,117 +834,157 @@ dvc_ScaLBL_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int dist[n] = fq; // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); nread = neighborList[n+Np]; dist[nread] = fq; // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); nread = neighborList[n]; dist[nread] = fq; // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+3*Np]; dist[nread] = fq; // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+2*Np]; dist[nread] = fq; // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+5*Np]; dist[nread] = fq; // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+4*Np]; dist[nread] = fq; // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+ +// mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+ +// mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+ +// mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ +// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ - mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); nread = neighborList[n+16*Np]; dist[nread] = fq; @@ -1222,11 +1287,36 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); +// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); +// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); +// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); +// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); +// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); +// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); +// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); +// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); +// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); + + //Body force is executed in the moment space + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) + + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1237,6 +1327,7 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... //.................inverse transformation...................................................... @@ -1245,105 +1336,145 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se dist[n] = fq; // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); dist[1*Np+n] = fq; // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); dist[2*Np+n] = fq; // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[3*Np+n] = fq; // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[4*Np+n] = fq; // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[5*Np+n] = fq; // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[6*Np+n] = fq; // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + +// 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + - 0.08333333333*(Fx+Fy); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); dist[7*Np+n] = fq; // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); dist[8*Np+n] = fq; // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17)+ +// 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17)+ - 0.08333333333*(Fx-Fy); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); dist[9*Np+n] = fq; // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- +// 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- - 0.08333333333*(Fx-Fy); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); dist[10*Np+n] = fq; // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); dist[11*Np+n] = fq; // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ +// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18)- +// 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ - mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18)- - 0.08333333333*(Fx+Fz); + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); dist[12*Np+n] = fq; // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); dist[13*Np+n] = fq; // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); dist[14*Np+n] = fq; // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); dist[15*Np+n] = fq; // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); dist[16*Np+n] = fq; // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); dist[17*Np+n] = fq; // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); dist[18*Np+n] = fq; //........................................................................ } From 7405344dac2a59513e4434497dd17dd578502f5a Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 25 Feb 2020 14:19:23 -0500 Subject: [PATCH 043/270] CPU standard MRT absperm simulator: move the body force execution from normal space to moment space --- cpu/D3Q19.cpp | 231 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 185 insertions(+), 46 deletions(-) diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 2c0e686d..5bfe305f 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -1197,11 +1197,35 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); +// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); +// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); +// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); +// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); +// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); +// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); +// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); +// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); +// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); + + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) + + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1212,6 +1236,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... //.................inverse transformation...................................................... @@ -1220,105 +1245,149 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int dist[n] = fq; // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); dist[1*Np+n] = fq; // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); dist[2*Np+n] = fq; // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[3*Np+n] = fq; // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); dist[4*Np+n] = fq; // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[5*Np+n] = fq; // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); dist[6*Np+n] = fq; // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m16-m17); dist[7*Np+n] = fq; // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); dist[8*Np+n] = fq; // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + +mrt_V12*m12-0.25*m13+0.125*(m16+m17); dist[9*Np+n] = fq; // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + +mrt_V12*m12-0.25*m13-0.125*(m16+m17); dist[10*Np+n] = fq; // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); dist[11*Np+n] = fq; // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m16-m18); dist[12*Np+n] = fq; // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); dist[13*Np+n] = fq; // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); dist[14*Np+n] = fq; // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); dist[15*Np+n] = fq; // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); dist[16*Np+n] = fq; // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); dist[17*Np+n] = fq; // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); dist[18*Np+n] = fq; //........................................................................ @@ -1657,11 +1726,36 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); +// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); +// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); +// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); +// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); +// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); +// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); +// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); +// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); +// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); +// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); + + //Body force is executed in the moment space + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) + + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) + + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1672,6 +1766,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... //.................inverse transformation...................................................... @@ -1680,120 +1775,164 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star dist[n] = fq; // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); nread = neighborList[n+Np]; dist[nread] = fq; // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); nread = neighborList[n]; dist[nread] = fq; // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+3*Np]; dist[nread] = fq; // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); nread = neighborList[n+2*Np]; dist[nread] = fq; // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+5*Np]; dist[nread] = fq; // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); nread = neighborList[n+4*Np]; dist[nread] = fq; // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m16-m17); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + +mrt_V12*m12-0.25*m13+0.125*(m16+m17); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + +mrt_V12*m12-0.25*m13-0.125*(m16+m17); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + -mrt_V12*m12+0.25*m15+0.125*(m16-m18); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); nread = neighborList[n+16*Np]; dist[nread] = fq; From fa61d19095187f8fac5c1671ee756eedda565c4d Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 4 Mar 2020 14:50:53 -0500 Subject: [PATCH 044/270] Update helper functions to read input database --- example/Workflow/HelperFunctions.R | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/example/Workflow/HelperFunctions.R b/example/Workflow/HelperFunctions.R index 497cb262..6c8bd903 100644 --- a/example/Workflow/HelperFunctions.R +++ b/example/Workflow/HelperFunctions.R @@ -2,6 +2,31 @@ require("ggplot2") GG_THEME=theme_bw()+theme(panel.grid.major = element_blank(),panel.grid.minor = element_blank()) +ReadDatabase<-function(FILE){ + + INPUT<-gsub(';','',readLines(FILE)) + + S<-gsub('tauA = ','',gsub("\\s+"," ",(grep("tauA",INPUT,value=TRUE)))) + TAU_A = as.numeric(S) + S<-gsub('tauB = ','',gsub("\\s+"," ",(grep("tauB",INPUT,value=TRUE)))) + TAU_B = as.numeric(S) + S<-gsub('rhoA = ','',gsub("\\s+"," ",(grep("rhoA",INPUT,value=TRUE)))) + RHO_A = as.numeric(S) + S<-gsub('rhoB = ','',gsub("\\s+"," ",(grep("rhoB",INPUT,value=TRUE)))) + RHO_B = as.numeric(S) + + S<-gsub('alpha = ','',gsub("\\s+"," ",(grep("alpha",INPUT,value=TRUE)))) + ALPHA = as.numeric(S) + + # Read the affinity + S<-gsub('ComponentAffinity = ','',gsub("\\s+"," ",(grep("ComponentAffinity",INPUT,value=TRUE)))) + AFFINITY<-as.numeric(unlist(strsplit(S,", "))) + + PARAMETERS<-c(TAU_A,TAU_B,RHO_A,RHO_B,ALPHA,AFFINITY) + + return(PARAMETERS) +} + ReadSubphase<-function(PATH){ FILE=paste0(PATH,"/subphase.csv") S<-read.csv(FILE,head=TRUE,sep=" ") From 7bb01557d838a1231bc5780a5737f0af4da5d43c Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 17 Mar 2020 13:45:51 -0400 Subject: [PATCH 045/270] updated bugfix with old ScaLBL --- common/ScaLBL.cpp | 251 +++++++++++++++++++++++----------------------- common/ScaLBL.h | 9 +- 2 files changed, 134 insertions(+), 126 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6f2966e7..21656757 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -5,7 +5,9 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ Lock=false; // unlock the communicator //...................................................................................... // Create a separate copy of the communicator for the device - MPI_COMM_SCALBL = Dm->Comm.dup(); + //MPI_Comm_group(Dm->Comm,&Group); + //MPI_Comm_create(Dm->Comm,Group,&MPI_COMM_SCALBL); + MPI_Comm_dup(Dm->Comm,&MPI_COMM_SCALBL); //...................................................................................... // Copy the domain size and communication information directly from Dm Nx = Dm->Nx; @@ -213,7 +215,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int)); //...................................................................................... - MPI_COMM_SCALBL.barrier(); + MPI_Barrier(MPI_COMM_SCALBL); //................................................................................... // Set up the recieve distribution lists @@ -286,7 +288,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ //................................................................................... //...................................................................................... - MPI_COMM_SCALBL.barrier(); + MPI_Barrier(MPI_COMM_SCALBL); ScaLBL_DeviceBarrier(); //...................................................................................... SendCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+ @@ -363,7 +365,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis int idx,i,j,k,n; // Check that Map has size matching sub-domain - if ( (int) Map.size(0) != Nx) + if (Map.size(0) != Nx) ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Map array dimensions do not match! \n"); // Initialize Map @@ -867,8 +869,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,dist,N); ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,dist,N); - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); //...Packing for X face(1,7,9,11,13)................................ ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,dist,N); ScaLBL_D3Q19_Pack(7,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,dist,N); @@ -876,8 +878,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,dist,N); ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,dist,N); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); //...Packing for y face(4,8,9,16,18)................................. ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,dist,N); ScaLBL_D3Q19_Pack(8,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,dist,N); @@ -885,8 +887,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,dist,N); ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,dist,N); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,dist,N); ScaLBL_D3Q19_Pack(7,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); @@ -894,8 +896,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); //...Packing for z face(6,12,13,16,17)................................ ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,dist,N); ScaLBL_D3Q19_Pack(12,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,dist,N); @@ -903,8 +905,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,dist,N); ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,dist,N); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); //...Packing for Z face(5,11,14,15,18)................................ ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,dist,N); @@ -913,57 +915,57 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N); ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); //...Pack the xy edge (8)................................ ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,dist,N); - req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]); //...Pack the Xy edge (9)................................ ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,dist,N); - req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]); //...Pack the xY edge (10)................................ ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,dist,N); - req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]); //...Pack the XY edge (7)................................ ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,dist,N); - req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]); //...Pack the xz edge (12)................................ ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,dist,N); - req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]); //...Pack the xZ edge (14)................................ ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,dist,N); - req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]); //...Pack the Xz edge (13)................................ ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,dist,N); - req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]); //...Pack the XZ edge (11)................................ ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,dist,N); - req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]); //...Pack the yz edge (16)................................ ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,dist,N); - req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]); //...Pack the yZ edge (18)................................ ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,dist,N); - req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); //...Pack the Yz edge (17)................................ ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,dist,N); - req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); //...Pack the YZ edge (15)................................ ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,dist,N); - req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]); //................................................................................... } @@ -973,8 +975,8 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1057,8 +1059,8 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ // Recieves halo and incorporates into D3Q19 based stencil gradient computation //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1151,36 +1153,36 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N); ScaLBL_D3Q19_Pack(2,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,Bq,N); - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); //...Packing for X face(1,7,9,11,13)................................ ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N); ScaLBL_D3Q19_Pack(1,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,Bq,N); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); //...Packing for y face(4,8,9,16,18)................................. ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N); ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,Bq,N); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N); ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); //...Packing for z face(6,12,13,16,17)................................ ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N); ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,Bq,N); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); //...Packing for Z face(5,11,14,15,18)................................ ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N); @@ -1188,8 +1190,8 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ //................................................................................... // Send all the distributions - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); } @@ -1199,8 +1201,8 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(6,req1); - MPI_COMM_SCALBL.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1291,18 +1293,18 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){ //................................................................................... // Send all the distributions - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 3*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 3*recvCount_X,rank_X,recvtag); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 3*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 3*recvCount_x,rank_x,recvtag); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 3*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 3*recvCount_Y,rank_Y,recvtag); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 3*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 3*recvCount_y,rank_y,recvtag); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 3*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 3*recvCount_Z,rank_Z,recvtag); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 3*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 3*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 3*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 3*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); + MPI_Isend(sendbuf_X, 3*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 3*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); + MPI_Isend(sendbuf_y, 3*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 3*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); + MPI_Isend(sendbuf_Y, 3*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 3*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); + MPI_Isend(sendbuf_z, 3*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 3*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); + MPI_Isend(sendbuf_Z, 3*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 3*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); } @@ -1312,8 +1314,8 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(6,req1); - MPI_COMM_SCALBL.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1407,49 +1409,49 @@ void ScaLBL_Communicator::SendHalo(double *data){ // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); - req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); //................................................................................... } void ScaLBL_Communicator::RecvHalo(double *data){ //................................................................................... - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -1478,6 +1480,7 @@ void ScaLBL_Communicator::RecvHalo(double *data){ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray ®data){ // Gets data from the device and stores in regular layout + int i,j,k,n,idx; int Nx = map.size(0); int Ny = map.size(1); int Nz = map.size(2); @@ -1489,10 +1492,11 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double double value; TmpDat = new double [N]; ScaLBL_CopyToHost(&TmpDat[0],&data[0], N*sizeof(double)); - for (int k=0; k Date: Tue, 17 Mar 2020 21:23:18 -0400 Subject: [PATCH 046/270] Revert "Moving more MPI calls to the wrapper" This reverts commit 0f91767b6c870101084fbae0978280c04c85a004. --- IO/netcdf.cpp | 2 +- StackTrace/ErrorHandlers.h | 2 +- StackTrace/Utilities.cpp | 2 +- analysis/TwoPhase.cpp | 7 +- analysis/morphology.cpp | 132 ++--- common/Communication.h | 216 ++++---- common/Domain.cpp | 206 ++++---- common/Domain.h | 3 + common/MPI.I | 33 -- common/MPI.cpp | 48 -- common/MPI.h | 7 - common/ScaLBL.h | 1 + common/Utilities.cpp | 2 +- cpu/exe/lb2_Color_mpi.cpp | 538 +++++++++---------- cpu/exe/lb2_Color_wia_mpi_bubble.cpp | 711 ++++++++++++++------------ gpu/exe/lb1_MRT_mpi.cpp | 348 +++++++------ gpu/exe/lb1_MRT_mpi.cu | 352 +++++++------ gpu/exe/lb2_Color.cu | 65 ++- gpu/exe/lb2_Color_mpi.cpp | 541 ++++++++++---------- gpu/exe/lb2_Color_pBC_wia_mpi.cpp | 621 ++++++++++++---------- models/ColorModel.cpp | 8 +- models/DFHModel.cpp | 4 +- models/MRTModel.cpp | 4 +- tests/BlobAnalyzeParallel.cpp | 22 +- tests/GenerateSphereTest.cpp | 54 +- tests/TestBlobAnalyze.cpp | 28 +- tests/TestBubble.cpp | 41 +- tests/TestBubbleDFH.cpp | 4 +- tests/TestColorGrad.cpp | 20 +- tests/TestCommD3Q19.cpp | 4 +- tests/TestForceD3Q19.cpp | 4 +- tests/TestForceMoments.cpp | 4 +- tests/TestMRT.cpp | 28 +- tests/TestMicroCTReader.cpp | 1 + tests/TestMomentsD3Q19.cpp | 2 +- tests/TestNetcdf.cpp | 2 +- tests/TestSegDist.cpp | 4 +- tests/lb2_CMT_wia.cpp | 30 +- tests/lb2_Color_blob_wia_mpi.cpp | 427 ++++++++-------- tests/lbpm_BGK_simulator.cpp | 48 +- tests/lbpm_color_macro_simulator.cpp | 61 +-- tests/lbpm_disc_pp.cpp | 34 +- tests/lbpm_inkbottle_pp.cpp | 22 +- tests/lbpm_juanes_bench_disc_pp.cpp | 35 +- tests/lbpm_nondarcy_simulator.cpp | 52 +- tests/lbpm_nonnewtonian_simulator.cpp | 26 +- tests/lbpm_plates_pp.cpp | 24 +- tests/lbpm_porenetwork_pp.cpp | 25 +- tests/lbpm_random_pp.cpp | 92 ++-- tests/lbpm_segmented_decomp.cpp | 48 +- tests/lbpm_segmented_pp.cpp | 2 +- tests/lbpm_sphere_pp.cpp | 16 +- tests/lbpm_squaretube_pp.cpp | 25 +- 53 files changed, 2678 insertions(+), 2360 deletions(-) diff --git a/IO/netcdf.cpp b/IO/netcdf.cpp index 6c3773e3..e061579a 100644 --- a/IO/netcdf.cpp +++ b/IO/netcdf.cpp @@ -119,7 +119,7 @@ std::string VariableTypeName( VariableType type ) int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm ) { int fid = 0; - if ( comm.isNull() ) { + if ( comm == MPI_COMM_NULL ) { if ( mode == READ ) { int err = nc_open( filename.c_str(), NC_NOWRITE, &fid ); CHECK_NC_ERR( err ); diff --git a/StackTrace/ErrorHandlers.h b/StackTrace/ErrorHandlers.h index e43a4688..12b8d7de 100644 --- a/StackTrace/ErrorHandlers.h +++ b/StackTrace/ErrorHandlers.h @@ -6,7 +6,7 @@ #include -#include "common/MPI.h" +#include "mpi.h" namespace StackTrace diff --git a/StackTrace/Utilities.cpp b/StackTrace/Utilities.cpp index 5fb8e9b8..11f05777 100644 --- a/StackTrace/Utilities.cpp +++ b/StackTrace/Utilities.cpp @@ -14,7 +14,7 @@ #include #ifdef USE_MPI -#include "common/MPI.h" +#include "mpi.h" #endif #ifdef USE_TIMER diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index d878a663..812490e7 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -890,14 +890,14 @@ void TwoPhase::ComponentAverages() RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP); /* for (int b=0; bComm.barrier(); - Dm->Comm.sumReduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT); + MPI_Barrier(Dm->Comm); + MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); for (int idx=0; idxComm.barrier(); Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP); - // Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT); + // MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); if (Dm->rank()==0){ printf("rescaling... \n"); @@ -994,6 +994,7 @@ void TwoPhase::ComponentAverages() // reduce the wetting phase averages for (int b=0; bComm.barrier(); +// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT); for (int idx=0; idx PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag,recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag,recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -285,7 +303,7 @@ double morph_open() fillHalo fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); - GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); + MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); count = 0.f; for (int k=1; kComm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); return countGlobal; } */ @@ -488,42 +506,42 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrsendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id); PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm->Comm.sendrecv(sendID_x,Dm->sendCount_x,Dm->rank_x(),sendtag, - recvID_X,Dm->recvCount_X,Dm->rank_X(),recvtag); - Dm->Comm.sendrecv(sendID_X,Dm->sendCount_X,Dm->rank_X(),sendtag, - recvID_x,Dm->recvCount_x,Dm->rank_x(),recvtag); - Dm->Comm.sendrecv(sendID_y,Dm->sendCount_y,Dm->rank_y(),sendtag, - recvID_Y,Dm->recvCount_Y,Dm->rank_Y(),recvtag); - Dm->Comm.sendrecv(sendID_Y,Dm->sendCount_Y,Dm->rank_Y(),sendtag, - recvID_y,Dm->recvCount_y,Dm->rank_y(),recvtag); - Dm->Comm.sendrecv(sendID_z,Dm->sendCount_z,Dm->rank_z(),sendtag, - recvID_Z,Dm->recvCount_Z,Dm->rank_Z(),recvtag); - Dm->Comm.sendrecv(sendID_Z,Dm->sendCount_Z,Dm->rank_Z(),sendtag, - recvID_z,Dm->recvCount_z,Dm->rank_z(),recvtag); - Dm->Comm.sendrecv(sendID_xy,Dm->sendCount_xy,Dm->rank_xy(),sendtag, - recvID_XY,Dm->recvCount_XY,Dm->rank_XY(),recvtag); - Dm->Comm.sendrecv(sendID_XY,Dm->sendCount_XY,Dm->rank_XY(),sendtag, - recvID_xy,Dm->recvCount_xy,Dm->rank_xy(),recvtag); - Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount_Xy,Dm->rank_Xy(),sendtag, - recvID_xY,Dm->recvCount_xY,Dm->rank_xY(),recvtag); - Dm->Comm.sendrecv(sendID_xY,Dm->sendCount_xY,Dm->rank_xY(),sendtag, - recvID_Xy,Dm->recvCount_Xy,Dm->rank_Xy(),recvtag); - Dm->Comm.sendrecv(sendID_xz,Dm->sendCount_xz,Dm->rank_xz(),sendtag, - recvID_XZ,Dm->recvCount_XZ,Dm->rank_XZ(),recvtag); - Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount_XZ,Dm->rank_XZ(),sendtag, - recvID_xz,Dm->recvCount_xz,Dm->rank_xz(),recvtag); - Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount_Xz,Dm->rank_Xz(),sendtag, - recvID_xZ,Dm->recvCount_xZ,Dm->rank_xZ(),recvtag); - Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount_xZ,Dm->rank_xZ(),sendtag, - recvID_Xz,Dm->recvCount_Xz,Dm->rank_Xz(),recvtag); - Dm->Comm.sendrecv(sendID_yz,Dm->sendCount_yz,Dm->rank_yz(),sendtag, - recvID_YZ,Dm->recvCount_YZ,Dm->rank_YZ(),recvtag); - Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount_YZ,Dm->rank_YZ(),sendtag, - recvID_yz,Dm->recvCount_yz,Dm->rank_yz(),recvtag); - Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount_Yz,Dm->rank_Yz(),sendtag, - recvID_yZ,Dm->recvCount_yZ,Dm->rank_yZ(),recvtag); - Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount_yZ,Dm->rank_yZ(),sendtag, - recvID_Yz,Dm->recvCount_Yz,Dm->rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -599,7 +617,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrrank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); for (int k=1; k -void MPI_CLASS::sendrecv( const char*, int, int, int, char*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const int*, int, int, int, int*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const float*, int, int, int, float*, int, int, int ) const; -template<> -void MPI_CLASS::sendrecv( const double*, int, int, int, double*, int, int, int ) const; -template -void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, - TYPE *recvbuf, int recvcount, int source, int recvtag ) const -{ - ERROR( "Not implimented" ); -} -#else -template -void MPI_CLASS::sendrecv( const TYPE *sendbuf, int sendcount, int dest, int sendtag, - TYPE *recvbuf, int recvcount, int source, int recvtag ) const -{ - ASSERT( dest == 0 ); - ASSERT( source == 0 ); - ASSERT( sendcount == recvcount ); - ASSERT( sendtag == recvtag ); - memcpy( recvbuf, sendbuf, sendcount * sizeof( TYPE ) ); -} -#endif - - - /************************************************************************ * allGather * ************************************************************************/ diff --git a/common/MPI.cpp b/common/MPI.cpp index 8b09bc49..73932d03 100644 --- a/common/MPI.cpp +++ b/common/MPI.cpp @@ -2805,54 +2805,6 @@ MPI_Request MPI_CLASS::IrecvBytes( } - -/************************************************************************ - * sendrecv * - ************************************************************************/ -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::sendrecv( const char* sendbuf, int sendcount, int dest, int sendtag, - char* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag, - recvbuf, recvcount, MPI_CHAR, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const int* sendbuf, int sendcount, int dest, int sendtag, - int* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag, - recvbuf, recvcount, MPI_INT, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const float* sendbuf, int sendcount, int dest, int sendtag, - float* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag, - recvbuf, recvcount, MPI_FLOAT, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -template<> -void MPI_CLASS::sendrecv( const double* sendbuf, int sendcount, int dest, int sendtag, - double* recvbuf, int recvcount, int source, int recvtag ) const -{ - PROFILE_START( "sendrecv", profile_level ); - MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag, - recvbuf, recvcount, MPI_DOUBLE, source, recvtag, - communicator, MPI_STATUS_IGNORE ); - PROFILE_STOP( "sendrecv", profile_level ); -} -#endif - - /************************************************************************ * allGather * * Note: these specializations are only called when using MPI. * diff --git a/common/MPI.h b/common/MPI.h index 4161d6a7..e3fd3e13 100644 --- a/common/MPI.h +++ b/common/MPI.h @@ -792,13 +792,6 @@ public: // Member functions void *buf, const int N_bytes, const int send_proc, const int tag ) const; - /*! - * @brief This function sends and recieves data using a blocking call - */ - template - void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf, int recvcount, int source, int recvtag ) const; - - /*! * Each processor sends every other processor a single value. * @param[in] x Input value for allGather diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 610fce5d..51195f5a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -201,6 +201,7 @@ private: int sendtag,recvtag; // Give the object it's own MPI communicator RankInfoStruct rank_info; + MPI_Group Group; // Group of processors associated with this domain Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain MPI_Request req1[18],req2[18]; //...................................................................................... diff --git a/common/Utilities.cpp b/common/Utilities.cpp index 723b34f8..9c89e024 100644 --- a/common/Utilities.cpp +++ b/common/Utilities.cpp @@ -8,7 +8,7 @@ #endif #ifdef USE_MPI -#include "common/MPI.h" +#include "mpi.h" #endif #include diff --git a/cpu/exe/lb2_Color_mpi.cpp b/cpu/exe/lb2_Color_mpi.cpp index cdf56af9..0cade21e 100644 --- a/cpu/exe/lb2_Color_mpi.cpp +++ b/cpu/exe/lb2_Color_mpi.cpp @@ -36,11 +36,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -54,6 +58,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -110,30 +115,31 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -163,7 +169,7 @@ int main(int argc, char **argv) } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -445,7 +451,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -582,7 +588,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -593,46 +599,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.Send(&sendCount_x,1,rank_X,sendtag); - comm.Recv(&recvCount_X,1,rank_x,recvtag); - comm.Send(&sendCount_X,1,rank_x,sendtag); - comm.Recv(&recvCount_x,1,rank_X,recvtag); - comm.Send(&sendCount_y,1,rank_Y,sendtag); - comm.Recv(&recvCount_Y,1,rank_y,recvtag); - comm.Send(&sendCount_Y,1,rank_y,sendtag); - comm.Recv(&recvCount_y,1,rank_Y,recvtag); - comm.Send(&sendCount_z,1,rank_Z,sendtag); - comm.Recv(&recvCount_Z,1,rank_z,recvtag); - comm.Send(&sendCount_Z,1,rank_z,sendtag); - comm.Recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xy,1,rank_XY,sendtag); - comm.Recv(&recvCount_XY,1,rank_xy,recvtag); - comm.Send(&sendCount_XY,1,rank_xy,sendtag); - comm.Recv(&recvCount_xy,1,rank_XY,recvtag); - comm.Send(&sendCount_Xy,1,rank_xY,sendtag); - comm.Recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.Send(&sendCount_xY,1,rank_Xy,sendtag); - comm.Recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xz,1,rank_XZ,sendtag); - comm.Recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.Send(&sendCount_XZ,1,rank_xz,sendtag); - comm.Recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.Send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.Send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_yz,1,rank_YZ,sendtag); - comm.Recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.Send(&sendCount_YZ,1,rank_yz,sendtag); - comm.Recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.Send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.Send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -663,48 +669,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idx #include #include -#include "common/MPI.h" +#include #include using namespace std; @@ -64,11 +64,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -82,6 +86,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; //********************************** //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -131,23 +136,24 @@ int main(int argc, char **argv) // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** double rlx_setA = 1.f/tau; @@ -170,7 +176,7 @@ int main(int argc, char **argv) printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -451,7 +457,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -588,7 +594,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -599,46 +605,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.send(&sendCount_x,1,rank_X,sendtag); - comm.recv(&recvCount_X,1,rank_x,recvtag); - comm.send(&sendCount_X,1,rank_x,sendtag); - comm.recv(&recvCount_x,1,rank_X,recvtag); - comm.send(&sendCount_y,1,rank_Y,sendtag); - comm.recv(&recvCount_Y,1,rank_y,recvtag); - comm.send(&sendCount_Y,1,rank_y,sendtag); - comm.recv(&recvCount_y,1,rank_Y,recvtag); - comm.send(&sendCount_z,1,rank_Z,sendtag); - comm.recv(&recvCount_Z,1,rank_z,recvtag); - comm.send(&sendCount_Z,1,rank_z,sendtag); - comm.recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xy,1,rank_XY,sendtag); - comm.recv(&recvCount_XY,1,rank_xy,recvtag); - comm.send(&sendCount_XY,1,rank_xy,sendtag); - comm.recv(&recvCount_xy,1,rank_XY,recvtag); - comm.send(&sendCount_Xy,1,rank_xY,sendtag); - comm.recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.send(&sendCount_xY,1,rank_Xy,sendtag); - comm.recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xz,1,rank_XZ,sendtag); - comm.recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.send(&sendCount_XZ,1,rank_xz,sendtag); - comm.recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_yz,1,rank_YZ,sendtag); - comm.recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.send(&sendCount_YZ,1,rank_yz,sendtag); - comm.recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -669,48 +675,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z; double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ; @@ -909,24 +915,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -959,7 +983,7 @@ int main(int argc, char **argv) free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz); //...................................................................................... if (rank==0) printf ("Devices are ready to communicate. \n"); - comm.barrier(); + MPI_Barrier(comm); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); @@ -999,8 +1023,8 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); - starttime = Utilities::MPI::time(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); // Old cuda timer is below // cudaEvent_t start, stop; // float time; @@ -1112,48 +1136,48 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag); - req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag); - req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag); - req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1236,7 +1260,7 @@ int main(int argc, char **argv) //***************************************************************************** //***************************************************************************** - comm.barrier(); + MPI_Barrier(comm); // Iteration completed! timestep++; //................................................................... @@ -1245,8 +1269,8 @@ int main(int argc, char **argv) // cudaThreadSynchronize(); dvc_Barrier(); - comm.barrier(); - stoptime = Utilities::MPI::time(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; @@ -1280,7 +1304,7 @@ int main(int argc, char **argv) // dvc_CopyToDevice(velocity, vel, 3*dist_mem_size, dvc_CopyToDeviceDeviceToHost); //.............................................................................. // cudaThreadSynchronize(); -// comm.barrier(); +// MPI_Barrier(comm); //............................................................ //....Write the z-velocity to test poiseuille flow............ // double vz,vz_avg; @@ -1309,7 +1333,7 @@ int main(int argc, char **argv) // free (velocity); free(id); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/gpu/exe/lb1_MRT_mpi.cu b/gpu/exe/lb1_MRT_mpi.cu index 776ea29f..0c0863c7 100644 --- a/gpu/exe/lb1_MRT_mpi.cu +++ b/gpu/exe/lb1_MRT_mpi.cu @@ -1,10 +1,8 @@ -#include "common/MPI.h" - #include #include #include #include - +#include inline void PackID(int *list, int count, char *sendbuf, char *ID){ // Fill in the phase ID values from neighboring processors @@ -555,11 +553,15 @@ void Write_Out(double *array, int Nx, int Ny, int Nz){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -573,6 +575,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; //********************************** //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //!!!!!!!!!!! Random debugging communications!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -622,21 +625,24 @@ int main(int argc, char **argv) // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&iterMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&iterMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + //................................................. + MPI_Barrier(comm); // ************************************************************** double rlx_setA = 1.f/tau; @@ -659,7 +665,7 @@ int main(int argc, char **argv) printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -940,7 +946,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -1077,7 +1083,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -1088,46 +1094,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.send(&sendCount_x,1,rank_X,sendtag); - comm.recv(&recvCount_X,1,rank_x,recvtag); - comm.send(&sendCount_X,1,rank_x,sendtag); - comm.recv(&recvCount_x,1,rank_X,recvtag); - comm.send(&sendCount_y,1,rank_Y,sendtag); - comm.recv(&recvCount_Y,1,rank_y,recvtag); - comm.send(&sendCount_Y,1,rank_y,sendtag); - comm.recv(&recvCount_y,1,rank_Y,recvtag); - comm.send(&sendCount_z,1,rank_Z,sendtag); - comm.recv(&recvCount_Z,1,rank_z,recvtag); - comm.send(&sendCount_Z,1,rank_z,sendtag); - comm.recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xy,1,rank_XY,sendtag); - comm.recv(&recvCount_XY,1,rank_xy,recvtag); - comm.send(&sendCount_XY,1,rank_xy,sendtag); - comm.recv(&recvCount_xy,1,rank_XY,recvtag); - comm.send(&sendCount_Xy,1,rank_xY,sendtag); - comm.recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.send(&sendCount_xY,1,rank_Xy,sendtag); - comm.recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_xz,1,rank_XZ,sendtag); - comm.recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.send(&sendCount_XZ,1,rank_xz,sendtag); - comm.recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.send(&sendCount_yz,1,rank_YZ,sendtag); - comm.recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.send(&sendCount_YZ,1,rank_yz,sendtag); - comm.recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //recvCount_x = sendCount_x; //recvCount_X = sendCount_X; @@ -1151,7 +1157,7 @@ int main(int argc, char **argv) //...................................................................................... // Use MPI to fill in the appropriate values // int tag = 5; - // Mcomm.sendrecv(sendCount_x,1,rank_x,tag,sendCount_X,1); + // MPI_Sendrecv(sendCount_x,1,MPI_INT,rank_x,tag,sendCount_X,1,MPI_INT,comm,req); //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; @@ -1181,48 +1187,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z; double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ; @@ -1421,24 +1427,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_X,sendtag,recvID_X,recvCount_X,rank_x,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_x,sendtag,recvID_x,recvCount_x,rank_X,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_Y,sendtag,recvID_Y,recvCount_Y,rank_y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_y,sendtag,recvID_y,recvCount_y,rank_Y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_Z,sendtag,recvID_Z,recvCount_Z,rank_z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_z,sendtag,recvID_z,recvCount_z,rank_Z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_XY,sendtag,recvID_XY,recvCount_XY,rank_xy,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_xy,sendtag,recvID_xy,recvCount_xy,rank_XY,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_xY,sendtag,recvID_xY,recvCount_xY,rank_Xy,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_Xy,sendtag,recvID_Xy,recvCount_Xy,rank_xY,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_XZ,sendtag,recvID_XZ,recvCount_XZ,rank_xz,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_xz,sendtag,recvID_xz,recvCount_xz,rank_XZ,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_xZ,sendtag,recvID_xZ,recvCount_xZ,rank_Xz,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_Xz,sendtag,recvID_Xz,recvCount_Xz,rank_xZ,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_YZ,sendtag,recvID_YZ,recvCount_YZ,rank_yz,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_yz,sendtag,recvID_yz,recvCount_yz,rank_YZ,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_yZ,sendtag,recvID_yZ,recvCount_yZ,rank_Yz,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_Yz,sendtag,recvID_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_X,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_x,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_Y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_Z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_XY,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_xy,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_xY,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_Xy,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_XZ,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_xz,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_xZ,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_Xz,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_YZ,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_yz,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_yZ,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_Yz,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -1471,7 +1495,7 @@ int main(int argc, char **argv) free(recvID_yz); free(recvID_YZ); free(recvID_yZ); free(recvID_Yz); //...................................................................................... if (rank==0) printf ("Devices are ready to communicate. \n"); - comm.barrier(); + MPI_Barrier(comm); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); @@ -1511,8 +1535,8 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); - starttime = Utilities::MPI::time(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); // Old cuda timer is below // cudaEvent_t start, stop; // float time; @@ -1609,48 +1633,48 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,5*sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvbuf_X,5*recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendbuf_X,5*sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvbuf_x,5*recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendbuf_y,5*sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,5*recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,5*sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,5*recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendbuf_z,5*sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,5*recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,5*sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,5*recvCount_z,rank_Z,recvtag); - req1[6] = comm.Isend(sendbuf_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvbuf_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendbuf_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvbuf_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendbuf_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvbuf_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendbuf_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy,recvCount_Xy,rank_xY,recvtag); - req1[10] = comm.Isend(sendbuf_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendbuf_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvbuf_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendbuf_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendbuf_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz,recvCount_Xz,rank_xZ,recvtag); - req1[14] = comm.Isend(sendbuf_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendbuf_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvbuf_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendbuf_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendbuf_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz,recvCount_Yz,rank_yZ,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1734,7 +1758,7 @@ int main(int argc, char **argv) //***************************************************************************** //***************************************************************************** - comm.barrier(); + MPI_Barrier(comm); // Iteration completed! iter++; //................................................................... @@ -1742,8 +1766,8 @@ int main(int argc, char **argv) //************************************************************************/ cudaThreadSynchronize(); - comm.barrier(); - stoptime = Utilities::MPI::time(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*iter)/cputime/1000000 << " MLUPS" << endl; @@ -1778,7 +1802,7 @@ int main(int argc, char **argv) cudaMemcpy(velocity, vel, 3*dist_mem_size, cudaMemcpyDeviceToHost); //.............................................................................. cudaThreadSynchronize(); - comm.barrier(); + MPI_Barrier(comm); //............................................................ //....Write the z-velocity to test poiseuille flow............ double vz,vz_avg; @@ -1807,7 +1831,7 @@ int main(int argc, char **argv) free (velocity); free(id); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/gpu/exe/lb2_Color.cu b/gpu/exe/lb2_Color.cu index 1f227d08..1871b23c 100644 --- a/gpu/exe/lb2_Color.cu +++ b/gpu/exe/lb2_Color.cu @@ -1,4 +1,6 @@ -#include "common/MPI.h" +#ifdef useMPI +#include +#endif #include #include @@ -60,10 +62,18 @@ int main(int argc, char *argv[]) { //********** Initialize MPI **************** + int numprocs,rank; +#ifdef useMPI + MPI_Status stat; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int numprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_size(comm,&numprocs); + MPI_Comm_rank(comm,&rank); +#else + MPI_Comm comm = MPI_COMM_WORLD; + numprocs = 1; + rank = 0; +#endif //****************************************** if (rank == 0){ @@ -113,31 +123,32 @@ int main(int argc, char *argv[]) input >> tol; // error tolerance //............................................................. } +#ifdef useMPI // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** +#endif double rlxA = 1.f/tau; double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA); @@ -232,7 +243,11 @@ int main(int argc, char *argv[]) if (k==4) k=Nz-5; } } - comm.bcast(&id[0],N,0); +#ifdef useMPI //............................................................ + MPI_Barrier(comm); + MPI_Bcast(&id[0],N,MPI_CHAR,0,comm); + MPI_Barrier(comm); +#endif if (rank == 0) printf("Domain set.\n"); //........................................................................... diff --git a/gpu/exe/lb2_Color_mpi.cpp b/gpu/exe/lb2_Color_mpi.cpp index a2f3d8a9..fe11d32f 100644 --- a/gpu/exe/lb2_Color_mpi.cpp +++ b/gpu/exe/lb2_Color_mpi.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "common/MPI.h" +#include using namespace std; @@ -98,11 +98,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -116,6 +120,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -172,30 +177,31 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); + + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -225,7 +231,7 @@ int main(int argc, char **argv) } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -507,7 +513,7 @@ int main(int argc, char **argv) PM.close(); // printf("File porosity = %f\n", double(sum)/N); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... // Write the communcation structure into a file for debugging @@ -644,7 +650,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -655,46 +661,46 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - comm.Send(&sendCount_x,1,rank_X,sendtag); - comm.Recv(&recvCount_X,1,rank_x,recvtag); - comm.Send(&sendCount_X,1,rank_x,sendtag); - comm.Recv(&recvCount_x,1,rank_X,recvtag); - comm.Send(&sendCount_y,1,rank_Y,sendtag); - comm.Recv(&recvCount_Y,1,rank_y,recvtag); - comm.Send(&sendCount_Y,1,rank_y,sendtag); - comm.Recv(&recvCount_y,1,rank_Y,recvtag); - comm.Send(&sendCount_z,1,rank_Z,sendtag); - comm.Recv(&recvCount_Z,1,rank_z,recvtag); - comm.Send(&sendCount_Z,1,rank_z,sendtag); - comm.Recv(&recvCount_z,1,rank_Z,recvtag); + MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xy,1,rank_XY,sendtag); - comm.Recv(&recvCount_XY,1,rank_xy,recvtag); - comm.Send(&sendCount_XY,1,rank_xy,sendtag); - comm.Recv(&recvCount_xy,1,rank_XY,recvtag); - comm.Send(&sendCount_Xy,1,rank_xY,sendtag); - comm.Recv(&recvCount_xY,1,rank_Xy,recvtag); - comm.Send(&sendCount_xY,1,rank_Xy,sendtag); - comm.Recv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_xz,1,rank_XZ,sendtag); - comm.Recv(&recvCount_XZ,1,rank_xz,recvtag); - comm.Send(&sendCount_XZ,1,rank_xz,sendtag); - comm.Recv(&recvCount_xz,1,rank_XZ,recvtag); - comm.Send(&sendCount_Xz,1,rank_xZ,sendtag); - comm.Recv(&recvCount_xZ,1,rank_Xz,recvtag); - comm.Send(&sendCount_xZ,1,rank_Xz,sendtag); - comm.Recv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - comm.Send(&sendCount_yz,1,rank_YZ,sendtag); - comm.Recv(&recvCount_YZ,1,rank_yz,recvtag); - comm.Send(&sendCount_YZ,1,rank_yz,sendtag); - comm.Recv(&recvCount_yz,1,rank_YZ,recvtag); - comm.Send(&sendCount_Yz,1,rank_yZ,sendtag); - comm.Recv(&recvCount_yZ,1,rank_Yz,recvtag); - comm.Send(&sendCount_yZ,1,rank_Yz,sendtag); - comm.Recv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.barrier(); + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; @@ -725,48 +731,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idx #include #include -#include "common/MPI.h" +#include #include "pmmc.h" #include "Domain.h" @@ -101,11 +101,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -119,6 +123,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -198,35 +203,35 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nz,1,0); - comm.bcast(&nBlocks,1,0); - comm.bcast(&nthreads,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); + MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** double Ps = -(das-dbs)/(das+dbs); @@ -258,7 +263,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -556,14 +561,14 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Reading the sphere packing \n"); if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //....................................................................... // sprintf(LocalRankString,"%05d",rank); @@ -713,7 +718,7 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -724,49 +729,89 @@ int main(int argc, char **argv) //********************************************************************************** // Fill in the recieve counts using MPI sendtag = recvtag = 3; - req1[0] = comm.Isend(&sendCount_x,1,rank_X,sendtag); - req2[0] = comm.Irecv(&recvCount_X,1,rank_x,recvtag); - req1[1] = comm.Isend(&sendCount_X,1,rank_x,sendtag); - req2[1] = comm.Irecv(&recvCount_x,1,rank_X,recvtag); - req1[2] = comm.Isend(&sendCount_y,1,rank_Y,sendtag); - req2[2] = comm.Irecv(&recvCount_Y,1,rank_y,recvtag); - req1[3] = comm.Isend(&sendCount_Y,1,rank_y,sendtag); - req2[3] = comm.Irecv(&recvCount_y,1,rank_Y,recvtag); - req1[4] = comm.Isend(&sendCount_z,1,rank_Z,sendtag); - req2[4] = comm.Irecv(&recvCount_Z,1,rank_z,recvtag); - req1[5] = comm.Isend(&sendCount_Z,1,rank_z,sendtag); - req2[5] = comm.Irecv(&recvCount_z,1,rank_Z,recvtag); + MPI_Isend(&sendCount_x, 1,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(&sendCount_X, 1,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(&sendCount_y, 1,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(&sendCount_z, 1,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(&sendCount_xy,1,rank_XY,sendtag); - req2[6] = comm.Irecv(&recvCount_XY,1,rank_xy,recvtag); - req1[7] = comm.Isend(&sendCount_XY,1,rank_xy,sendtag); - req2[7] = comm.Irecv(&recvCount_xy,1,rank_XY,recvtag); - req1[8] = comm.Isend(&sendCount_Xy,1,rank_xY,sendtag); - req2[8] = comm.Irecv(&recvCount_xY,1,rank_Xy,recvtag); - req1[9] = comm.Isend(&sendCount_xY,1,rank_Xy,sendtag); - req2[9] = comm.Irecv(&recvCount_Xy,1,rank_xY,recvtag); + MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(&sendCount_xz,1,rank_XZ,sendtag); - req2[10] = comm.Irecv(&recvCount_XZ,1,rank_xz,recvtag); - req1[11] = comm.Isend(&sendCount_XZ,1,rank_xz,sendtag); - req2[11] = comm.Irecv(&recvCount_xz,1,rank_XZ,recvtag); - req1[12] = comm.Isend(&sendCount_Xz,1,rank_xZ,sendtag); - req2[12] = comm.Irecv(&recvCount_xZ,1,rank_Xz,recvtag); - req1[13] = comm.Isend(&sendCount_xZ,1,rank_Xz,sendtag); - req2[13] = comm.Irecv(&recvCount_Xz,1,rank_xZ,recvtag); + MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(&sendCount_yz,1,rank_YZ,sendtag); - req2[14] = comm.Irecv(&recvCount_YZ,1,rank_yz,recvtag); - req1[15] = comm.Isend(&sendCount_YZ,1,rank_yz,sendtag); - req2[15] = comm.Irecv(&recvCount_yz,1,rank_YZ,recvtag); - req1[16] = comm.Isend(&sendCount_Yz,1,rank_yZ,sendtag); - req2[16] = comm.Irecv(&recvCount_yZ,1,rank_Yz,recvtag); - req1[17] = comm.Isend(&sendCount_yZ,1,rank_Yz,sendtag); - req2[17] = comm.Irecv(&recvCount_Yz,1,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); - //********************************************************************************** + MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); +/* MPI_Send(&sendCount_x,1,MPI_INT,rank_X,sendtag,comm); + MPI_Recv(&recvCount_X,1,MPI_INT,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_X,1,MPI_INT,rank_x,sendtag,comm); + MPI_Recv(&recvCount_x,1,MPI_INT,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_y,1,MPI_INT,rank_Y,sendtag,comm); + MPI_Recv(&recvCount_Y,1,MPI_INT,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Y,1,MPI_INT,rank_y,sendtag,comm); + MPI_Recv(&recvCount_y,1,MPI_INT,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_z,1,MPI_INT,rank_Z,sendtag,comm); + MPI_Recv(&recvCount_Z,1,MPI_INT,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Z,1,MPI_INT,rank_z,sendtag,comm); + MPI_Recv(&recvCount_z,1,MPI_INT,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_xy,1,MPI_INT,rank_XY,sendtag,comm); + MPI_Recv(&recvCount_XY,1,MPI_INT,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XY,1,MPI_INT,rank_xy,sendtag,comm); + MPI_Recv(&recvCount_xy,1,MPI_INT,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xy,1,MPI_INT,rank_xY,sendtag,comm); + MPI_Recv(&recvCount_xY,1,MPI_INT,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xY,1,MPI_INT,rank_Xy,sendtag,comm); + MPI_Recv(&recvCount_Xy,1,MPI_INT,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_xz,1,MPI_INT,rank_XZ,sendtag,comm); + MPI_Recv(&recvCount_XZ,1,MPI_INT,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_XZ,1,MPI_INT,rank_xz,sendtag,comm); + MPI_Recv(&recvCount_xz,1,MPI_INT,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Xz,1,MPI_INT,rank_xZ,sendtag,comm); + MPI_Recv(&recvCount_xZ,1,MPI_INT,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_xZ,1,MPI_INT,rank_Xz,sendtag,comm); + MPI_Recv(&recvCount_Xz,1,MPI_INT,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + + MPI_Send(&sendCount_yz,1,MPI_INT,rank_YZ,sendtag,comm); + MPI_Recv(&recvCount_YZ,1,MPI_INT,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_YZ,1,MPI_INT,rank_yz,sendtag,comm); + MPI_Recv(&recvCount_yz,1,MPI_INT,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_Yz,1,MPI_INT,rank_yZ,sendtag,comm); + MPI_Recv(&recvCount_yZ,1,MPI_INT,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Send(&sendCount_yZ,1,MPI_INT,rank_Yz,sendtag,comm); + MPI_Recv(&recvCount_Yz,1,MPI_INT,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Barrier(comm); +*/ //********************************************************************************** //...................................................................................... int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; @@ -796,48 +841,48 @@ int main(int argc, char **argv) // Use MPI to fill in the appropriate values for recvList // Fill in the recieve lists using MPI sendtag = recvtag = 4; - req1[0] = comm.Isend(sendList_x,sendCount_x,rank_X,sendtag); - req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_x,recvtag); - req1[1] = comm.Isend(sendList_X,sendCount_X,rank_x,sendtag); - req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_X,recvtag); - req1[2] = comm.Isend(sendList_y,sendCount_y,rank_Y,sendtag); - req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_y,recvtag); - req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_y,sendtag); - req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_Y,recvtag); - req1[4] = comm.Isend(sendList_z,sendCount_z,rank_Z,sendtag); - req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_z,recvtag); - req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_z,sendtag); - req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_Z,recvtag); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_X,sendtag,comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_x,recvtag,comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_x,sendtag,comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_X,recvtag,comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_Y,sendtag,comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_y,recvtag,comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_y,sendtag,comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_Y,recvtag,comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_Z,sendtag,comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_z,recvtag,comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_z,sendtag,comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_Z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_XY,sendtag); - req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_xy,recvtag); - req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_xy,sendtag); - req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_XY,recvtag); - req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_xY,sendtag); - req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_Xy,recvtag); - req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_Xy,sendtag); - req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_xY,recvtag); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_XY,sendtag,comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_xy,recvtag,comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_xy,sendtag,comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_XY,recvtag,comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_xY,sendtag,comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_Xy,recvtag,comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_Xy,sendtag,comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_xY,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_XZ,sendtag); - req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_xz,recvtag); - req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_xz,sendtag); - req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_XZ,recvtag); - req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_xZ,sendtag); - req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_Xz,recvtag); - req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_Xz,sendtag); - req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_xZ,recvtag); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_XZ,sendtag,comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_xz,recvtag,comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_xz,sendtag,comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_XZ,recvtag,comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_xZ,sendtag,comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_Xz,recvtag,comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_Xz,sendtag,comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_xZ,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_YZ,sendtag); - req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_yz,recvtag); - req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_yz,sendtag); - req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_YZ,recvtag); - req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_yZ,sendtag); - req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_Yz,recvtag); - req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_Yz,sendtag); - req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_yZ,recvtag); - comm.waitAll(18,req1); - comm.waitAll(18,req2); - comm.barrier(); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_YZ,sendtag,comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_yz,recvtag,comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_yz,sendtag,comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_YZ,recvtag,comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_yZ,sendtag,comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_Yz,recvtag,comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_Yz,sendtag,comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_yZ,recvtag,comm,&req2[17]); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); + MPI_Barrier(comm); //...................................................................................... for (int idx=0; idxkeyExists( "GridFile" )){ // Read the local domain data - auto input_id = readMicroCT( *domain_db, comm ); + auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD ); // Fill the halo (assuming GCW of 1) array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); - fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + fillHalo fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); Array id_view; id_view.viewRaw( size1, Mask->id ); fill.copy( input_id, id_view ); @@ -652,7 +652,7 @@ void ScaLBL_ColorModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -991,7 +991,7 @@ void ScaLBL_ColorModel::Run(){ //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/models/DFHModel.cpp b/models/DFHModel.cpp index 9709b107..ced5853f 100644 --- a/models/DFHModel.cpp +++ b/models/DFHModel.cpp @@ -487,7 +487,7 @@ void ScaLBL_DFHModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -583,7 +583,7 @@ void ScaLBL_DFHModel::Run(){ //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 60847e54..23925930 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -227,7 +227,7 @@ void ScaLBL_MRTModel::Run(){ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); if (rank==0) printf("********************************************************\n"); timestep=0; @@ -325,7 +325,7 @@ void ScaLBL_MRTModel::Run(){ } } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/BlobAnalyzeParallel.cpp b/tests/BlobAnalyzeParallel.cpp index 773309f9..48e9e230 100644 --- a/tests/BlobAnalyzeParallel.cpp +++ b/tests/BlobAnalyzeParallel.cpp @@ -138,16 +138,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -291,7 +291,7 @@ int main(int argc, char **argv) } Dm.CommInit(); // Initialize communications for domains - sum_global = comm.sumReduce( sum ); + MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum_global/Dm.Volume; if (rank==0) printf("Porosity = %f \n",porosity); diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index d4340964..43434092 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -213,24 +213,42 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... - Dm.Comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag); - Dm.Comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag); - Dm.Comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag); - Dm.Comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag); - Dm.Comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag); - Dm.Comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag); - Dm.Comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag); - Dm.Comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag); - Dm.Comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag); - Dm.Comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag); - Dm.Comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag); - Dm.Comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag); - Dm.Comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag); - Dm.Comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag); - Dm.Comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag); - Dm.Comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag); - Dm.Comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag); - Dm.Comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); diff --git a/tests/TestBlobAnalyze.cpp b/tests/TestBlobAnalyze.cpp index 19360fe3..63d928c1 100644 --- a/tests/TestBlobAnalyze.cpp +++ b/tests/TestBlobAnalyze.cpp @@ -190,16 +190,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -255,10 +255,10 @@ int main(int argc, char **argv) comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); //....................................................................... diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index 6eb74b37..e7e0ced8 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -45,6 +45,7 @@ int main(int argc, char **argv) int nprocx,nprocy,nprocz; MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -433,7 +434,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... //........................................................................... // MAIN VARIABLES INITIALIZED HERE @@ -808,25 +809,25 @@ int main(int argc, char **argv) } //........................................................................... comm.barrier(); - nwp_volume_global = comm.sumReduce( nwp_volume ); - awn_global = comm.sumReduce( awn ); - ans_global = comm.sumReduce( ans ); - aws_global = comm.sumReduce( aws ); - lwns_global = comm.sumReduce( lwns ); - As_global = comm.sumReduce( As ); - Jwn_global = comm.sumReduce( Jwn ); - efawns_global = comm.sumReduce( efawns ); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm); // Phase averages - vol_w_global = comm.sumReduce( vol_w ); - vol_n_global = comm.sumReduce( vol_n ); - paw_global = comm.sumReduce( paw ); - pan_global = comm.sumReduce( pan ); - vaw_global(0) = comm.sumReduce( vaw(0) ); - van_global(0) = comm.sumReduce( van(0) ); - vawn_global(0) = comm.sumReduce( vawn(0) ); - Gwn_global(0) = comm.sumReduce( Gwn(0) ); - Gns_global(0) = comm.sumReduce( Gns(0) ); - Gws_global(0) = comm.sumReduce( Gws(0) ); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm); comm.barrier(); //......................................................................... // Compute the change in the total surface energy based on the defined interval @@ -951,7 +952,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 8b4f1a9b..7f5d0047 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -387,7 +387,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... err = 1.0; @@ -487,7 +487,7 @@ int main(int argc, char **argv) //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/TestColorGrad.cpp b/tests/TestColorGrad.cpp index 2566f8c0..df1c1daf 100644 --- a/tests/TestColorGrad.cpp +++ b/tests/TestColorGrad.cpp @@ -114,16 +114,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); // ************************************************************** diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index c4a045ae..d2799355 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -378,7 +378,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... @@ -403,7 +403,7 @@ int main(int argc, char **argv) //................................................................... } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; diff --git a/tests/TestForceD3Q19.cpp b/tests/TestForceD3Q19.cpp index 31151584..f8569624 100644 --- a/tests/TestForceD3Q19.cpp +++ b/tests/TestForceD3Q19.cpp @@ -450,7 +450,7 @@ int main (int argc, char **argv) for (int i=0; iSendD3Q19(dist, &dist[10*Np]); @@ -244,7 +244,7 @@ int main(int argc, char **argv) //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; diff --git a/tests/TestMRT.cpp b/tests/TestMRT.cpp index e4acba99..5f2c4449 100644 --- a/tests/TestMRT.cpp +++ b/tests/TestMRT.cpp @@ -580,16 +580,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); // ************************************************************** @@ -668,7 +668,7 @@ int main(int argc, char **argv) } } comm.barrier(); - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -731,7 +731,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); while (timestep < timesteps) { @@ -752,7 +752,7 @@ int main(int argc, char **argv) } //************************************************************************/ - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; @@ -795,7 +795,7 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); double PoreVel = sum*iVol_global; if (rank==0) printf("Velocity = %f \n",PoreVel); diff --git a/tests/TestMicroCTReader.cpp b/tests/TestMicroCTReader.cpp index 52a5b9d3..9a54610c 100644 --- a/tests/TestMicroCTReader.cpp +++ b/tests/TestMicroCTReader.cpp @@ -62,6 +62,7 @@ int main(int argc, char **argv) int N_errors = ut.NumFailGlobal(); // Close MPI + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return N_errors; } diff --git a/tests/TestMomentsD3Q19.cpp b/tests/TestMomentsD3Q19.cpp index 2660ed26..6bd3e8ff 100644 --- a/tests/TestMomentsD3Q19.cpp +++ b/tests/TestMomentsD3Q19.cpp @@ -539,7 +539,7 @@ int main (int argc, char **argv) error=count; // Finished - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return error; } diff --git a/tests/TestNetcdf.cpp b/tests/TestNetcdf.cpp index 3d0498d2..8768c9ea 100644 --- a/tests/TestNetcdf.cpp +++ b/tests/TestNetcdf.cpp @@ -116,7 +116,7 @@ int main(int argc, char **argv) PROFILE_SAVE("TestNetcdf"); // Close MPI - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return N_errors; } diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index ecb6d6b9..b5e23ec8 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -100,10 +100,10 @@ int main(int argc, char **argv) comm.barrier(); if (rank==0) printf("Initialized! Converting to Signed Distance function \n"); - double t1 = Utilities::MPI::time(); + double t1 = MPI_Wtime(); DoubleArray Distance(nx,ny,nz); CalcDist(Distance,id,Dm,{false,false,false}); - double t2 = Utilities::MPI::time(); + double t2 = MPI_Wtime(); if (rank==0) printf("Total time: %f seconds \n",t2-t1); diff --git a/tests/lb2_CMT_wia.cpp b/tests/lb2_CMT_wia.cpp index 389bc8a8..820428a3 100644 --- a/tests/lb2_CMT_wia.cpp +++ b/tests/lb2_CMT_wia.cpp @@ -292,18 +292,18 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x,2*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X,2*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X,2*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x,2*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y,2*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y,2*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y,2*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y,2*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z,2*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z,2*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z,2*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z,2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); */ //................................................................................... ScaLBL_D3Q7_Swap(ID, &packed_even[0], &packed_odd[0], Nx, Ny, Nz); @@ -311,8 +311,8 @@ int main(int argc, char **argv) /* //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(6,req1); - comm.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -358,7 +358,7 @@ int main(int argc, char **argv) fclose(PHASE); // Close MPI - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); MPI_Finalize(); return 0; } diff --git a/tests/lb2_Color_blob_wia_mpi.cpp b/tests/lb2_Color_blob_wia_mpi.cpp index e3323612..70342176 100644 --- a/tests/lb2_Color_blob_wia_mpi.cpp +++ b/tests/lb2_Color_blob_wia_mpi.cpp @@ -114,6 +114,7 @@ int main(int argc, char **argv) int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -206,36 +207,36 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&das,1,0); - comm.bcast(&dbs,1,0); - comm.bcast(&phi_s,1,0); - comm.bcast(&wp_saturation,1,0); - comm.bcast(&pBC,1,0); - comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); -// comm.bcast(&nBlocks,1,0); -// comm.bcast(&nthreads,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); +// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); +// MPI_Bcast(&nthreads,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -398,10 +399,10 @@ int main(int argc, char **argv) if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0) cout << "Domain set." << endl; @@ -417,7 +418,7 @@ int main(int argc, char **argv) D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx; printf("Sauter Mean Diameter (computed from sphere packing) = %f \n ",D); } - comm.bcast(&D,1,0); + MPI_Bcast(&D,1,MPI_DOUBLE,0,comm); //....................................................................... // sprintf(LocalRankString,"%05d",rank); @@ -477,7 +478,7 @@ int main(int argc, char **argv) id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; //......................................................... sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -885,24 +886,42 @@ int main(int argc, char **argv) PackID(sendList_yZ, sendCount_yZ ,sendID_yZ, id); PackID(sendList_YZ, sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,sendCount_x,rank_x,sendtag,recvID_X,recvCount_X,rank_X,recvtag); - comm.sendrecv(sendID_X,sendCount_X,rank_X,sendtag,recvID_x,recvCount_x,rank_x,recvtag); - comm.sendrecv(sendID_y,sendCount_y,rank_y,sendtag,recvID_Y,recvCount_Y,rank_Y,recvtag); - comm.sendrecv(sendID_Y,sendCount_Y,rank_Y,sendtag,recvID_y,recvCount_y,rank_y,recvtag); - comm.sendrecv(sendID_z,sendCount_z,rank_z,sendtag,recvID_Z,recvCount_Z,rank_Z,recvtag); - comm.sendrecv(sendID_Z,sendCount_Z,rank_Z,sendtag,recvID_z,recvCount_z,rank_z,recvtag); - comm.sendrecv(sendID_xy,sendCount_xy,rank_xy,sendtag,recvID_XY,recvCount_XY,rank_XY,recvtag); - comm.sendrecv(sendID_XY,sendCount_XY,rank_XY,sendtag,recvID_xy,recvCount_xy,rank_xy,recvtag); - comm.sendrecv(sendID_Xy,sendCount_Xy,rank_Xy,sendtag,recvID_xY,recvCount_xY,rank_xY,recvtag); - comm.sendrecv(sendID_xY,sendCount_xY,rank_xY,sendtag,recvID_Xy,recvCount_Xy,rank_Xy,recvtag); - comm.sendrecv(sendID_xz,sendCount_xz,rank_xz,sendtag,recvID_XZ,recvCount_XZ,rank_XZ,recvtag); - comm.sendrecv(sendID_XZ,sendCount_XZ,rank_XZ,sendtag,recvID_xz,recvCount_xz,rank_xz,recvtag); - comm.sendrecv(sendID_Xz,sendCount_Xz,rank_Xz,sendtag,recvID_xZ,recvCount_xZ,rank_xZ,recvtag); - comm.sendrecv(sendID_xZ,sendCount_xZ,rank_xZ,sendtag,recvID_Xz,recvCount_Xz,rank_Xz,recvtag); - comm.sendrecv(sendID_yz,sendCount_yz,rank_yz,sendtag,recvID_YZ,recvCount_YZ,rank_YZ,recvtag); - comm.sendrecv(sendID_YZ,sendCount_YZ,rank_YZ,sendtag,recvID_yz,recvCount_yz,rank_yz,recvtag); - comm.sendrecv(sendID_Yz,sendCount_Yz,rank_Yz,sendtag,recvID_yZ,recvCount_yZ,rank_yZ,recvtag); - comm.sendrecv(sendID_yZ,sendCount_yZ,rank_yZ,sendtag,recvID_Yz,recvCount_Yz,rank_Yz,recvtag); + MPI_Sendrecv(sendID_x,sendCount_x,MPI_CHAR,rank_x,sendtag, + recvID_X,recvCount_X,MPI_CHAR,rank_X,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,sendCount_X,MPI_CHAR,rank_X,sendtag, + recvID_x,recvCount_x,MPI_CHAR,rank_x,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,sendCount_y,MPI_CHAR,rank_y,sendtag, + recvID_Y,recvCount_Y,MPI_CHAR,rank_Y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,sendCount_Y,MPI_CHAR,rank_Y,sendtag, + recvID_y,recvCount_y,MPI_CHAR,rank_y,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,sendCount_z,MPI_CHAR,rank_z,sendtag, + recvID_Z,recvCount_Z,MPI_CHAR,rank_Z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,sendCount_Z,MPI_CHAR,rank_Z,sendtag, + recvID_z,recvCount_z,MPI_CHAR,rank_z,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,sendCount_xy,MPI_CHAR,rank_xy,sendtag, + recvID_XY,recvCount_XY,MPI_CHAR,rank_XY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,sendCount_XY,MPI_CHAR,rank_XY,sendtag, + recvID_xy,recvCount_xy,MPI_CHAR,rank_xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,sendCount_Xy,MPI_CHAR,rank_Xy,sendtag, + recvID_xY,recvCount_xY,MPI_CHAR,rank_xY,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,sendCount_xY,MPI_CHAR,rank_xY,sendtag, + recvID_Xy,recvCount_Xy,MPI_CHAR,rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,sendCount_xz,MPI_CHAR,rank_xz,sendtag, + recvID_XZ,recvCount_XZ,MPI_CHAR,rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,sendCount_XZ,MPI_CHAR,rank_XZ,sendtag, + recvID_xz,recvCount_xz,MPI_CHAR,rank_xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,sendCount_Xz,MPI_CHAR,rank_Xz,sendtag, + recvID_xZ,recvCount_xZ,MPI_CHAR,rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,sendCount_xZ,MPI_CHAR,rank_xZ,sendtag, + recvID_Xz,recvCount_Xz,MPI_CHAR,rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,sendCount_yz,MPI_CHAR,rank_yz,sendtag, + recvID_YZ,recvCount_YZ,MPI_CHAR,rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,sendCount_YZ,MPI_CHAR,rank_YZ,sendtag, + recvID_yz,recvCount_yz,MPI_CHAR,rank_yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,sendCount_Yz,MPI_CHAR,rank_Yz,sendtag, + recvID_yZ,recvCount_yZ,MPI_CHAR,rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,sendCount_yZ,MPI_CHAR,rank_yZ,sendtag, + recvID_Yz,recvCount_Yz,MPI_CHAR,rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(recvList_x, recvCount_x ,recvID_x, id); UnpackID(recvList_X, recvCount_X ,recvID_X, id); @@ -1361,48 +1380,48 @@ int main(int argc, char **argv) //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of Indicator Field communication //................................................................................... - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -1478,7 +1497,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... sendtag = recvtag = 5; @@ -1574,42 +1593,42 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //************************************************************************* @@ -1629,8 +1648,8 @@ int main(int argc, char **argv) //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); //................................................................................... // Unpack the distributions on the device @@ -1724,18 +1743,18 @@ int main(int argc, char **argv) //................................................................................... // Send all the distributions - req1[0] = comm.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); - req2[0] = comm.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); - req1[1] = comm.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); - req2[1] = comm.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); - req1[2] = comm.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); - req2[2] = comm.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); - req1[3] = comm.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); - req2[3] = comm.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); - req1[4] = comm.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); - req2[4] = comm.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); - req1[5] = comm.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); - req2[5] = comm.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); //................................................................................... ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); @@ -1743,8 +1762,8 @@ int main(int argc, char **argv) //................................................................................... // Wait for completion of D3Q19 communication - comm.waitAll(6,req1); - comm.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); //................................................................................... // Unpack the distributions on the device //................................................................................... @@ -1805,48 +1824,48 @@ int main(int argc, char **argv) //................................................................................... // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = comm.Isend(sendbuf_x, sendCount_x,rank_x,sendtag,comm,&req1[0]); - req2[0] = comm.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag,comm,&req2[0]); - req1[1] = comm.Isend(sendbuf_X, sendCount_X,rank_X,sendtag,comm,&req1[1]); - req2[1] = comm.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag,comm,&req2[1]); - req1[2] = comm.Isend(sendbuf_y, sendCount_y,rank_y,sendtag,comm,&req1[2]); - req2[2] = comm.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag,comm,&req2[2]); - req1[3] = comm.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag,comm,&req1[3]); - req2[3] = comm.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag,comm,&req2[3]); - req1[4] = comm.Isend(sendbuf_z, sendCount_z,rank_z,sendtag,comm,&req1[4]); - req2[4] = comm.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag,comm,&req2[4]); - req1[5] = comm.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag,comm,&req1[5]); - req2[5] = comm.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag,comm,&req2[5]); - req1[6] = comm.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag,comm,&req1[6]); - req2[6] = comm.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag,comm,&req2[6]); - req1[7] = comm.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag,comm,&req1[7]); - req2[7] = comm.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag,comm,&req2[7]); - req1[8] = comm.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag,comm,&req1[8]); - req2[8] = comm.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag,comm,&req2[8]); - req1[9] = comm.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag,comm,&req1[9]); - req2[9] = comm.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag,comm,&req2[9]); - req1[10] = comm.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag,comm,&req1[10]); - req2[10] = comm.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag,comm,&req2[10]); - req1[11] = comm.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag,comm,&req1[11]); - req2[11] = comm.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag,comm,&req2[11]); - req1[12] = comm.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag,comm,&req1[12]); - req2[12] = comm.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag,comm,&req2[12]); - req1[13] = comm.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag,comm,&req1[13]); - req2[13] = comm.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag,comm,&req2[13]); - req1[14] = comm.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag,comm,&req1[14]); - req2[14] = comm.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag,comm,&req2[14]); - req1[15] = comm.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag,comm,&req1[15]); - req2[15] = comm.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag,comm,&req2[15]); - req1[16] = comm.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag,comm,&req1[16]); - req2[16] = comm.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag,comm,&req2[16]); - req1[17] = comm.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag,comm,&req1[17]); - req2[17] = comm.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag,comm,&req2[17]); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,comm,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,comm,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,comm,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,comm,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,comm,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,comm,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,comm,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,comm,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,comm,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,comm,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,comm,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,comm,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,comm,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,comm,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,comm,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,comm,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,comm,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,comm,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,comm,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,comm,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,comm,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,comm,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,comm,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,comm,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,comm,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,comm,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,comm,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,comm,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,comm,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,comm,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,comm,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,comm,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,comm,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,comm,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,comm,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,comm,&req2[17]); //................................................................................... //................................................................................... // Wait for completion of Indicator Field communication //................................................................................... - comm.waitAll(18,req1); - comm.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -2423,28 +2442,28 @@ int main(int argc, char **argv) //........................................................................... comm.barrier(); - nwp_volume_global = comm.sumReduce( nwp_volume ); - awn_global = comm.sumReduce( awn ); - ans_global = comm.sumReduce( ans ); - aws_global = comm.sumReduce( aws ); - lwns_global = comm.sumReduce( lwns ); - As_global = comm.sumReduce( As ); - Jwn_global = comm.sumReduce( Jwn ); - Kwn_global = comm.sumReduce( Kwn ); - efawns_global = comm.sumReduce( efawns ); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,comm); // Phase averages - vol_w_global = comm.sumReduce( vol_w ); - vol_n_global = comm.sumReduce( vol_n ); - paw_global = comm.sumReduce( paw ); - pan_global = comm.sumReduce( pan ); - vaw_global(0) = comm.sumReduce( vaw(0) ); - van_global(0) = comm.sumReduce( van(0) ); - vawn_global(0) = comm.sumReduce( vawn(0) ); - Gwn_global(0) = comm.sumReduce( Gwn(0) ); - Gns_global(0) = comm.sumReduce( Gns(0) ); - Gws_global(0) = comm.sumReduce( Gws(0) ); - trawn_global = comm.sumReduce( trawn ); - trJwn_global = comm.sumReduce( trJwn ); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,comm); + MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,comm); comm.barrier(); //......................................................................... // Compute the change in the total surface energy based on the defined interval @@ -2670,7 +2689,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_BGK_simulator.cpp b/tests/lbpm_BGK_simulator.cpp index 1ac61853..8b079900 100644 --- a/tests/lbpm_BGK_simulator.cpp +++ b/tests/lbpm_BGK_simulator.cpp @@ -97,28 +97,28 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - //comm.bcast(&pBC,1,0); - //comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - //comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + //MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -249,7 +249,7 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -331,7 +331,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... double D32,Fo,Re,velocity,err1D,mag_force,vel_prev; @@ -410,7 +410,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index c92b0c45..97df6812 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -39,6 +39,9 @@ int main(int argc, char **argv) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + if (rank == 0){ printf("********************************************************\n"); printf("Running Color LBM \n"); @@ -169,32 +172,32 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tauA,1,0); - comm.bcast(&tauB,1,0); - comm.bcast(&rhoA,1,0); - comm.bcast(&rhoB,1,0); - comm.bcast(&alpha,1,0); - comm.bcast(&beta,1,0); - comm.bcast(&BoundaryCondition,1,0); - comm.bcast(&InitialCondition,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&RESTART_INTERVAL,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm); + MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. flux = 0.f; @@ -319,7 +322,7 @@ int main(int argc, char **argv) timestep=0; } } - comm.bcast(×tep,1,0); + MPI_Bcast(×tep,1,MPI_INT,0,comm); FILE *RESTART = fopen(LocalRestartFile,"rb"); if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); readID=fread(id,1,N,RESTART); @@ -358,7 +361,7 @@ int main(int argc, char **argv) } } } - sum - comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... @@ -534,7 +537,7 @@ int main(int argc, char **argv) double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... err = 1.0; @@ -634,7 +637,7 @@ int main(int argc, char **argv) //************************************************************************ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_disc_pp.cpp b/tests/lbpm_disc_pp.cpp index 41825c7d..20d41884 100644 --- a/tests/lbpm_disc_pp.cpp +++ b/tests/lbpm_disc_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI.h" // This includes mpi.h #include "common/SpherePack.h" /* @@ -147,6 +147,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; int depth; @@ -187,16 +189,16 @@ int main(int argc, char **argv) comm.barrier(); //................................................. // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&ndiscs,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&ndiscs,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -273,9 +275,9 @@ int main(int argc, char **argv) if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,ndiscs,0); - comm.bcast(cy,ndiscs,0); - comm.bcast(rad,ndiscs,0); + MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0){ @@ -344,7 +346,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -360,7 +362,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_inkbottle_pp.cpp b/tests/lbpm_inkbottle_pp.cpp index ca188633..669ab8c0 100644 --- a/tests/lbpm_inkbottle_pp.cpp +++ b/tests/lbpm_inkbottle_pp.cpp @@ -81,16 +81,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -197,7 +197,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_juanes_bench_disc_pp.cpp b/tests/lbpm_juanes_bench_disc_pp.cpp index a90d43f8..47d8cb84 100644 --- a/tests/lbpm_juanes_bench_disc_pp.cpp +++ b/tests/lbpm_juanes_bench_disc_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI.h" // This includes mpi.h #include "common/SpherePack.h" /* @@ -147,6 +147,9 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + if (rank == 0){ printf("********************************************************\n"); @@ -190,16 +193,16 @@ int main(int argc, char **argv) comm.barrier(); //................................................. // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&ndiscs,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&ndiscs,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -289,9 +292,9 @@ int main(int argc, char **argv) if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,ndiscs,0); - comm.bcast(cy,ndiscs,0); - comm.bcast(rad,ndiscs,0); + MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); /* if (rank == 0){ @@ -433,7 +436,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -449,7 +452,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index a25fef69..096dc790 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -94,6 +94,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; double REYNOLDS_NUMBER = 100.f; if (argc > 1){ @@ -156,28 +158,28 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); //................................................. - comm.bcast(&tau,1,0); - //comm.bcast(&pBC,1,0); - //comm.bcast(&Restart,1,0); - comm.bcast(&din,1,0); - comm.bcast(&dout,1,0); - comm.bcast(&Fx,1,0); - comm.bcast(&Fy,1,0); - comm.bcast(&Fz,1,0); - comm.bcast(×tepMax,1,0); - comm.bcast(&interval,1,0); - comm.bcast(&tol,1,0); + MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); + //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); + // MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); + MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); + MPI_Bcast(×tepMax,1,MPI_INT,0,comm); + MPI_Bcast(&interval,1,MPI_INT,0,comm); + MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -306,8 +308,8 @@ int main(int argc, char **argv) } } } - por_vol = comm.sumReduce( sum_local ); - //porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); + // MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = pore_vol*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... @@ -431,7 +433,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); //......................................... double D32,vawx,vawy,vawz,Fo,Re,velocity,err1D,mag_force,vel_prev; @@ -552,7 +554,7 @@ int main(int argc, char **argv) fclose(NONDARCY); ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; diff --git a/tests/lbpm_nonnewtonian_simulator.cpp b/tests/lbpm_nonnewtonian_simulator.cpp index bea3a814..ff8792e7 100644 --- a/tests/lbpm_nonnewtonian_simulator.cpp +++ b/tests/lbpm_nonnewtonian_simulator.cpp @@ -124,6 +124,8 @@ int main(int argc, char **argv) // int rank_xz,rank_XZ,rank_xZ,rank_Xz; // int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -426,8 +428,8 @@ int main(int argc, char **argv) } } - pore_vol = comm.sumReduce( sum_local ); /* 6 */ - //porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); /* 6 */ + //MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = pore_vol*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -572,7 +574,7 @@ int main(int argc, char **argv) timestep=5; } } - comm.bcast(×tep,1,0); + MPI_Bcast(×tep,1,MPI_INT,0,comm); // Read in the restart file to CPU buffers double *cDen = new double[2*N]; @@ -660,7 +662,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; comm.barrier(); - starttime = Utilities::MPI::time(); + starttime = MPI_Wtime(); /* * Create the thread pool @@ -808,7 +810,7 @@ int main(int argc, char **argv) //************************************************************************/ ScaLBL_DeviceBarrier(); comm.barrier(); - stoptime = Utilities::MPI::time(); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; @@ -833,6 +835,20 @@ int main(int argc, char **argv) + + + + + + + + + + + + + + // Scrap // if (rank==0){ diff --git a/tests/lbpm_plates_pp.cpp b/tests/lbpm_plates_pp.cpp index 37191979..acd64f52 100644 --- a/tests/lbpm_plates_pp.cpp +++ b/tests/lbpm_plates_pp.cpp @@ -31,6 +31,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; double TubeRadius =15.0; double WIDTH; @@ -75,16 +77,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -174,7 +176,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_porenetwork_pp.cpp b/tests/lbpm_porenetwork_pp.cpp index 1715811f..4a6ccda7 100644 --- a/tests/lbpm_porenetwork_pp.cpp +++ b/tests/lbpm_porenetwork_pp.cpp @@ -24,6 +24,9 @@ int main(int argc, char **argv) int iproc,jproc,kproc; int sendtag,recvtag; //***************************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; + //********************************** int nsph,ncyl, BC; nsph = atoi(argv[1]); @@ -64,16 +67,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -266,7 +269,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); if (rank==0) printf("Pore volume = %f \n",pore_vol/double(Nx*Ny*Nz)); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_random_pp.cpp b/tests/lbpm_random_pp.cpp index 8318f50f..ad4b83cc 100644 --- a/tests/lbpm_random_pp.cpp +++ b/tests/lbpm_random_pp.cpp @@ -98,16 +98,16 @@ int main(int argc, char **argv) } comm.barrier(); // Computational domain - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -166,7 +166,7 @@ int main(int argc, char **argv) } } // total Global is the number of nodes in the pore-space - totalGlobal = sumReduce( count ); + MPI_Allreduce(&count,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float porosity=float(totalGlobal)/(nprocx*nprocy*nprocz*(nx-2)*(ny-2)*(nz-2)); if (rank==0) printf("Media Porosity: %f \n",porosity); @@ -216,12 +216,12 @@ int main(int argc, char **argv) sizeY = SizeY[bin]; sizeZ = SizeZ[bin]; } - comm.bcast(&x,1,0); - comm.bcast(&y,1,0); - comm.bcast(&z,1,0); - comm.bcast(&sizeX,1,0); - comm.bcast(&sizeY,1,0); - comm.bcast(&sizeZ,1,0); + MPI_Bcast(&x,1,MPI_INT,0,comm); + MPI_Bcast(&y,1,MPI_INT,0,comm); + MPI_Bcast(&z,1,MPI_INT,0,comm); + MPI_Bcast(&sizeX,1,MPI_INT,0,comm); + MPI_Bcast(&sizeY,1,MPI_INT,0,comm); + MPI_Bcast(&sizeZ,1,MPI_INT,0,comm); //if (rank==0) printf("Broadcast block at %i,%i,%i \n",x,y,z); @@ -269,7 +269,7 @@ int main(int argc, char **argv) } } } - countGlobal = sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); sat = float(countGlobal)/totalGlobal; //if (rank==0) printf("New count=%i\n",countGlobal); //if (rank==0) printf("New saturation=%f\n",sat); @@ -345,24 +345,42 @@ int main(int argc, char **argv) PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... - comm.sendrecv(sendID_x,Dm.sendCount_x,Dm.rank_x(),sendtag,recvID_X,Dm.recvCount_X,Dm.rank_X(),recvtag); - comm.sendrecv(sendID_X,Dm.sendCount_X,Dm.rank_X(),sendtag,recvID_x,Dm.recvCount_x,Dm.rank_x(),recvtag); - comm.sendrecv(sendID_y,Dm.sendCount_y,Dm.rank_y(),sendtag,recvID_Y,Dm.recvCount_Y,Dm.rank_Y(),recvtag); - comm.sendrecv(sendID_Y,Dm.sendCount_Y,Dm.rank_Y(),sendtag,recvID_y,Dm.recvCount_y,Dm.rank_y(),recvtag); - comm.sendrecv(sendID_z,Dm.sendCount_z,Dm.rank_z(),sendtag,recvID_Z,Dm.recvCount_Z,Dm.rank_Z(),recvtag); - comm.sendrecv(sendID_Z,Dm.sendCount_Z,Dm.rank_Z(),sendtag,recvID_z,Dm.recvCount_z,Dm.rank_z(),recvtag); - comm.sendrecv(sendID_xy,Dm.sendCount_xy,Dm.rank_xy(),sendtag,recvID_XY,Dm.recvCount_XY,Dm.rank_XY(),recvtag); - comm.sendrecv(sendID_XY,Dm.sendCount_XY,Dm.rank_XY(),sendtag,recvID_xy,Dm.recvCount_xy,Dm.rank_xy(),recvtag); - comm.sendrecv(sendID_Xy,Dm.sendCount_Xy,Dm.rank_Xy(),sendtag,recvID_xY,Dm.recvCount_xY,Dm.rank_xY(),recvtag); - comm.sendrecv(sendID_xY,Dm.sendCount_xY,Dm.rank_xY(),sendtag,recvID_Xy,Dm.recvCount_Xy,Dm.rank_Xy(),recvtag); - comm.sendrecv(sendID_xz,Dm.sendCount_xz,Dm.rank_xz(),sendtag,recvID_XZ,Dm.recvCount_XZ,Dm.rank_XZ(),recvtag); - comm.sendrecv(sendID_XZ,Dm.sendCount_XZ,Dm.rank_XZ(),sendtag,recvID_xz,Dm.recvCount_xz,Dm.rank_xz(),recvtag); - comm.sendrecv(sendID_Xz,Dm.sendCount_Xz,Dm.rank_Xz(),sendtag,recvID_xZ,Dm.recvCount_xZ,Dm.rank_xZ(),recvtag); - comm.sendrecv(sendID_xZ,Dm.sendCount_xZ,Dm.rank_xZ(),sendtag,recvID_Xz,Dm.recvCount_Xz,Dm.rank_Xz(),recvtag); - comm.sendrecv(sendID_yz,Dm.sendCount_yz,Dm.rank_yz(),sendtag,recvID_YZ,Dm.recvCount_YZ,Dm.rank_YZ(),recvtag); - comm.sendrecv(sendID_YZ,Dm.sendCount_YZ,Dm.rank_YZ(),sendtag,recvID_yz,Dm.recvCount_yz,Dm.rank_yz(),recvtag); - comm.sendrecv(sendID_Yz,Dm.sendCount_Yz,Dm.rank_Yz(),sendtag,recvID_yZ,Dm.recvCount_yZ,Dm.rank_yZ(),recvtag); - comm.sendrecv(sendID_yZ,Dm.sendCount_yZ,Dm.rank_yZ(),sendtag,recvID_Yz,Dm.recvCount_Yz,Dm.rank_Yz(),recvtag); + MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); @@ -394,7 +412,7 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); sat = float(countGlobal)/totalGlobal; if (rank==0) printf("Final saturation=%f\n",sat); diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index 65b8576f..1bc89adb 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -85,23 +85,23 @@ int main(int argc, char **argv) comm.barrier(); // Computational domain //................................................. - comm.bcast(&nx,1,0); - comm.bcast(&ny,1,0); - comm.bcast(&nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&nx,1,MPI_INT,0,comm); + MPI_Bcast(&ny,1,MPI_INT,0,comm); + MPI_Bcast(&nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&xStart,1,0); - comm.bcast(&yStart,1,0); - comm.bcast(&zStart,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&xStart,1,MPI_INT,0,comm); + MPI_Bcast(&yStart,1,MPI_INT,0,comm); + MPI_Bcast(&zStart,1,MPI_INT,0,comm); //................................................. comm.barrier(); @@ -191,7 +191,7 @@ int main(int argc, char **argv) } else{ printf("Sending data to process %i \n", rnk); - comm.send(tmp,N,rnk,15); + MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm); } } } @@ -200,7 +200,7 @@ int main(int argc, char **argv) else{ // Recieve the subdomain from rank = 0 printf("Ready to recieve data %i at process %i \n", N,rank); - comm.recv(Dm.id,N,0,15); + MPI_Recv(Dm.id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE); } comm.barrier(); @@ -243,8 +243,8 @@ int main(int argc, char **argv) printf("Original label=%i, New label=%i \n",oldlabel,newlabel); } } - comm.barrier(); - comm.bcast(LabelList,2*NLABELS,0); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Bcast(LabelList,2*NLABELS,MPI_INT,0,MPI_COMM_WORLD); char *newIDs; newIDs= new char [nx*ny*nz]; @@ -278,8 +278,8 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); - totalGlobal = comm.sumReduce( total ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); + MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float porosity = float(totalGlobal-countGlobal)/totalGlobal; @@ -321,8 +321,8 @@ int main(int argc, char **argv) } } } - countGlobal = comm.sumReduce( count ); - totalGlobal = comm.sumReduce( total ); + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); + MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm); float saturation = float(countGlobal)/totalGlobal; if (rank==0) printf("wetting phase saturation=%f\n",saturation); diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 484a11e2..39cf0bd1 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -180,7 +180,7 @@ int main(int argc, char **argv) fflush(stdout); porosity = ReadFromBlock(Dm->id,Dm->iproc(),Dm->jproc(),Dm->kproc(),nx,ny,nz); - comm.barrier(); + MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("Writing local ID files (poros=%f) \n",porosity); fflush(stdout); FILE *ID = fopen(LocalRankFilename,"wb"); diff --git a/tests/lbpm_sphere_pp.cpp b/tests/lbpm_sphere_pp.cpp index 0df11b96..2e053eed 100644 --- a/tests/lbpm_sphere_pp.cpp +++ b/tests/lbpm_sphere_pp.cpp @@ -38,6 +38,8 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; if (rank == 0){ printf("********************************************************\n"); @@ -123,10 +125,10 @@ int main(int argc, char **argv) if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); comm.barrier(); // Broadcast the sphere packing to all processes - comm.bcast(cx,nspheres,0); - comm.bcast(cy,nspheres,0); - comm.bcast(cz,nspheres,0); - comm.bcast(rad,nspheres,0); + MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); + MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... comm.barrier(); if (rank == 0) cout << "Domain set." << endl; @@ -142,7 +144,7 @@ int main(int argc, char **argv) D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx; printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D); } - comm.bcast(&D,1,0); + MPI_Bcast(&D,1,MPI_DOUBLE,0,comm); //....................................................................... SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz, @@ -175,7 +177,7 @@ int main(int argc, char **argv) } } sum_local = 1.0*sum; - porosity = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); porosity = porosity*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); @@ -191,7 +193,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners diff --git a/tests/lbpm_squaretube_pp.cpp b/tests/lbpm_squaretube_pp.cpp index a4ee5f60..c1f05aee 100644 --- a/tests/lbpm_squaretube_pp.cpp +++ b/tests/lbpm_squaretube_pp.cpp @@ -30,6 +30,9 @@ int main(int argc, char **argv) int rank_xy,rank_XY,rank_xY,rank_Xy; int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; + //********************************** + MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; int ORIENTATION=2; //default: the tube is aligned with Z axis //ORIENTATION = 0: tube is aligned with X axis @@ -80,16 +83,16 @@ int main(int argc, char **argv) // Broadcast simulation parameters from rank 0 to all other procs comm.barrier(); // Computational domain - comm.bcast(&Nx,1,0); - comm.bcast(&Ny,1,0); - comm.bcast(&Nz,1,0); - comm.bcast(&nprocx,1,0); - comm.bcast(&nprocy,1,0); - comm.bcast(&nprocz,1,0); - comm.bcast(&nspheres,1,0); - comm.bcast(&Lx,1,0); - comm.bcast(&Ly,1,0); - comm.bcast(&Lz,1,0); + MPI_Bcast(&Nx,1,MPI_INT,0,comm); + MPI_Bcast(&Ny,1,MPI_INT,0,comm); + MPI_Bcast(&Nz,1,MPI_INT,0,comm); + MPI_Bcast(&nprocx,1,MPI_INT,0,comm); + MPI_Bcast(&nprocy,1,MPI_INT,0,comm); + MPI_Bcast(&nprocz,1,MPI_INT,0,comm); + MPI_Bcast(&nspheres,1,MPI_INT,0,comm); + MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); + MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. comm.barrier(); @@ -232,7 +235,7 @@ int main(int argc, char **argv) } } } - pore_vol = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); //......................................................... // don't perform computations at the eight corners From 05cafcb525c1a82a922df455eb6e207b430560c9 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 17 Mar 2020 21:44:45 -0400 Subject: [PATCH 047/270] fix failed merge --- CMakeLists.txt | 344 +- IO/MeshDatabase.cpp | 145 +- IO/MeshDatabase.h | 4 +- IO/PIO.cpp | 12 +- IO/PackData.cpp | 105 - IO/PackData.h | 78 - IO/Writer.cpp | 20 +- IO/Writer.h | 4 +- IO/netcdf.cpp | 12 +- IO/netcdf.h | 4 +- IO/silo.cpp | 2 +- IO/silo.h | 2 +- IO/silo.hpp | 2 +- analysis/Minkowski.cpp | 16 +- analysis/Minkowski.h | 2 +- analysis/SubPhase.cpp | 154 +- analysis/SubPhase.h | 2 +- analysis/TwoPhase.cpp | 89 +- analysis/TwoPhase.h | 2 +- analysis/analysis.cpp | 83 +- analysis/analysis.h | 8 +- analysis/distance.cpp | 2 +- analysis/morphology.cpp | 106 +- analysis/runAnalysis.cpp | 49 +- analysis/runAnalysis.h | 8 +- analysis/uCT.cpp | 11 +- cmake/FindHIP.cmake | 579 ---- common/Communication.h | 198 +- common/Communication.hpp | 53 +- common/Domain.cpp | 230 +- common/Domain.h | 6 +- common/MPI.I | 1143 ------- common/MPI.cpp | 3758 --------------------- common/MPI.h | 1152 ------- common/MPI_Helpers.cpp | 266 ++ common/MPI_Helpers.h | 239 ++ IO/PackData.hpp => common/MPI_Helpers.hpp | 9 +- common/ReadMicroCT.cpp | 4 +- common/ReadMicroCT.h | 3 +- common/ScaLBL.cpp | 226 +- common/ScaLBL.h | 3 +- common/SpherePack.cpp | 1 + common/SpherePack.h | 1 + common/UnitTest.cpp | 211 +- common/UnitTest.h | 71 +- common/UtilityMacros.h | 28 +- cpu/BGK.cpp | 5 +- cpu/Color.cpp | 51 +- cpu/exe/lb2_Color_mpi.cpp | 2 +- cpu/exe/lb2_Color_wia_mpi_bubble.cpp | 2 +- models/ColorModel.cpp | 57 +- models/ColorModel.h | 6 +- models/DFHModel.cpp | 35 +- models/DFHModel.h | 6 +- models/MRTModel.cpp | 49 +- models/MRTModel.h | 6 +- tests/BlobAnalyzeParallel.cpp | 21 +- tests/BlobIdentifyParallel.cpp | 9 +- tests/ColorToBinary.cpp | 9 +- tests/ComponentLabel.cpp | 9 +- tests/GenerateSphereTest.cpp | 75 +- tests/TestBlobAnalyze.cpp | 17 +- tests/TestBlobIdentify.cpp | 37 +- tests/TestBlobIdentifyCorners.cpp | 5 +- tests/TestBubble.cpp | 46 +- tests/TestBubbleDFH.cpp | 32 +- tests/TestColorBubble.cpp | 14 +- tests/TestColorGrad.cpp | 24 +- tests/TestColorGradDFH.cpp | 18 +- tests/TestColorMassBounceback.cpp | 32 +- tests/TestColorSquareTube.cpp | 14 +- tests/TestCommD3Q19.cpp | 23 +- tests/TestDatabase.cpp | 9 +- tests/TestFluxBC.cpp | 18 +- tests/TestForceD3Q19.cpp | 7 +- tests/TestForceMoments.cpp | 30 +- tests/TestInterfaceSpeed.cpp | 32 +- tests/TestMRT.cpp | 38 +- tests/TestMap.cpp | 17 +- tests/TestMassConservationD3Q7.cpp | 11 +- tests/TestMicroCTReader.cpp | 10 +- tests/TestMomentsD3Q19.cpp | 9 +- tests/TestNetcdf.cpp | 10 +- tests/TestPoiseuille.cpp | 18 +- tests/TestPressVel.cpp | 23 +- tests/TestSegDist.cpp | 13 +- tests/TestSubphase.cpp | 9 +- tests/TestTopo3D.cpp | 9 +- tests/TestTorus.cpp | 9 +- tests/TestTorusEvolve.cpp | 9 +- tests/TestTwoPhase.cpp | 11 +- tests/TestWriter.cpp | 21 +- tests/convertIO.cpp | 15 +- tests/hello_world.cpp | 11 +- tests/lb2_CMT_wia.cpp | 2 +- tests/lb2_Color_blob_wia_mpi.cpp | 48 +- tests/lbpm_BGK_simulator.cpp | 33 +- tests/lbpm_captube_pp.cpp | 16 +- tests/lbpm_color_macro_simulator.cpp | 36 +- tests/lbpm_color_simulator.cpp | 14 +- tests/lbpm_dfh_simulator.cpp | 12 +- tests/lbpm_disc_pp.cpp | 24 +- tests/lbpm_inkbottle_pp.cpp | 20 +- tests/lbpm_juanes_bench_disc_pp.cpp | 26 +- tests/lbpm_minkowski_scalar.cpp | 23 +- tests/lbpm_morph_pp.cpp | 22 +- tests/lbpm_morphdrain_pp.cpp | 12 +- tests/lbpm_morphopen_pp.cpp | 12 +- tests/lbpm_nondarcy_simulator.cpp | 28 +- tests/lbpm_nonnewtonian_simulator.cpp | 83 +- tests/lbpm_nonnewtonian_simulator.h | 40 +- tests/lbpm_permeability_simulator.cpp | 13 +- tests/lbpm_plates_pp.cpp | 20 +- tests/lbpm_porenetwork_pp.cpp | 20 +- tests/lbpm_random_pp.cpp | 13 +- tests/lbpm_refine_pp.cpp | 9 +- tests/lbpm_segmented_decomp.cpp | 20 +- tests/lbpm_segmented_pp.cpp | 9 +- tests/lbpm_sphere_pp.cpp | 18 +- tests/lbpm_squaretube_pp.cpp | 20 +- tests/lbpm_uCT_maskfilter.cpp | 16 +- tests/lbpm_uCT_pp.cpp | 37 +- tests/testCommunication.cpp | 34 +- tests/test_dcel_minkowski.cpp | 8 +- tests/test_dcel_tri_normal.cpp | 4 +- 125 files changed, 2544 insertions(+), 8538 deletions(-) delete mode 100644 IO/PackData.cpp delete mode 100644 IO/PackData.h delete mode 100644 cmake/FindHIP.cmake delete mode 100644 common/MPI.I delete mode 100644 common/MPI.cpp delete mode 100644 common/MPI.h create mode 100644 common/MPI_Helpers.cpp create mode 100644 common/MPI_Helpers.h rename IO/PackData.hpp => common/MPI_Helpers.hpp (95%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e7eeaea..acc2c2dc 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,174 +1,170 @@ -# Set some CMake properties -CMAKE_MINIMUM_REQUIRED( VERSION 3.9 ) - - -MESSAGE("====================") -MESSAGE("Configuring LBPM-WIA") -MESSAGE("====================") - - -# Set the project name -SET( PROJ LBPM ) # Set the project name for CMake -SET( LBPM_LIB lbpm-wia ) # Set the final library name -SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...) -SET( TEST_MAX_PROCS 16 ) - - -# Initialize the project -PROJECT( ${PROJ} LANGUAGES CXX ) - - -# Prevent users from building in place -IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" ) - MESSAGE( FATAL_ERROR "Building code in place is a bad idea" ) -ENDIF() - - -# Set the default C++ standard -SET( CMAKE_CXX_EXTENSIONS OFF ) -IF ( NOT CMAKE_CXX_STANDARD ) - IF ( CXX_STD ) - MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" ) - ENDIF() - SET( CMAKE_CXX_STANDARD 14 ) -ENDIF() -IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) ) - MESSAGE( FATAL_ERROR "C++14 or newer required" ) -ENDIF() - - -# Set source/install paths -SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ) -SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) -IF( ${PROJ}_INSTALL_DIR ) - SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" ) -ELSEIF( PREFIX ) - SET( ${PROJ}_INSTALL_DIR "${PREFIX}" ) -ELSEIF( NOT ${PROJ}_INSTALL_DIR ) - SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) -ENDIF() -INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" ) -SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake ) - - -# Include macros -INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" ) -INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" ) -INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" ) - - -# Check if we are only compiling docs -CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 ) - - -# Set testing paramaters -SET( DROP_METHOD "http" ) -SET( DROP_SITE "" ) -SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" ) -SET( TRIGGER_SITE "" ) -SET( DROP_SITE_CDASH TRUE ) -ENABLE_TESTING() -INCLUDE( CTest ) - - -# Check the compile mode and compile flags -IF ( NOT ONLY_BUILD_DOCS ) - CONFIGURE_SYSTEM() -ENDIF() - - -# Add some directories to include -INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" ) - - -# Create the target for documentation -ADD_CUSTOM_TARGET( doc ) -ADD_CUSTOM_TARGET( latex_docs ) -CHECK_ENABLE_FLAG( USE_DOXYGEN 1 ) -CHECK_ENABLE_FLAG( USE_LATEX 1 ) -FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" ) -IF ( USE_DOXYGEN ) - SET( DOXYFILE_LATEX YES ) - SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" ) - SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" ) - SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" ) - SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" ) - SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" ) - SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" ) - SET( REL_PACKAGE_HTML "" ) - SET( DOXYGEN_MACROS "" ) - MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}") - INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" ) - IF ( DOXYGEN_FOUND ) - ADD_DEPENDENCIES( doxygen latex_docs ) - ADD_DEPENDENCIES( doc latex_docs doxygen ) - ELSE() - SET( USE_DOXYGEN 0 ) - ENDIF() -ENDIF() - - -# Create custom targets for build-test, check, and distclean -ADD_CUSTOM_TARGET( build-test ) -ADD_CUSTOM_TARGET( build-examples ) -ADD_CUSTOM_TARGET( check COMMAND make test ) -ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace ) - - -# Check for CUDA -CHECK_ENABLE_FLAG( USE_CUDA 0 ) -CHECK_ENABLE_FLAG( USE_HIP 0 ) -NULL_USE( CMAKE_CUDA_FLAGS ) -IF ( USE_CUDA ) - ADD_DEFINITIONS( -DUSE_CUDA ) - ENABLE_LANGUAGE( CUDA ) -ELSEIF ( USE_HIP ) - FIND_PACKAGE( HIP ) - MESSAGE( FATAL_ERROR "STOP" ) -ENDIF() - - -# Configure external packages -IF ( NOT ONLY_BUILD_DOCS ) - CONFIGURE_MPI() # MPI must be before other libraries - CONFIGURE_MIC() - CONFIGURE_NETCDF() - CONFIGURE_SILO() - CONFIGURE_LBPM() - CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" ) - CONFIGURE_LINE_COVERAGE() - # Set the external library link list - SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} ) -ENDIF() - - - -# Macro to create 1,2,4 processor tests -MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} ) - ADD_LBPM_TEST( ${EXENAME} ${ARGN} ) - ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} ) - ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} ) -ENDMACRO() - - -# Add the src directories -IF ( NOT ONLY_BUILD_DOCS ) - BEGIN_PACKAGE_CONFIG( lbpm-wia-library ) - ADD_PACKAGE_SUBDIRECTORY( common ) - ADD_PACKAGE_SUBDIRECTORY( analysis ) - ADD_PACKAGE_SUBDIRECTORY( IO ) - ADD_PACKAGE_SUBDIRECTORY( threadpool ) - ADD_PACKAGE_SUBDIRECTORY( StackTrace ) - ADD_PACKAGE_SUBDIRECTORY( models ) - IF ( USE_CUDA ) - ADD_PACKAGE_SUBDIRECTORY( gpu ) - ELSE() - ADD_PACKAGE_SUBDIRECTORY( cpu ) - ENDIF() - INSTALL_LBPM_TARGET( lbpm-wia-library ) - ADD_SUBDIRECTORY( tests ) - ADD_SUBDIRECTORY( example ) - #ADD_SUBDIRECTORY( workflows ) - INSTALL_PROJ_LIB() -ENDIF() - +# Set some CMake properties +CMAKE_MINIMUM_REQUIRED( VERSION 3.9 ) + + +MESSAGE("====================") +MESSAGE("Configuring LBPM-WIA") +MESSAGE("====================") + + +# Set the project name +SET( PROJ LBPM ) # Set the project name for CMake +SET( LBPM_LIB lbpm-wia ) # Set the final library name +SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...) +SET( TEST_MAX_PROCS 16 ) + + +# Initialize the project +PROJECT( ${PROJ} LANGUAGES CXX ) + + +# Prevent users from building in place +IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" ) + MESSAGE( FATAL_ERROR "Building code in place is a bad idea" ) +ENDIF() + + +# Set the default C++ standard +SET( CMAKE_CXX_EXTENSIONS OFF ) +IF ( NOT CMAKE_CXX_STANDARD ) + IF ( CXX_STD ) + MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" ) + ENDIF() + SET( CMAKE_CXX_STANDARD 14 ) +ENDIF() +IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) ) + MESSAGE( FATAL_ERROR "C++14 or newer required" ) +ENDIF() + + +# Set source/install paths +SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" ) +SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) +IF( ${PROJ}_INSTALL_DIR ) + SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" ) +ELSEIF( PREFIX ) + SET( ${PROJ}_INSTALL_DIR "${PREFIX}" ) +ELSEIF( NOT ${PROJ}_INSTALL_DIR ) + SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) +ENDIF() +INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" ) +SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake ) + + +# Include macros +INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" ) +INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" ) +INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" ) + + +# Check if we are only compiling docs +CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 ) + + +# Set testing paramaters +SET( DROP_METHOD "http" ) +SET( DROP_SITE "" ) +SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" ) +SET( TRIGGER_SITE "" ) +SET( DROP_SITE_CDASH TRUE ) +ENABLE_TESTING() +INCLUDE( CTest ) + + +# Check the compile mode and compile flags +IF ( NOT ONLY_BUILD_DOCS ) + CONFIGURE_SYSTEM() +ENDIF() + + +# Add some directories to include +INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" ) + + +# Create the target for documentation +ADD_CUSTOM_TARGET( doc ) +ADD_CUSTOM_TARGET( latex_docs ) +CHECK_ENABLE_FLAG( USE_DOXYGEN 1 ) +CHECK_ENABLE_FLAG( USE_LATEX 1 ) +FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" ) +IF ( USE_DOXYGEN ) + SET( DOXYFILE_LATEX YES ) + SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" ) + SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" ) + SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" ) + SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" ) + SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" ) + SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" ) + SET( REL_PACKAGE_HTML "" ) + SET( DOXYGEN_MACROS "" ) + MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}") + INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" ) + IF ( DOXYGEN_FOUND ) + ADD_DEPENDENCIES( doxygen latex_docs ) + ADD_DEPENDENCIES( doc latex_docs doxygen ) + ELSE() + SET( USE_DOXYGEN 0 ) + ENDIF() +ENDIF() + + +# Create custom targets for build-test, check, and distclean +ADD_CUSTOM_TARGET( build-test ) +ADD_CUSTOM_TARGET( build-examples ) +ADD_CUSTOM_TARGET( check COMMAND make test ) +ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace ) + + +# Check for CUDA +CHECK_ENABLE_FLAG( USE_CUDA 0 ) +NULL_USE( CMAKE_CUDA_FLAGS ) +IF ( USE_CUDA ) + ADD_DEFINITIONS( -DUSE_CUDA ) + ENABLE_LANGUAGE( CUDA ) +ENDIF() + + +# Configure external packages +IF ( NOT ONLY_BUILD_DOCS ) + CONFIGURE_MPI() # MPI must be before other libraries + CONFIGURE_MIC() + CONFIGURE_NETCDF() + CONFIGURE_SILO() + CONFIGURE_LBPM() + CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" ) + CONFIGURE_LINE_COVERAGE() + # Set the external library link list + SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} ) +ENDIF() + + + +# Macro to create 1,2,4 processor tests +MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} ) + ADD_LBPM_TEST( ${EXENAME} ${ARGN} ) + ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} ) + ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} ) +ENDMACRO() + + +# Add the src directories +IF ( NOT ONLY_BUILD_DOCS ) + BEGIN_PACKAGE_CONFIG( lbpm-wia-library ) + ADD_PACKAGE_SUBDIRECTORY( common ) + ADD_PACKAGE_SUBDIRECTORY( analysis ) + ADD_PACKAGE_SUBDIRECTORY( IO ) + ADD_PACKAGE_SUBDIRECTORY( threadpool ) + ADD_PACKAGE_SUBDIRECTORY( StackTrace ) + ADD_PACKAGE_SUBDIRECTORY( models ) + IF ( USE_CUDA ) + ADD_PACKAGE_SUBDIRECTORY( gpu ) + ELSE() + ADD_PACKAGE_SUBDIRECTORY( cpu ) + ENDIF() + INSTALL_LBPM_TARGET( lbpm-wia-library ) + ADD_SUBDIRECTORY( tests ) + ADD_SUBDIRECTORY( example ) + #ADD_SUBDIRECTORY( workflows ) + INSTALL_PROJ_LIB() +ENDIF() + diff --git a/IO/MeshDatabase.cpp b/IO/MeshDatabase.cpp index 2c03ddde..1fad9231 100644 --- a/IO/MeshDatabase.cpp +++ b/IO/MeshDatabase.cpp @@ -1,8 +1,7 @@ #include "IO/MeshDatabase.h" #include "IO/Mesh.h" -#include "IO/PackData.h" #include "IO/IOHelpers.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include @@ -14,6 +13,8 @@ +/**************************************************** +****************************************************/ // MeshType template<> size_t packsize( const IO::MeshType& rhs ) @@ -246,76 +247,80 @@ void DatabaseEntry::read( const std::string& line ) // Gather the mesh databases from all processors inline int tod( int N ) { return (N+7)/sizeof(double); } -std::vector gatherAll( const std::vector& meshes, const Utilities::MPI& comm ) +std::vector gatherAll( const std::vector& meshes, MPI_Comm comm ) { - if ( comm.getSize() == 1 ) - return meshes; - PROFILE_START("gatherAll"); - PROFILE_START("gatherAll-pack",2); - int size = comm.getSize(); - // First pack the mesh data to local buffers - int localsize = 0; - for (size_t i=0; i data; - pos = 0; - while ( pos < globalsize ) { - MeshDatabase tmp; - unpack(tmp,(char*)&globalbuf[pos]); - pos += tod(packsize(tmp)); - std::map::iterator it = data.find(tmp.name); - if ( it==data.end() ) { - data[tmp.name] = tmp; - } else { - for (size_t i=0; isecond.domains.push_back(tmp.domains[i]); - for (size_t i=0; isecond.variables.push_back(tmp.variables[i]); - it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end()); + #ifdef USE_MPI + PROFILE_START("gatherAll"); + PROFILE_START("gatherAll-pack",2); + int size = MPI_WORLD_SIZE(); + // First pack the mesh data to local buffers + int localsize = 0; + for (size_t i=0; i data2(it->second.variables.begin(),it->second.variables.end()); - it->second.variables = std::vector(data2.begin(),data2.end()); - } - // Free temporary memory - delete [] localbuf; - delete [] disp; - delete [] globalbuf; - // Return the results - std::vector data2(data.size()); - size_t i=0; - for (std::map::iterator it=data.begin(); it!=data.end(); ++it, ++i) - data2[i] = it->second; - PROFILE_STOP("gatherAll-unpack",2); - PROFILE_STOP("gatherAll"); - return data2; + PROFILE_STOP("gatherAll-pack",2); + // Get the number of bytes each processor will be sending/recieving + PROFILE_START("gatherAll-send1",2); + auto recvsize = new int[size]; + MPI_Allgather(&localsize,1,MPI_INT,recvsize,1,MPI_INT,comm); + int globalsize = recvsize[0]; + auto disp = new int[size]; + disp[0] = 0; + for (int i=1; i data; + pos = 0; + while ( pos < globalsize ) { + MeshDatabase tmp; + unpack(tmp,(char*)&globalbuf[pos]); + pos += tod(packsize(tmp)); + std::map::iterator it = data.find(tmp.name); + if ( it==data.end() ) { + data[tmp.name] = tmp; + } else { + for (size_t i=0; isecond.domains.push_back(tmp.domains[i]); + for (size_t i=0; isecond.variables.push_back(tmp.variables[i]); + it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end()); + } + } + for (std::map::iterator it=data.begin(); it!=data.end(); ++it) { + // Get the unique variables + std::set data2(it->second.variables.begin(),it->second.variables.end()); + it->second.variables = std::vector(data2.begin(),data2.end()); + } + // Free temporary memory + delete [] localbuf; + delete [] recvsize; + delete [] disp; + delete [] globalbuf; + // Return the results + std::vector data2(data.size()); + size_t i=0; + for (std::map::iterator it=data.begin(); it!=data.end(); ++it, ++i) + data2[i] = it->second; + PROFILE_STOP("gatherAll-unpack",2); + PROFILE_STOP("gatherAll"); + return data2; + #else + return meshes; + #endif } diff --git a/IO/MeshDatabase.h b/IO/MeshDatabase.h index 8e501624..9f544925 100644 --- a/IO/MeshDatabase.h +++ b/IO/MeshDatabase.h @@ -2,7 +2,7 @@ #define MeshDatabase_INC #include "IO/Mesh.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include #include @@ -70,7 +70,7 @@ public: //! Gather the mesh databases from all processors -std::vector gatherAll( const std::vector& meshes, const Utilities::MPI& comm ); +std::vector gatherAll( const std::vector& meshes, MPI_Comm comm ); //! Write the mesh databases to a file diff --git a/IO/PIO.cpp b/IO/PIO.cpp index 3c2f3934..6c6ece2d 100644 --- a/IO/PIO.cpp +++ b/IO/PIO.cpp @@ -1,6 +1,6 @@ #include "IO/PIO.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include #include @@ -36,7 +36,10 @@ static void shutdownFilestream( ) } void Utilities::logOnlyNodeZero( const std::string &filename ) { - int rank = ::Utilities::MPI( MPI_COMM_WORLD ).getRank(); + int rank = 0; + #ifdef USE_MPI + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + #endif if ( rank == 0 ) logAllNodes(filename,true); } @@ -51,7 +54,10 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream ) // Open the log stream and redirect output std::string full_filename = filename; if ( !singleStream ) { - int rank = ::Utilities::MPI( MPI_COMM_WORLD ).getRank(); + int rank = 0; + #ifdef USE_MPI + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); + #endif char tmp[100]; sprintf(tmp,".%04i",rank); full_filename += std::string(tmp); diff --git a/IO/PackData.cpp b/IO/PackData.cpp deleted file mode 100644 index f10d9ca7..00000000 --- a/IO/PackData.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "IO/PackData.h" - -#include - - -/******************************************************** -* Concrete implimentations for packing/unpacking * -********************************************************/ -// unsigned char -template<> -size_t packsize( const unsigned char& rhs ) -{ - return sizeof(unsigned char); -} -template<> -void pack( const unsigned char& rhs, char *buffer ) -{ - memcpy(buffer,&rhs,sizeof(unsigned char)); -} -template<> -void unpack( unsigned char& data, const char *buffer ) -{ - memcpy(&data,buffer,sizeof(unsigned char)); -} -// char -template<> -size_t packsize( const char& rhs ) -{ - return sizeof(char); -} -template<> -void pack( const char& rhs, char *buffer ) -{ - memcpy(buffer,&rhs,sizeof(char)); -} -template<> -void unpack( char& data, const char *buffer ) -{ - memcpy(&data,buffer,sizeof(char)); -} -// int -template<> -size_t packsize( const int& rhs ) -{ - return sizeof(int); -} -template<> -void pack( const int& rhs, char *buffer ) -{ - memcpy(buffer,&rhs,sizeof(int)); -} -template<> -void unpack( int& data, const char *buffer ) -{ - memcpy(&data,buffer,sizeof(int)); -} -// unsigned int -template<> -size_t packsize( const unsigned int& rhs ) -{ - return sizeof(unsigned int); -} -template<> -void pack( const unsigned int& rhs, char *buffer ) -{ - memcpy(buffer,&rhs,sizeof(int)); -} -template<> -void unpack( unsigned int& data, const char *buffer ) -{ - memcpy(&data,buffer,sizeof(int)); -} -// size_t -template<> -size_t packsize( const size_t& rhs ) -{ - return sizeof(size_t); -} -template<> -void pack( const size_t& rhs, char *buffer ) -{ - memcpy(buffer,&rhs,sizeof(size_t)); -} -template<> -void unpack( size_t& data, const char *buffer ) -{ - memcpy(&data,buffer,sizeof(size_t)); -} -// std::string -template<> -size_t packsize( const std::string& rhs ) -{ - return rhs.size()+1; -} -template<> -void pack( const std::string& rhs, char *buffer ) -{ - memcpy(buffer,rhs.c_str(),rhs.size()+1); -} -template<> -void unpack( std::string& data, const char *buffer ) -{ - data = std::string(buffer); -} - diff --git a/IO/PackData.h b/IO/PackData.h deleted file mode 100644 index 85326c0b..00000000 --- a/IO/PackData.h +++ /dev/null @@ -1,78 +0,0 @@ -// This file contains unctions to pack/unpack data structures -#ifndef included_PackData -#define included_PackData - -#include -#include -#include - - -//! Template function to return the buffer size required to pack a class -template -size_t packsize( const TYPE& rhs ); - -//! Template function to pack a class to a buffer -template -void pack( const TYPE& rhs, char *buffer ); - -//! Template function to unpack a class from a buffer -template -void unpack( TYPE& data, const char *buffer ); - - -//! Template function to return the buffer size required to pack a std::vector -template -size_t packsize( const std::vector& rhs ); - -//! Template function to pack a class to a buffer -template -void pack( const std::vector& rhs, char *buffer ); - -//! Template function to pack a class to a buffer -template -void unpack( std::vector& data, const char *buffer ); - - -//! Template function to return the buffer size required to pack a std::pair -template -size_t packsize( const std::pair& rhs ); - -//! Template function to pack a class to a buffer -template -void pack( const std::pair& rhs, char *buffer ); - -//! Template function to pack a class to a buffer -template -void unpack( std::pair& data, const char *buffer ); - - -//! Template function to return the buffer size required to pack a std::map -template -size_t packsize( const std::map& rhs ); - -//! Template function to pack a class to a buffer -template -void pack( const std::map& rhs, char *buffer ); - -//! Template function to pack a class to a buffer -template -void unpack( std::map& data, const char *buffer ); - - -//! Template function to return the buffer size required to pack a std::set -template -size_t packsize( const std::set& rhs ); - -//! Template function to pack a class to a buffer -template -void pack( const std::set& rhs, char *buffer ); - -//! Template function to pack a class to a buffer -template -void unpack( std::set& data, const char *buffer ); - - -#include "IO/PackData.hpp" - -#endif - diff --git a/IO/Writer.cpp b/IO/Writer.cpp index 61c333af..6581ad42 100644 --- a/IO/Writer.cpp +++ b/IO/Writer.cpp @@ -2,7 +2,7 @@ #include "IO/MeshDatabase.h" #include "IO/IOHelpers.h" #include "IO/silo.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include @@ -36,7 +36,7 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap global_IO_format = Format::SILO; else ERROR("Unknown format"); - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = comm_rank(MPI_COMM_WORLD); if ( !append && rank==0 ) { mkdir(path.c_str(),S_IRWXU|S_IRGRP); std::string filename; @@ -55,7 +55,7 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap // Write the mesh data in the original format static std::vector writeMeshesOrigFormat( const std::vector& meshData, const std::string& path ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = MPI_WORLD_RANK(); std::vector meshes_written; for (size_t i=0; i writeMeshesOrigFormat( const std::vector& meshes_written, cons static std::vector writeMeshesNewFormat( const std::vector& meshData, const std::string& path, int format ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = MPI_WORLD_RANK(); std::vector meshes_written; char filename[100], fullpath[200]; sprintf(filename,"%05i",rank); @@ -419,7 +419,7 @@ static std::vector writeMeshesSilo( const std::vector& meshData, const std::string& path, int format ) { #ifdef USE_SILO - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = MPI_WORLD_RANK(); std::vector meshes_written; char filename[100], fullpath[200]; sprintf(filename,"%05i.silo",rank); @@ -441,12 +441,12 @@ static std::vector writeMeshesSilo( /**************************************************** * Write the mesh data * ****************************************************/ -void IO::writeData( const std::string& subdir, const std::vector& meshData, const Utilities::MPI& comm ) +void IO::writeData( const std::string& subdir, const std::vector& meshData, MPI_Comm comm ) { if ( global_IO_path.empty() ) IO::initialize( ); PROFILE_START("writeData"); - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = comm_rank(comm); // Check the meshData before writing for ( const auto& data : meshData ) { if ( !data.check() ) @@ -457,7 +457,7 @@ void IO::writeData( const std::string& subdir, const std::vector meshes_written; if ( global_IO_format == Format::OLD ) { diff --git a/IO/Writer.h b/IO/Writer.h index dfc22db8..710fa0d8 100644 --- a/IO/Writer.h +++ b/IO/Writer.h @@ -34,7 +34,7 @@ void initialize( const std::string& path="", const std::string& format="silo", b * @param[in] meshData The data to write * @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof) */ -void writeData( const std::string& subdir, const std::vector& meshData, const Utilities::MPI& comm ); +void writeData( const std::string& subdir, const std::vector& meshData, MPI_Comm comm ); /*! @@ -44,7 +44,7 @@ void writeData( const std::string& subdir, const std::vector * @param[in] meshData The data to write * @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof) */ -inline void writeData( int timestep, const std::vector& meshData, const Utilities::MPI& comm ) +inline void writeData( int timestep, const std::vector& meshData, MPI_Comm comm ) { char subdir[100]; sprintf(subdir,"vis%03i",timestep); diff --git a/IO/netcdf.cpp b/IO/netcdf.cpp index e061579a..b36bb6d6 100644 --- a/IO/netcdf.cpp +++ b/IO/netcdf.cpp @@ -1,6 +1,6 @@ #include "IO/netcdf.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" @@ -116,7 +116,7 @@ std::string VariableTypeName( VariableType type ) /**************************************************** * Open/close a file * ****************************************************/ -int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm ) +int open( const std::string& filename, FileMode mode, MPI_Comm comm ) { int fid = 0; if ( comm == MPI_COMM_NULL ) { @@ -134,13 +134,13 @@ int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm } } else { if ( mode == READ ) { - int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else if ( mode == WRITE ) { - int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else if ( mode == CREATE ) { - int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid ); + int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm, MPI_INFO_NULL, &fid ); CHECK_NC_ERR( err ); } else { ERROR("Unknown file mode"); @@ -375,7 +375,7 @@ Array getVar( int fid, const std::string& var, const std::vector& sta std::vector var_size = getVarDim( fid, var ); for (int d=0; d<(int)var_size.size(); d++) { if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) { - int rank = Utilities::MPI(MPI_COMM_WORLD).getRank(); + int rank = comm_rank(MPI_COMM_WORLD); char tmp[1000]; sprintf(tmp,"%i: Range exceeded array dimension:\n" " start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i", diff --git a/IO/netcdf.h b/IO/netcdf.h index b4559e51..657747bf 100644 --- a/IO/netcdf.h +++ b/IO/netcdf.h @@ -5,7 +5,7 @@ #include #include "common/Array.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" @@ -32,7 +32,7 @@ std::string VariableTypeName( VariableType type ); * @param mode Open the file for reading or writing * @param comm MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf) */ -int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm=MPI_COMM_NULL ); +int open( const std::string& filename, FileMode mode, MPI_Comm comm=MPI_COMM_NULL ); /*! diff --git a/IO/silo.cpp b/IO/silo.cpp index ddf3646a..eece8583 100644 --- a/IO/silo.cpp +++ b/IO/silo.cpp @@ -1,6 +1,6 @@ #include "IO/silo.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" diff --git a/IO/silo.h b/IO/silo.h index 339a5c34..4c7081e5 100644 --- a/IO/silo.h +++ b/IO/silo.h @@ -6,7 +6,7 @@ #include #include "common/Array.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" diff --git a/IO/silo.hpp b/IO/silo.hpp index 35852004..312f32d8 100644 --- a/IO/silo.hpp +++ b/IO/silo.hpp @@ -3,7 +3,7 @@ #include "IO/silo.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 3e3fb35e..faac6142 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -4,7 +4,7 @@ #include "common/Domain.h" #include "common/Communication.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" @@ -109,13 +109,13 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) // convert X for 2D manifold to 3D object Xi *= 0.5; - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); // Phase averages - Vi_global = Dm->Comm.sumReduce( Vi ); - Xi_global = Dm->Comm.sumReduce( Xi ); - Ai_global = Dm->Comm.sumReduce( Ai ); - Ji_global = Dm->Comm.sumReduce( Ji ); - Dm->Comm.barrier(); + MPI_Allreduce(&Vi,&Vi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Xi,&Xi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Ai,&Ai_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Ji,&Ji_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Barrier(Dm->Comm); PROFILE_STOP("ComputeScalar"); } @@ -168,7 +168,7 @@ int Minkowski::MeasureConnectedPathway(){ double vF=0.0; n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm); // int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm ) - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); for (int k=0; kComm.sumReduce( wb.V); - gnb.V = Dm->Comm.sumReduce( nb.V); - gwb.M = Dm->Comm.sumReduce( wb.M); - gnb.M = Dm->Comm.sumReduce( nb.M); - gwb.Px = Dm->Comm.sumReduce( wb.Px); - gwb.Py = Dm->Comm.sumReduce( wb.Py); - gwb.Pz = Dm->Comm.sumReduce( wb.Pz); - gnb.Px = Dm->Comm.sumReduce( nb.Px); - gnb.Py = Dm->Comm.sumReduce( nb.Py); - gnb.Pz = Dm->Comm.sumReduce( nb.Pz); + gwb.V=sumReduce( Dm->Comm, wb.V); + gnb.V=sumReduce( Dm->Comm, nb.V); + gwb.M=sumReduce( Dm->Comm, wb.M); + gnb.M=sumReduce( Dm->Comm, nb.M); + gwb.Px=sumReduce( Dm->Comm, wb.Px); + gwb.Py=sumReduce( Dm->Comm, wb.Py); + gwb.Pz=sumReduce( Dm->Comm, wb.Pz); + gnb.Px=sumReduce( Dm->Comm, nb.Px); + gnb.Py=sumReduce( Dm->Comm, nb.Py); + gnb.Pz=sumReduce( Dm->Comm, nb.Pz); - count_w = Dm->Comm.sumReduce( count_w); - count_n = Dm->Comm.sumReduce( count_n); + count_w=sumReduce( Dm->Comm, count_w); + count_n=sumReduce( Dm->Comm, count_n); if (count_w > 0.0) - gwb.p = Dm->Comm.sumReduce(wb.p) / count_w; + gwb.p=sumReduce( Dm->Comm, wb.p) / count_w; else gwb.p = 0.0; if (count_n > 0.0) - gnb.p = Dm->Comm.sumReduce( nb.p) / count_n; + gnb.p=sumReduce( Dm->Comm, nb.p) / count_n; else gnb.p = 0.0; @@ -444,14 +444,14 @@ void SubPhase::Full(){ nd.X -= nc.X; // compute global entities - gnc.V = Dm->Comm.sumReduce( nc.V ); - gnc.A = Dm->Comm.sumReduce( nc.A ); - gnc.H = Dm->Comm.sumReduce( nc.H ); - gnc.X = Dm->Comm.sumReduce( nc.X ); - gnd.V = Dm->Comm.sumReduce( nd.V ); - gnd.A = Dm->Comm.sumReduce( nd.A ); - gnd.H = Dm->Comm.sumReduce( nd.H ); - gnd.X = Dm->Comm.sumReduce( nd.X ); + gnc.V=sumReduce( Dm->Comm, nc.V); + gnc.A=sumReduce( Dm->Comm, nc.A); + gnc.H=sumReduce( Dm->Comm, nc.H); + gnc.X=sumReduce( Dm->Comm, nc.X); + gnd.V=sumReduce( Dm->Comm, nd.V); + gnd.A=sumReduce( Dm->Comm, nd.A); + gnd.H=sumReduce( Dm->Comm, nd.H); + gnd.X=sumReduce( Dm->Comm, nd.X); gnd.Nc = nd.Nc; // wetting for (k=0; kComm.sumReduce( wc.V ); - gwc.A = Dm->Comm.sumReduce( wc.A ); - gwc.H = Dm->Comm.sumReduce( wc.H ); - gwc.X = Dm->Comm.sumReduce( wc.X ); - gwd.V = Dm->Comm.sumReduce( wd.V ); - gwd.A = Dm->Comm.sumReduce( wd.A ); - gwd.H = Dm->Comm.sumReduce( wd.H ); - gwd.X = Dm->Comm.sumReduce( wd.X ); + gwc.V=sumReduce( Dm->Comm, wc.V); + gwc.A=sumReduce( Dm->Comm, wc.A); + gwc.H=sumReduce( Dm->Comm, wc.H); + gwc.X=sumReduce( Dm->Comm, wc.X); + gwd.V=sumReduce( Dm->Comm, wd.V); + gwd.A=sumReduce( Dm->Comm, wd.A); + gwd.H=sumReduce( Dm->Comm, wd.H); + gwd.X=sumReduce( Dm->Comm, wd.X); gwd.Nc = wd.Nc; /* Set up geometric analysis of interface region */ @@ -526,20 +526,20 @@ void SubPhase::Full(){ iwn.A = morph_i->A(); iwn.H = morph_i->H(); iwn.X = morph_i->X(); - giwn.V = Dm->Comm.sumReduce( iwn.V ); - giwn.A = Dm->Comm.sumReduce( iwn.A ); - giwn.H = Dm->Comm.sumReduce( iwn.H ); - giwn.X = Dm->Comm.sumReduce( iwn.X ); + giwn.V=sumReduce( Dm->Comm, iwn.V); + giwn.A=sumReduce( Dm->Comm, iwn.A); + giwn.H=sumReduce( Dm->Comm, iwn.H); + giwn.X=sumReduce( Dm->Comm, iwn.X); // measure only the connected part iwnc.Nc = morph_i->MeasureConnectedPathway(); iwnc.V = morph_i->V(); iwnc.A = morph_i->A(); iwnc.H = morph_i->H(); iwnc.X = morph_i->X(); - giwnc.V = Dm->Comm.sumReduce( iwnc.V ); - giwnc.A = Dm->Comm.sumReduce( iwnc.A ); - giwnc.H = Dm->Comm.sumReduce( iwnc.H ); - giwnc.X = Dm->Comm.sumReduce( iwnc.X ); + giwnc.V=sumReduce( Dm->Comm, iwnc.V); + giwnc.A=sumReduce( Dm->Comm, iwnc.A); + giwnc.H=sumReduce( Dm->Comm, iwnc.H); + giwnc.X=sumReduce( Dm->Comm, iwnc.X); giwnc.Nc = iwnc.Nc; double vol_nc_bulk = 0.0; @@ -630,46 +630,46 @@ void SubPhase::Full(){ } } - gnd.M = Dm->Comm.sumReduce( nd.M ); - gnd.Px = Dm->Comm.sumReduce( nd.Px ); - gnd.Py = Dm->Comm.sumReduce( nd.Py ); - gnd.Pz = Dm->Comm.sumReduce( nd.Pz ); - gnd.K = Dm->Comm.sumReduce( nd.K ); + gnd.M=sumReduce( Dm->Comm, nd.M); + gnd.Px=sumReduce( Dm->Comm, nd.Px); + gnd.Py=sumReduce( Dm->Comm, nd.Py); + gnd.Pz=sumReduce( Dm->Comm, nd.Pz); + gnd.K=sumReduce( Dm->Comm, nd.K); - gwd.M = Dm->Comm.sumReduce( wd.M ); - gwd.Px = Dm->Comm.sumReduce( wd.Px ); - gwd.Py = Dm->Comm.sumReduce( wd.Py ); - gwd.Pz = Dm->Comm.sumReduce( wd.Pz ); - gwd.K = Dm->Comm.sumReduce( wd.K ); + gwd.M=sumReduce( Dm->Comm, wd.M); + gwd.Px=sumReduce( Dm->Comm, wd.Px); + gwd.Py=sumReduce( Dm->Comm, wd.Py); + gwd.Pz=sumReduce( Dm->Comm, wd.Pz); + gwd.K=sumReduce( Dm->Comm, wd.K); - gnc.M = Dm->Comm.sumReduce( nc.M ); - gnc.Px = Dm->Comm.sumReduce( nc.Px ); - gnc.Py = Dm->Comm.sumReduce( nc.Py ); - gnc.Pz = Dm->Comm.sumReduce( nc.Pz ); - gnc.K = Dm->Comm.sumReduce( nc.K ); + gnc.M=sumReduce( Dm->Comm, nc.M); + gnc.Px=sumReduce( Dm->Comm, nc.Px); + gnc.Py=sumReduce( Dm->Comm, nc.Py); + gnc.Pz=sumReduce( Dm->Comm, nc.Pz); + gnc.K=sumReduce( Dm->Comm, nc.K); - gwc.M = Dm->Comm.sumReduce( wc.M ); - gwc.Px = Dm->Comm.sumReduce( wc.Px ); - gwc.Py = Dm->Comm.sumReduce( wc.Py ); - gwc.Pz = Dm->Comm.sumReduce( wc.Pz ); - gwc.K = Dm->Comm.sumReduce( wc.K ); + gwc.M=sumReduce( Dm->Comm, wc.M); + gwc.Px=sumReduce( Dm->Comm, wc.Px); + gwc.Py=sumReduce( Dm->Comm, wc.Py); + gwc.Pz=sumReduce( Dm->Comm, wc.Pz); + gwc.K=sumReduce( Dm->Comm, wc.K); - giwn.Mn = Dm->Comm.sumReduce( iwn.Mn ); - giwn.Pnx = Dm->Comm.sumReduce( iwn.Pnx ); - giwn.Pny = Dm->Comm.sumReduce( iwn.Pny ); - giwn.Pnz = Dm->Comm.sumReduce( iwn.Pnz ); - giwn.Kn = Dm->Comm.sumReduce( iwn.Kn ); - giwn.Mw = Dm->Comm.sumReduce( iwn.Mw ); - giwn.Pwx = Dm->Comm.sumReduce( iwn.Pwx ); - giwn.Pwy = Dm->Comm.sumReduce( iwn.Pwy ); - giwn.Pwz = Dm->Comm.sumReduce( iwn.Pwz ); - giwn.Kw = Dm->Comm.sumReduce( iwn.Kw ); + giwn.Mn=sumReduce( Dm->Comm, iwn.Mn); + giwn.Pnx=sumReduce( Dm->Comm, iwn.Pnx); + giwn.Pny=sumReduce( Dm->Comm, iwn.Pny); + giwn.Pnz=sumReduce( Dm->Comm, iwn.Pnz); + giwn.Kn=sumReduce( Dm->Comm, iwn.Kn); + giwn.Mw=sumReduce( Dm->Comm, iwn.Mw); + giwn.Pwx=sumReduce( Dm->Comm, iwn.Pwx); + giwn.Pwy=sumReduce( Dm->Comm, iwn.Pwy); + giwn.Pwz=sumReduce( Dm->Comm, iwn.Pwz); + giwn.Kw=sumReduce( Dm->Comm, iwn.Kw); // pressure averaging - gnc.p = Dm->Comm.sumReduce( nc.p ); - gnd.p = Dm->Comm.sumReduce( nd.p ); - gwc.p = Dm->Comm.sumReduce( wc.p ); - gwd.p = Dm->Comm.sumReduce( wd.p ); + gnc.p=sumReduce( Dm->Comm, nc.p); + gnd.p=sumReduce( Dm->Comm, nd.p); + gwc.p=sumReduce( Dm->Comm, wc.p); + gwd.p=sumReduce( Dm->Comm, wd.p); if (vol_wc_bulk > 0.0) wc.p = wc.p /vol_wc_bulk; @@ -680,10 +680,10 @@ void SubPhase::Full(){ if (vol_nd_bulk > 0.0) nd.p = nd.p /vol_nd_bulk; - vol_wc_bulk = Dm->Comm.sumReduce( vol_wc_bulk ); - vol_wd_bulk = Dm->Comm.sumReduce( vol_wd_bulk ); - vol_nc_bulk = Dm->Comm.sumReduce( vol_nc_bulk ); - vol_nd_bulk = Dm->Comm.sumReduce( vol_nd_bulk ); + vol_wc_bulk=sumReduce( Dm->Comm, vol_wc_bulk); + vol_wd_bulk=sumReduce( Dm->Comm, vol_wd_bulk); + vol_nc_bulk=sumReduce( Dm->Comm, vol_nc_bulk); + vol_nd_bulk=sumReduce( Dm->Comm, vol_nd_bulk); if (vol_wc_bulk > 0.0) gwc.p = gwc.p /vol_wc_bulk; @@ -719,7 +719,7 @@ void SubPhase::AggregateLabels( const std::string& filename ) } } } - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); Dm->AggregateLabels( filename ); diff --git a/analysis/SubPhase.h b/analysis/SubPhase.h index 691c654f..71b87ef0 100644 --- a/analysis/SubPhase.h +++ b/analysis/SubPhase.h @@ -12,7 +12,7 @@ #include "analysis/distance.h" #include "analysis/Minkowski.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index 812490e7..9b2e5fd8 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -5,7 +5,7 @@ #include "common/Domain.h" #include "common/Communication.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" @@ -882,7 +882,7 @@ void TwoPhase::ComponentAverages() } } - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); if (Dm->rank()==0){ printf("Component averages computed locally -- reducing result... \n"); } @@ -895,8 +895,8 @@ void TwoPhase::ComponentAverages() for (int idx=0; idxComm.barrier(); - Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP); + MPI_Barrier(Dm->Comm); + MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm->Comm); // MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); if (Dm->rank()==0){ @@ -993,9 +993,9 @@ void TwoPhase::ComponentAverages() // reduce the wetting phase averages for (int b=0; bComm.barrier(); + MPI_Barrier(Dm->Comm); // MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); - Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT); + MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); for (int idx=0; idxComm.barrier(); - nwp_volume_global = Dm->Comm.sumReduce( nwp_volume ); - wp_volume_global = Dm->Comm.sumReduce( wp_volume ); - awn_global = Dm->Comm.sumReduce( awn ); - ans_global = Dm->Comm.sumReduce( ans ); - aws_global = Dm->Comm.sumReduce( aws ); - lwns_global = Dm->Comm.sumReduce( lwns ); - As_global = Dm->Comm.sumReduce( As ); - Jwn_global = Dm->Comm.sumReduce( Jwn ); - Kwn_global = Dm->Comm.sumReduce( Kwn ); - KGwns_global = Dm->Comm.sumReduce( KGwns ); - KNwns_global = Dm->Comm.sumReduce( KNwns ); - efawns_global = Dm->Comm.sumReduce( efawns ); - wwndnw_global = Dm->Comm.sumReduce( wwndnw ); - wwnsdnwn_global = Dm->Comm.sumReduce( wwnsdnwn ); - Jwnwwndnw_global = Dm->Comm.sumReduce( Jwnwwndnw ); + MPI_Barrier(Dm->Comm); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); // Phase averages - vol_w_global = Dm->Comm.sumReduce( vol_w ); - vol_n_global = Dm->Comm.sumReduce( vol_n ); - paw_global = Dm->Comm.sumReduce( paw ); - pan_global = Dm->Comm.sumReduce( pan ); - for (int idx=0; idx<3; idx++) - vaw_global(idx) = Dm->Comm.sumReduce( vaw(idx) ); - for (int idx=0; idx<3; idx++) - van_global(idx) = Dm->Comm.sumReduce( van(idx)); - for (int idx=0; idx<3; idx++) - vawn_global(idx) = Dm->Comm.sumReduce( vawn(idx) ); - for (int idx=0; idx<3; idx++) - vawns_global(idx) = Dm->Comm.sumReduce( vawns(idx) ); - for (int idx=0; idx<6; idx++){ - Gwn_global(idx) = Dm->Comm.sumReduce( Gwn(idx) ); - Gns_global(idx) = Dm->Comm.sumReduce( Gns(idx) ); - Gws_global(idx) = Dm->Comm.sumReduce( Gws(idx) ); - } - trawn_global = Dm->Comm.sumReduce( trawn ); - trJwn_global = Dm->Comm.sumReduce( trJwn ); - trRwn_global = Dm->Comm.sumReduce( trRwn ); - euler_global = Dm->Comm.sumReduce( euler ); - An_global = Dm->Comm.sumReduce( An ); - Jn_global = Dm->Comm.sumReduce( Jn ); - Kn_global = Dm->Comm.sumReduce( Kn ); - Dm->Comm.barrier(); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + + MPI_Barrier(Dm->Comm); // Normalize the phase averages // (density of both components = 1.0) diff --git a/analysis/TwoPhase.h b/analysis/TwoPhase.h index 4d500a89..fddd04e8 100644 --- a/analysis/TwoPhase.h +++ b/analysis/TwoPhase.h @@ -12,7 +12,7 @@ #include "common/Domain.h" #include "common/Communication.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" diff --git a/analysis/analysis.cpp b/analysis/analysis.cpp index 4298750e..7587f3c5 100644 --- a/analysis/analysis.cpp +++ b/analysis/analysis.cpp @@ -188,7 +188,7 @@ int ComputeLocalPhaseComponent(const IntArray &PhaseID, int &VALUE, BlobIDArray /****************************************************************** * Reorder the global blob ids * ******************************************************************/ -static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, const Utilities::MPI& comm ) +static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, MPI_Comm comm ) { if ( N_blobs==0 ) return 0; @@ -212,7 +212,7 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int } } ASSERT(max_id > map1(N_blobs); int N_blobs2 = 0; for (int i=0; i& N_recv, int64_t *send_buf, std::vector& recv_buf, std::map& remote_map, - const Utilities::MPI& comm ) + MPI_Comm comm ) { std::vector send_req(neighbors.size()); std::vector recv_req(neighbors.size()); - auto it = map.begin(); + std::vector status(neighbors.size()); + std::map::const_iterator it = map.begin(); ASSERT(N_send==(int)map.size()); for (size_t i=0; ifirst; send_buf[2*i+1] = it->second.new_id; } for (size_t i=0; ifirst] = it->second.new_id; } for (size_t i=0; i& remote_map, @@ -303,18 +304,18 @@ static bool updateLocalIds( const std::map& remote_map, return changed; } static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, - int nblobs, BlobIDArray& IDs, const Utilities::MPI& comm ) + int nblobs, BlobIDArray& IDs, MPI_Comm comm ) { PROFILE_START("LocalToGlobalIDs",1); const int rank = rank_info.rank[1][1][1]; - int nprocs = comm.getSize(); + int nprocs = comm_size(comm); const int ngx = (IDs.size(0)-nx)/2; const int ngy = (IDs.size(1)-ny)/2; const int ngz = (IDs.size(2)-nz)/2; // Get the number of blobs for each rank std::vector N_blobs(nprocs,0); PROFILE_START("LocalToGlobalIDs-Allgather",1); - comm.allGather(nblobs,getPtr(N_blobs)); + MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,comm); PROFILE_STOP("LocalToGlobalIDs-Allgather",1); int64_t N_blobs_tot = 0; int offset = 0; @@ -362,12 +363,13 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_ std::vector N_recv(neighbors.size(),0); std::vector send_req(neighbors.size()); std::vector recv_req(neighbors.size()); + std::vector status(neighbors.size()); for (size_t i=0; i recv_buf(neighbors.size()); @@ -396,7 +398,8 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_ bool changed = updateLocalIds( remote_map, map ); // Check if we are finished int test = changed ? 1:0; - int result = comm.sumReduce( test ); + int result = 0; + MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,comm); if ( result==0 ) break; } @@ -432,7 +435,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_ } int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS, - BlobIDArray& GlobalBlobID, const Utilities::MPI& comm ) + BlobIDArray& GlobalBlobID, MPI_Comm comm ) { PROFILE_START("ComputeGlobalBlobIDs"); // First compute the local ids @@ -443,7 +446,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf return nglobal; } int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info, - const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm ) + const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm ) { PROFILE_START("ComputeGlobalPhaseComponent"); // First compute the local ids @@ -459,27 +462,37 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r * Compute the mapping of blob ids between timesteps * ******************************************************************/ typedef std::map > map_type; +template inline MPI_Datatype getMPIType(); +template<> inline MPI_Datatype getMPIType() { return MPI_INT; } +template<> inline MPI_Datatype getMPIType() { + if ( sizeof(int64_t)==sizeof(long int) ) + return MPI_LONG; + else if ( sizeof(int64_t)==sizeof(double) ) + return MPI_DOUBLE; +} template -void gatherSet( std::set& set, const Utilities::MPI& comm ) +void gatherSet( std::set& set, MPI_Comm comm ) { - int nprocs = comm.getSize(); + int nprocs = comm_size(comm); + MPI_Datatype type = getMPIType(); std::vector send_data(set.begin(),set.end()); int send_count = send_data.size(); std::vector recv_count(nprocs,0), recv_disp(nprocs,0); - comm.allGather( send_count, getPtr(recv_count) ); + MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm); for (int i=1; i recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]); - comm.allGather( getPtr(send_data), send_count, getPtr(recv_data), - getPtr(recv_count), getPtr(recv_disp), true ); + MPI_Allgatherv(getPtr(send_data),send_count,type, + getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm); for (size_t i=0; i(); std::vector send_data; - for (auto it=src_map.begin(); it!=src_map.end(); ++it) { + for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) { int id = it->first; const std::map& src_ids = it->second; send_data.push_back(id); @@ -492,21 +505,21 @@ void gatherSrcIDMap( map_type& src_map, const Utilities::MPI& comm ) } int send_count = send_data.size(); std::vector recv_count(nprocs,0), recv_disp(nprocs,0); - comm.allGather(send_count,getPtr(recv_count)); + MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm); for (int i=1; i recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]); - comm.allGather(getPtr(send_data),send_count, - getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),true); + MPI_Allgatherv(getPtr(send_data),send_count,type, + getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm); size_t i=0; src_map.clear(); while ( i < recv_data.size() ) { BlobIDType id = recv_data[i]; size_t count = recv_data[i+1]; i += 2; - auto& src_ids = src_map[id]; + std::map& src_ids = src_map[id]; for (size_t j=0; j::iterator it = src_ids.find(recv_data[i]); if ( it == src_ids.end() ) src_ids.insert(std::pair(recv_data[i],recv_data[i+1])); else @@ -525,7 +538,7 @@ void addSrcDstIDs( BlobIDType src_id, map_type& src_map, map_type& dst_map, } } ID_map_struct computeIDMap( int nx, int ny, int nz, - const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm ) + const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm ) { ASSERT(ID1.size()==ID2.size()); PROFILE_START("computeIDMap"); @@ -767,7 +780,7 @@ void renumberIDs( const std::vector& new_ids, BlobIDArray& IDs ) ******************************************************************/ void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename ) { - int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank(); + int rank = MPI_WORLD_RANK(); if ( rank!=0 ) return; bool empty = map.created.empty() && map.destroyed.empty() && diff --git a/analysis/analysis.h b/analysis/analysis.h index ec377995..2ce531b1 100644 --- a/analysis/analysis.h +++ b/analysis/analysis.h @@ -58,7 +58,7 @@ int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &C */ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS, - BlobIDArray& GlobalBlobID, const Utilities::MPI& comm ); + BlobIDArray& GlobalBlobID, MPI_Comm comm ); /*! @@ -75,7 +75,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf * @return Return the number of components in the specified phase */ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info, - const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm ); + const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm ); /*! @@ -87,7 +87,7 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r * @param[in] nz Number of elements in the z-direction * @param[in/out] ID The ids of the blobs */ -void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm ); +void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm ); typedef std::pair > BlobIDSplitStruct; @@ -120,7 +120,7 @@ struct ID_map_struct { * @param[in] ID1 The blob ids at the first timestep * @param[in] ID2 The blob ids at the second timestep */ -ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm ); +ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm ); /*! diff --git a/analysis/distance.cpp b/analysis/distance.cpp index 9c605e1e..e297b435 100644 --- a/analysis/distance.cpp +++ b/analysis/distance.cpp @@ -176,7 +176,7 @@ void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, // Update distance double err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] ); // Check if we are finished - err = Dm.Comm.maxReduce( err ); + err = maxReduce( Dm.Comm, err ); if ( err < tol ) break; } diff --git a/analysis/morphology.cpp b/analysis/morphology.cpp index ab4312f8..72a17892 100644 --- a/analysis/morphology.cpp +++ b/analysis/morphology.cpp @@ -58,11 +58,11 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr } } } - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); // total Global is the number of nodes in the pore-space - totalGlobal = Dm->Comm.sumReduce( count ); - maxdistGlobal = Dm->Comm.sumReduce( maxdist ); + MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm); double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2); double volume_fraction=totalGlobal/volume; if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction); @@ -133,6 +133,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr double deltaR=0.05; // amount to change the radius in voxel units double Rcrit_old=0.0; + double GlobalNumber = 1.f; int imin,jmin,kmin,imax,jmax,kmax; if (ErodeLabel == 1){ @@ -202,41 +203,41 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, - recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, - recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, - recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, - recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, - recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, - recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, - recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, - recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, - recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, - recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, - recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, - recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, - recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, - recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, - recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, - recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, - recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, - recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -258,7 +259,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id); //...................................................................................... - //double GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); + MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); count = 0.f; for (int k=1; k } } } - countGlobal = Dm->Comm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); void_fraction_new = countGlobal/totalGlobal; void_fraction_diff_new = abs(void_fraction_new-VoidFraction); /* if (rank==0){ @@ -359,11 +360,11 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrComm.barrier(); + MPI_Barrier(Dm->Comm); // total Global is the number of nodes in the pore-space - totalGlobal = Dm->Comm.sumReduce( count ); - maxdistGlobal = Dm->Comm.sumReduce( maxdist ); + MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm); double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2); double volume_fraction=totalGlobal/volume; if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction); @@ -433,6 +434,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrComm.barrier(); + MPI_Barrier(Dm->Comm); FILE *DRAIN = fopen("morphdrain.csv","w"); @@ -507,41 +509,41 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrsendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id); //...................................................................................... MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag, - recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag, - recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag, - recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag, - recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag, - recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag, - recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag, - recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag, - recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag, - recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag, - recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag, - recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag, - recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag, - recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag, - recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag, - recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag, - recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag, - recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag, - recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id); UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id); @@ -562,7 +564,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptrrecvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id); UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id); //...................................................................................... - // double GlobalNumber = Dm->Comm.sumReduce( LocalNumber ); + MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); for (int k=0; krank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); for (int k=0; kComm.sumReduce( count ); + MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); void_fraction_new = countGlobal/totalGlobal; void_fraction_diff_new = abs(void_fraction_new-VoidFraction); if (rank==0){ @@ -700,7 +702,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, } } } - double count_original = Dm->Comm.sumReduce( count); + double count_original=sumReduce( Dm->Comm, count); // Estimate morph_delta double morph_delta = 0.0; @@ -730,8 +732,8 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, } } } - count = Dm->Comm.sumReduce( count ); - MAX_DISPLACEMENT = Dm->Comm.maxReduce( MAX_DISPLACEMENT ); + count=sumReduce( Dm->Comm, count); + MAX_DISPLACEMENT = maxReduce( Dm->Comm, MAX_DISPLACEMENT); GrowthEstimate = count - count_original; ERROR = fabs((GrowthEstimate-TargetGrowth) /TargetGrowth); @@ -774,7 +776,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, } } } - count = Dm->Comm.sumReduce( count ); + count=sumReduce( Dm->Comm, count); return count; } diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 89451c7b..6c76f58b 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -3,7 +3,7 @@ #include "analysis/analysis.h" #include "common/Array.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/ScaLBL.h" #include "models/ColorModel.h" @@ -462,7 +462,7 @@ private: /****************************************************************** * MPI comm wrapper for use with analysis * ******************************************************************/ -runAnalysis::commWrapper::commWrapper( int tag_, const Utilities::MPI& comm_, runAnalysis* analysis_ ): +runAnalysis::commWrapper::commWrapper( int tag_, MPI_Comm comm_, runAnalysis* analysis_ ): comm(comm_), tag(tag_), analysis(analysis_) @@ -479,7 +479,7 @@ runAnalysis::commWrapper::~commWrapper() { if ( tag == -1 ) return; - comm.barrier(); + MPI_Barrier( comm ); analysis->d_comm_used[tag] = false; } runAnalysis::commWrapper runAnalysis::getComm( ) @@ -496,10 +496,10 @@ runAnalysis::commWrapper runAnalysis::getComm( ) if ( tag == -1 ) ERROR("Unable to get comm"); } - tag = d_comm.bcast( tag, 0 ); + MPI_Bcast( &tag, 1, MPI_INT, 0, d_comm ); d_comm_used[tag] = true; - if ( d_comms[tag].isNull() ) - d_comms[tag] = d_comm.dup(); + if ( d_comms[tag] == MPI_COMM_NULL ) + MPI_Comm_dup( MPI_COMM_WORLD, &d_comms[tag] ); return commWrapper(tag,d_comms[tag],this); } @@ -507,20 +507,14 @@ runAnalysis::commWrapper runAnalysis::getComm( ) /****************************************************************** * Constructor/Destructors * ******************************************************************/ -runAnalysis::runAnalysis( std::shared_ptr input_db, - const RankInfoStruct& rank_info, - std::shared_ptr ScaLBL_Comm, - std::shared_ptr Dm, - int Np, - bool Regular, - IntArray Map ): - d_Np( Np ), - d_regular ( Regular), - d_rank_info( rank_info ), - d_Map( Map ), - d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1), - d_comm( Utilities::MPI( MPI_COMM_WORLD ).dup() ), - d_ScaLBL_Comm( ScaLBL_Comm) +runAnalysis::runAnalysis(std::shared_ptr input_db, const RankInfoStruct& rank_info, std::shared_ptr ScaLBL_Comm, std::shared_ptr Dm, + int Np, bool Regular, IntArray Map ): + d_Np( Np ), + d_regular ( Regular), + d_rank_info( rank_info ), + d_Map( Map ), + d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1), + d_ScaLBL_Comm( ScaLBL_Comm) { auto db = input_db->getDatabase( "Analysis" ); @@ -558,7 +552,7 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, d_restartFile = restart_file + "." + rankString; - d_rank = d_comm.getRank(); + d_rank = MPI_WORLD_RANK(); writeIDMap(ID_map_struct(),0,id_map_filename); // Initialize IO for silo IO::initialize("","silo","false"); @@ -627,8 +621,11 @@ runAnalysis::runAnalysis( std::shared_ptr input_db, // Initialize the comms - for (int i=0; i<1024; i++) + MPI_Comm_dup(MPI_COMM_WORLD,&d_comm); + for (int i=0; i<1024; i++) { + d_comms[i] = MPI_COMM_NULL; d_comm_used[i] = false; + } // Initialize the threads int N_threads = db->getWithDefault( "N_threads", 4 ); auto method = db->getWithDefault( "load_balance", "default" ); @@ -638,6 +635,12 @@ runAnalysis::~runAnalysis( ) { // Finish processing analysis finish(); + // Clear internal data + MPI_Comm_free( &d_comm ); + for (int i=0; i<1024; i++) { + if ( d_comms[i] != MPI_COMM_NULL ) + MPI_Comm_free(&d_comms[i]); + } } void runAnalysis::finish( ) { @@ -651,7 +654,7 @@ void runAnalysis::finish( ) d_wait_subphase.reset(); d_wait_restart.reset(); // Syncronize - d_comm.barrier(); + MPI_Barrier( d_comm ); PROFILE_STOP("finish"); } diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index 3c5bc7f0..0bf2f676 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -68,10 +68,10 @@ public: class commWrapper { public: - Utilities::MPI comm; + MPI_Comm comm; int tag; runAnalysis *analysis; - commWrapper( int tag, const Utilities::MPI& comm, runAnalysis *analysis ); + commWrapper( int tag, MPI_Comm comm, runAnalysis *analysis ); commWrapper( ) = delete; commWrapper( const commWrapper &rhs ) = delete; commWrapper& operator=( const commWrapper &rhs ) = delete; @@ -100,8 +100,8 @@ private: std::vector d_meshData; fillHalo d_fillData; std::string d_restartFile; - Utilities::MPI d_comm; - Utilities::MPI d_comms[1024]; + MPI_Comm d_comm; + MPI_Comm d_comms[1024]; volatile bool d_comm_used[1024]; std::shared_ptr d_ScaLBL_Comm; diff --git a/analysis/uCT.cpp b/analysis/uCT.cpp index 28d677c1..912f8e85 100644 --- a/analysis/uCT.cpp +++ b/analysis/uCT.cpp @@ -228,7 +228,8 @@ void filter_final( Array& ID, Array& Dist, Array& Mean, Array& Dist1, Array& Dist2 ) { PROFILE_SCOPED(timer,"filter_final"); - int rank = Dm.Comm.getRank(); + int rank; + MPI_Comm_rank(Dm.Comm,&rank); int Nx = Dm.Nx-2; int Ny = Dm.Ny-2; int Nz = Dm.Nz-2; @@ -241,7 +242,7 @@ void filter_final( Array& ID, Array& Dist, float tmp = 0; for (size_t i=0; i(Dist0.length()) ); + tmp = sqrt( sumReduce(Dm.Comm,tmp) / sumReduce(Dm.Comm,(float)Dist0.length()) ); const float dx1 = 0.3*tmp; const float dx2 = 1.05*dx1; if (rank==0) @@ -284,7 +285,7 @@ void filter_final( Array& ID, Array& Dist, Phase.fill(1); ComputeGlobalBlobIDs( Nx, Ny, Nz, Dm.rank_info, Phase, SignDist, 0, 0, GlobalBlobID, Dm.Comm ); fillInt.fill(GlobalBlobID); - int N_blobs = Dm.Comm.maxReduce(GlobalBlobID.max()+1); + int N_blobs = maxReduce(Dm.Comm,GlobalBlobID.max()+1); std::vector mean(N_blobs,0); std::vector count(N_blobs,0); for (int k=1; k<=Nz; k++) { @@ -320,8 +321,8 @@ void filter_final( Array& ID, Array& Dist, } } } - mean = Dm.Comm.sumReduce(mean); - count = Dm.Comm.sumReduce(count); + mean = sumReduce(Dm.Comm,mean); + count = sumReduce(Dm.Comm,count); for (size_t i=0; i -o ") -set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} -o -shared" ) -set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_PATH} -o ") - -############################################################################### -# FIND: HIP and associated helper binaries -############################################################################### -# HIP is supported on Linux only -if(UNIX AND NOT APPLE AND NOT CYGWIN) - # Search for HIP installation - if(NOT HIP_ROOT_DIR) - # Search in user specified path first - find_path( - HIP_ROOT_DIR - NAMES hipconfig - PATHS - ENV ROCM_PATH - ENV HIP_PATH - PATH_SUFFIXES bin - DOC "HIP installed location" - NO_DEFAULT_PATH - ) - # Now search in default path - find_path( - HIP_ROOT_DIR - NAMES hipconfig - PATHS - /opt/rocm - /opt/rocm/hip - PATH_SUFFIXES bin - DOC "HIP installed location" - ) - - # Check if we found HIP installation - if(HIP_ROOT_DIR) - # If so, fix the path - string(REGEX REPLACE "[/\\\\]?bin[64]*[/\\\\]?$" "" HIP_ROOT_DIR ${HIP_ROOT_DIR}) - # And push it back to the cache - set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE) - endif() - if(NOT EXISTS ${HIP_ROOT_DIR}) - if(HIP_FIND_REQUIRED) - message(FATAL_ERROR "Specify HIP_ROOT_DIR") - elseif(NOT HIP_FIND_QUIETLY) - message("HIP_ROOT_DIR not found or specified") - endif() - endif() - endif() - - # Find HIPCC executable - find_program( - HIP_HIPCC_EXECUTABLE - NAMES hipcc - PATHS - "${HIP_ROOT_DIR}" - ENV ROCM_PATH - ENV HIP_PATH - /opt/rocm - /opt/rocm/hip - PATH_SUFFIXES bin - NO_DEFAULT_PATH - ) - if(NOT HIP_HIPCC_EXECUTABLE) - # Now search in default paths - find_program(HIP_HIPCC_EXECUTABLE hipcc) - endif() - mark_as_advanced(HIP_HIPCC_EXECUTABLE) - - # Find HIPCONFIG executable - find_program( - HIP_HIPCONFIG_EXECUTABLE - NAMES hipconfig - PATHS - "${HIP_ROOT_DIR}" - ENV ROCM_PATH - ENV HIP_PATH - /opt/rocm - /opt/rocm/hip - PATH_SUFFIXES bin - NO_DEFAULT_PATH - ) - if(NOT HIP_HIPCONFIG_EXECUTABLE) - # Now search in default paths - find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig) - endif() - mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE) - - # Find HIPCC_CMAKE_LINKER_HELPER executable - find_program( - HIP_HIPCC_CMAKE_LINKER_HELPER - NAMES hipcc_cmake_linker_helper - PATHS - "${HIP_ROOT_DIR}" - ENV ROCM_PATH - ENV HIP_PATH - /opt/rocm - /opt/rocm/hip - PATH_SUFFIXES bin - NO_DEFAULT_PATH - ) - if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER) - # Now search in default paths - find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper) - endif() - mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER) - - if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION) - # Compute the version - execute_process( - COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version - OUTPUT_VARIABLE _hip_version - ERROR_VARIABLE _hip_error - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE - ) - if(NOT _hip_error) - set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc") - else() - set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()") - endif() - mark_as_advanced(HIP_VERSION) - endif() - if(HIP_VERSION) - string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}") - list(GET _hip_version_list 0 HIP_VERSION_MAJOR) - list(GET _hip_version_list 1 HIP_VERSION_MINOR) - list(GET _hip_version_list 2 HIP_VERSION_PATCH) - set(HIP_VERSION_STRING "${HIP_VERSION}") - endif() - - if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM) - # Compute the platform - execute_process( - COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform - OUTPUT_VARIABLE _hip_platform - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig") - mark_as_advanced(HIP_PLATFORM) - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - HIP - REQUIRED_VARS - HIP_ROOT_DIR - HIP_HIPCC_EXECUTABLE - HIP_HIPCONFIG_EXECUTABLE - HIP_PLATFORM - VERSION_VAR HIP_VERSION - ) - -############################################################################### -# MACRO: Locate helper files -############################################################################### -macro(HIP_FIND_HELPER_FILE _name _extension) - set(_hip_full_name "${_name}.${_extension}") - get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) - set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}") - if(NOT EXISTS "${HIP_${_name}}") - set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP") - if(HIP_FIND_REQUIRED) - message(FATAL_ERROR "${error_message}") - else() - if(NOT HIP_FIND_QUIETLY) - message(STATUS "${error_message}") - endif() - endif() - endif() - # Set this variable as internal, so the user isn't bugged with it. - set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE) -endmacro() - -############################################################################### -hip_find_helper_file(run_make2cmake cmake) -hip_find_helper_file(run_hipcc cmake) -############################################################################### - -############################################################################### -# MACRO: Reset compiler flags -############################################################################### -macro(HIP_RESET_FLAGS) - unset(HIP_HIPCC_FLAGS) - unset(HIP_HCC_FLAGS) - unset(HIP_NVCC_FLAGS) - foreach(config ${_hip_configuration_types}) - string(TOUPPER ${config} config_upper) - unset(HIP_HIPCC_FLAGS_${config_upper}) - unset(HIP_HCC_FLAGS_${config_upper}) - unset(HIP_NVCC_FLAGS_${config_upper}) - endforeach() -endmacro() - -############################################################################### -# MACRO: Separate the options from the sources -############################################################################### -macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _nvcc_options) - set(${_sources}) - set(${_cmake_options}) - set(${_hipcc_options}) - set(${_hcc_options}) - set(${_nvcc_options}) - set(_hipcc_found_options FALSE) - set(_hcc_found_options FALSE) - set(_nvcc_found_options FALSE) - foreach(arg ${ARGN}) - if("x${arg}" STREQUAL "xHIPCC_OPTIONS") - set(_hipcc_found_options TRUE) - set(_hcc_found_options FALSE) - set(_nvcc_found_options FALSE) - elseif("x${arg}" STREQUAL "xHCC_OPTIONS") - set(_hipcc_found_options FALSE) - set(_hcc_found_options TRUE) - set(_nvcc_found_options FALSE) - elseif("x${arg}" STREQUAL "xNVCC_OPTIONS") - set(_hipcc_found_options FALSE) - set(_hcc_found_options FALSE) - set(_nvcc_found_options TRUE) - elseif( - "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR - "x${arg}" STREQUAL "xSTATIC" OR - "x${arg}" STREQUAL "xSHARED" OR - "x${arg}" STREQUAL "xMODULE" - ) - list(APPEND ${_cmake_options} ${arg}) - else() - if(_hipcc_found_options) - list(APPEND ${_hipcc_options} ${arg}) - elseif(_hcc_found_options) - list(APPEND ${_hcc_options} ${arg}) - elseif(_nvcc_found_options) - list(APPEND ${_nvcc_options} ${arg}) - else() - # Assume this is a file - list(APPEND ${_sources} ${arg}) - endif() - endif() - endforeach() -endmacro() - -############################################################################### -# MACRO: Add include directories to pass to the hipcc command -############################################################################### -set(HIP_HIPCC_INCLUDE_ARGS_USER "") -macro(HIP_INCLUDE_DIRECTORIES) - foreach(dir ${ARGN}) - list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$:-I${dir}>) - endforeach() -endmacro() - -############################################################################### -# FUNCTION: Helper to avoid clashes of files with the same basename but different paths -############################################################################### -function(HIP_COMPUTE_BUILD_PATH path build_path) - # Convert to cmake style paths - file(TO_CMAKE_PATH "${path}" bpath) - if(IS_ABSOLUTE "${bpath}") - string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos) - if(_binary_dir_pos EQUAL 0) - file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}") - else() - file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}") - endif() - endif() - - # Remove leading / - string(REGEX REPLACE "^[/]+" "" bpath "${bpath}") - # Avoid absolute paths by removing ':' - string(REPLACE ":" "_" bpath "${bpath}") - # Avoid relative paths that go up the tree - string(REPLACE "../" "__/" bpath "${bpath}") - # Avoid spaces - string(REPLACE " " "_" bpath "${bpath}") - # Strip off the filename - get_filename_component(bpath "${bpath}" PATH) - - set(${build_path} "${bpath}" PARENT_SCOPE) -endfunction() - -############################################################################### -# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix -############################################################################### -macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix) - set(_hip_found_config) - foreach(arg ${ARGN}) - # Determine if we are dealing with a per-configuration flag - foreach(config ${_hip_configuration_types}) - string(TOUPPER ${config} config_upper) - if(arg STREQUAL "${config_upper}") - set(_hip_found_config _${arg}) - # Clear arg to prevent it from being processed anymore - set(arg) - endif() - endforeach() - if(arg) - list(APPEND ${_option_prefix}${_hip_found_config} "${arg}") - endif() - endforeach() -endmacro() - -############################################################################### -# MACRO: Try and include dependency file if it exists -############################################################################### -macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file) - set(HIP_HIPCC_DEPEND) - set(HIP_HIPCC_DEPEND_REGENERATE FALSE) - - # Create the dependency file if it doesn't exist - if(NOT EXISTS ${dependency_file}) - file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n") - endif() - # Include the dependency file - include(${dependency_file}) - - # Verify the existence of all the included files - if(HIP_HIPCC_DEPEND) - foreach(f ${HIP_HIPCC_DEPEND}) - if(NOT EXISTS ${f}) - # If they aren't there, regenerate the file again - set(HIP_HIPCC_DEPEND_REGENERATE TRUE) - endif() - endforeach() - else() - # No dependencies, so regenerate the file - set(HIP_HIPCC_DEPEND_REGENERATE TRUE) - endif() - - # Regenerate the dependency file if needed - if(HIP_HIPCC_DEPEND_REGENERATE) - set(HIP_HIPCC_DEPEND ${dependency_file}) - file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n") - endif() -endmacro() - -############################################################################### -# MACRO: Prepare cmake commands for the target -############################################################################### -macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files) - set(_hip_flags "") - string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration) - if(HIP_HOST_COMPILATION_CPP) - set(HIP_C_OR_CXX CXX) - else() - set(HIP_C_OR_CXX C) - endif() - set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION}) - - # Initialize list of includes with those specified by the user. Append with - # ones specified to cmake directly. - set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER}) - - # Add the include directories - set(include_directories_generator "$") - list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$:-I$>") - - get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES) - list(REMOVE_DUPLICATES _hip_include_directories) - if(_hip_include_directories) - foreach(dir ${_hip_include_directories}) - list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$:-I${dir}>) - endforeach() - endif() - - HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN}) - HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options}) - HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options}) - HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options}) - - # Add the compile definitions - set(compile_definition_generator "$") - list(APPEND HIP_HIPCC_FLAGS "$<$:-D$>") - - # Check if we are building shared library. - set(_hip_build_shared_libs FALSE) - list(FIND _hip_cmake_options SHARED _hip_found_SHARED) - list(FIND _hip_cmake_options MODULE _hip_found_MODULE) - if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1) - set(_hip_build_shared_libs TRUE) - endif() - list(FIND _hip_cmake_options STATIC _hip_found_STATIC) - if(_hip_found_STATIC GREATER -1) - set(_hip_build_shared_libs FALSE) - endif() - - # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS - if(_hip_build_shared_libs) - list(APPEND HIP_HCC_FLAGS "-fPIC") - list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'") - endif() - - # Set host compiler - set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}") - - # Set compiler flags - set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})") - set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})") - set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})") - set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})") - foreach(config ${_hip_configuration_types}) - string(TOUPPER ${config} config_upper) - set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})") - set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})") - set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})") - set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})") - endforeach() - - # Reset the output variable - set(_hip_generated_files "") - set(_hip_source_files "") - - # Iterate over all arguments and create custom commands for all source files - foreach(file ${ARGN}) - # Ignore any file marked as a HEADER_FILE_ONLY - get_source_file_property(_is_header ${file} HEADER_FILE_ONLY) - # Allow per source file overrides of the format. Also allows compiling non .cu files. - get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT) - if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header) - set(host_flag FALSE) - else() - set(host_flag TRUE) - endif() - - if(NOT host_flag) - # Determine output directory - HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path) - set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}") - - get_filename_component(basename ${file} NAME) - set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}") - set(generated_file_basename "${_target}_generated_${basename}${generated_extension}") - - # Set file names - set(generated_file "${generated_file_path}/${generated_file_basename}") - set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend") - set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen") - set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake") - - # Set properties for object files - set_source_files_properties("${generated_file}" - PROPERTIES - EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked - ) - - # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path - get_filename_component(file_path "${file}" PATH) - if(IS_ABSOLUTE "${file_path}") - set(source_file "${file}") - else() - set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}") - endif() - - # Bring in the dependencies - HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file}) - - # Configure the build script - configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY) - file(GENERATE - OUTPUT "${custom_target_script}" - INPUT "${custom_target_script_pregen}" - ) - set(main_dep DEPENDS ${source_file}) - if(CMAKE_GENERATOR MATCHES "Makefiles") - set(verbose_output "$(VERBOSE)") - elseif(HIP_VERBOSE_BUILD) - set(verbose_output ON) - else() - set(verbose_output OFF) - endif() - - # Create up the comment string - file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}") - set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}") - - # Build the generated file and dependency file - add_custom_command( - OUTPUT ${generated_file} - # These output files depend on the source_file and the contents of cmake_dependency_file - ${main_dep} - DEPENDS ${HIP_HIPCC_DEPEND} - DEPENDS ${custom_target_script} - # Make sure the output directory exists before trying to write to it. - COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}" - COMMAND ${CMAKE_COMMAND} ARGS - -D verbose:BOOL=${verbose_output} - -D build_configuration:STRING=${_hip_build_configuration} - -D "generated_file:STRING=${generated_file}" - -P "${custom_target_script}" - WORKING_DIRECTORY "${hip_compile_output_dir}" - COMMENT "${hip_build_comment_string}" - ) - - # Make sure the build system knows the file is generated - set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE) - list(APPEND _hip_generated_files ${generated_file}) - list(APPEND _hip_source_files ${file}) - endif() - endforeach() - - # Set the return parameter - set(${_generated_files} ${_hip_generated_files}) - set(${_source_files} ${_hip_source_files}) -endmacro() - -############################################################################### -# HIP_ADD_EXECUTABLE -############################################################################### -macro(HIP_ADD_EXECUTABLE hip_target) - # Separate the sources from the options - HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN}) - HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options}) - if(_source_files) - list(REMOVE_ITEM _sources ${_source_files}) - endif() - if("x${HCC_HOME}" STREQUAL "x") - set(HCC_HOME "/opt/rocm/hcc") - endif() - set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} -o ") - add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP) -endmacro() - -############################################################################### -# HIP_ADD_LIBRARY -############################################################################### -macro(HIP_ADD_LIBRARY hip_target) - # Separate the sources from the options - HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _nvcc_options ${ARGN}) - HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options}) - if(_source_files) - list(REMOVE_ITEM _sources ${_source_files}) - endif() - add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX}) -endmacro() - -# vim: ts=4:sw=4:expandtab:smartindent diff --git a/common/Communication.h b/common/Communication.h index 7c2f8d08..7819a0bb 100644 --- a/common/Communication.h +++ b/common/Communication.h @@ -1,7 +1,7 @@ #ifndef COMMUNICATION_H_INC #define COMMUNICATION_H_INC -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include "common/Array.h" @@ -38,7 +38,7 @@ struct RankInfoStruct { //! Redistribute domain data (dst may be smaller than the src) template Array redistribute( const RankInfoStruct& src_rank, const Array& src_data, - const RankInfoStruct& dst_rank, std::array dst_size, const Utilities::MPI& comm ); + const RankInfoStruct& dst_rank, std::array dst_size, MPI_Comm comm ); /*! @@ -59,7 +59,7 @@ public: * @param[in] fill Fill {faces,edges,corners} * @param[in] periodic Periodic dimensions */ - fillHalo( const Utilities::MPI& comm, const RankInfoStruct& info, + fillHalo( MPI_Comm comm, const RankInfoStruct& info, std::array n, std::array ng, int tag, int depth, std::array fill = {true,true,true}, std::array periodic = {true,true,true} ); @@ -83,7 +83,7 @@ public: private: - Utilities::MPI comm; + MPI_Comm comm; RankInfoStruct info; std::array n, ng; int depth; @@ -93,6 +93,8 @@ private: TYPE *mem; TYPE *send[3][3][3], *recv[3][3][3]; MPI_Request send_req[3][3][3], recv_req[3][3][3]; + size_t N_type; + MPI_Datatype datatype; fillHalo(); // Private empty constructor fillHalo(const fillHalo&); // Private copy constructor fillHalo& operator=(const fillHalo&); // Private assignment operator @@ -134,7 +136,7 @@ void InitializeRanks( const int rank, const int nprocx, const int nprocy, const //*************************************************************************************** -inline void CommunicateSendRecvCounts( const Utilities::MPI& Communicator, int sendtag, int recvtag, +inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag, int rank_x, int rank_y, int rank_z, int rank_X, int rank_Y, int rank_Z, int rank_xy, int rank_XY, int rank_xY, int rank_Xy, @@ -153,53 +155,53 @@ inline void CommunicateSendRecvCounts( const Utilities::MPI& Communicator, int s { MPI_Request req1[18], req2[18]; MPI_Status stat1[18],stat2[18]; - MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator.getCommunicator(),&req1[0]); - MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator.getCommunicator(),&req2[0]); - MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator.getCommunicator(),&req1[1]); - MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator.getCommunicator(),&req2[1]); - MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator.getCommunicator(),&req1[2]); - MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator.getCommunicator(),&req2[2]); - MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator.getCommunicator(),&req1[3]); - MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator.getCommunicator(),&req2[3]); - MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator.getCommunicator(),&req1[4]); - MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator.getCommunicator(),&req2[4]); - MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator.getCommunicator(),&req1[5]); - MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator.getCommunicator(),&req2[5]); + MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]); + MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]); + MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]); + MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]); + MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]); + MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]); + MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]); + MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]); + MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]); + MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]); + MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]); + MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]); - MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator.getCommunicator(),&req1[6]); - MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator.getCommunicator(),&req2[6]); - MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator.getCommunicator(),&req1[7]); - MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator.getCommunicator(),&req2[7]); - MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator.getCommunicator(),&req1[8]); - MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator.getCommunicator(),&req2[8]); - MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator.getCommunicator(),&req1[9]); - MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator.getCommunicator(),&req2[9]); + MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]); + MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]); + MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]); + MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]); + MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]); + MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]); + MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]); + MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]); - MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator.getCommunicator(),&req1[10]); - MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator.getCommunicator(),&req2[10]); - MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator.getCommunicator(),&req1[11]); - MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator.getCommunicator(),&req2[11]); - MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator.getCommunicator(),&req1[12]); - MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator.getCommunicator(),&req2[12]); - MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator.getCommunicator(),&req1[13]); - MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator.getCommunicator(),&req2[13]); + MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]); + MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]); + MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]); + MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]); + MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]); + MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]); + MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]); + MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]); - MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator.getCommunicator(),&req1[14]); - MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator.getCommunicator(),&req2[14]); - MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator.getCommunicator(),&req1[15]); - MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator.getCommunicator(),&req2[15]); - MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator.getCommunicator(),&req1[16]); - MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator.getCommunicator(),&req2[16]); - MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator.getCommunicator(),&req1[17]); - MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator.getCommunicator(),&req2[17]); + MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]); + MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]); + MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]); + MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]); + MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]); + MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]); + MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]); + MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - Communicator.barrier(); + MPI_Barrier(Communicator); } //*************************************************************************************** -inline void CommunicateRecvLists( const Utilities::MPI& Communicator, int sendtag, int recvtag, +inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag, int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z, int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy, int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz, @@ -221,52 +223,52 @@ inline void CommunicateRecvLists( const Utilities::MPI& Communicator, int sendta { MPI_Request req1[18], req2[18]; MPI_Status stat1[18],stat2[18]; - MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator.getCommunicator(),&req1[0]); - MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator.getCommunicator(),&req2[0]); - MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator.getCommunicator(),&req1[1]); - MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator.getCommunicator(),&req2[1]); - MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator.getCommunicator(),&req1[2]); - MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator.getCommunicator(),&req2[2]); - MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator.getCommunicator(),&req1[3]); - MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator.getCommunicator(),&req2[3]); - MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator.getCommunicator(),&req1[4]); - MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator.getCommunicator(),&req2[4]); - MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator.getCommunicator(),&req1[5]); - MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator.getCommunicator(),&req2[5]); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]); - MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator.getCommunicator(),&req1[6]); - MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator.getCommunicator(),&req2[6]); - MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator.getCommunicator(),&req1[7]); - MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator.getCommunicator(),&req2[7]); - MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator.getCommunicator(),&req1[8]); - MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator.getCommunicator(),&req2[8]); - MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator.getCommunicator(),&req1[9]); - MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator.getCommunicator(),&req2[9]); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]); - MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator.getCommunicator(),&req1[10]); - MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator.getCommunicator(),&req2[10]); - MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator.getCommunicator(),&req1[11]); - MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator.getCommunicator(),&req2[11]); - MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator.getCommunicator(),&req1[12]); - MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator.getCommunicator(),&req2[12]); - MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator.getCommunicator(),&req1[13]); - MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator.getCommunicator(),&req2[13]); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]); - MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator.getCommunicator(),&req1[14]); - MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator.getCommunicator(),&req2[14]); - MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator.getCommunicator(),&req1[15]); - MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator.getCommunicator(),&req2[15]); - MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator.getCommunicator(),&req1[16]); - MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator.getCommunicator(),&req2[16]); - MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator.getCommunicator(),&req1[17]); - MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator.getCommunicator(),&req2[17]); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); } //*************************************************************************************** -inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& Communicator, +inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator, double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z, double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy, double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz, @@ -317,41 +319,41 @@ inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& Communi PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData); //...................................................................................... MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag, - recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag, - recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag, - recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag, - recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag, - recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag, - recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag, - recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag, - recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag, - recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag, - recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag, - recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag, - recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag, - recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag, - recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag, - recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag, - recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag, - recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE); MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag, - recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator.getCommunicator(),MPI_STATUS_IGNORE); + recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE); //........................................................................................ UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData); UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData); diff --git a/common/Communication.hpp b/common/Communication.hpp index ca310ea5..33fed3a7 100644 --- a/common/Communication.hpp +++ b/common/Communication.hpp @@ -2,8 +2,9 @@ #define COMMUNICATION_HPP_INC #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" +//#include "ProfilerApp.h" /******************************************************** @@ -11,19 +12,17 @@ ********************************************************/ template Array redistribute( const RankInfoStruct& src_rank, const Array& src_data, - const RankInfoStruct& dst_rank, std::array dst_size, const Utilities::MPI& comm ) + const RankInfoStruct& dst_rank, std::array dst_size, MPI_Comm comm ) { - if ( comm.getSize() == 1 ) { - return src_data.subset( { 0, (size_t) dst_size[0]-1, 0, (size_t) dst_size[1]-1, 0, (size_t) dst_size[2]-1 } ); - } +#ifdef USE_MPI // Get the src size std::array src_size; int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) }; - comm.maxReduce( size0, src_size.data(), 3 ); + MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm ); if ( !src_data.empty() ) ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] ); // Check that dst_size matches on all ranks - comm.maxReduce( dst_size.data(), size0, 3 ); + MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm ); ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] ); // Function to get overlap range auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) { @@ -61,7 +60,7 @@ Array redistribute( const RankInfoStruct& src_rank, const Array& src } std::vector send_request( send_rank.size() ); for (size_t i=0; i dst_data( dst_size[0], dst_size[1], dst_size[2] ); int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz }; @@ -76,14 +75,17 @@ Array redistribute( const RankInfoStruct& src_rank, const Array& src continue; int rank = src_rank.getRankForBlock(i,j,k); Array data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 ); - comm.recv( data.data(), data.length(), rank, 5462 ); + MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE ); dst_data.copySubset( index, data ); } } } // Free data - comm.waitAll( send_request.size(), send_request.data() ); + MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE ); return dst_data; +#else + return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 ); +#endif } @@ -92,11 +94,27 @@ Array redistribute( const RankInfoStruct& src_rank, const Array& src * Structure to fill halo cells * ********************************************************/ template -fillHalo::fillHalo( const Utilities::MPI& comm_, const RankInfoStruct& info_, +fillHalo::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_, std::array n_, std::array ng_, int tag0, int depth_, std::array fill, std::array periodic ): comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_) { + if ( std::is_same() ) { + N_type = 1; + datatype = MPI_DOUBLE; + } else if ( std::is_same() ) { + N_type = 1; + datatype = MPI_FLOAT; + } else if ( sizeof(TYPE)%sizeof(double)==0 ) { + N_type = sizeof(TYPE) / sizeof(double); + datatype = MPI_DOUBLE; + } else if ( sizeof(TYPE)%sizeof(float)==0 ) { + N_type = sizeof(TYPE) / sizeof(float); + datatype = MPI_FLOAT; + } else { + N_type = sizeof(TYPE); + datatype = MPI_BYTE; + } // Set the fill pattern memset(fill_pattern,0,sizeof(fill_pattern)); if ( fill[0] ) { @@ -233,8 +251,8 @@ void fillHalo::fill( Array& data ) for (int k=0; k<3; k++) { if ( !fill_pattern[i][j][k] ) continue; - recv_req[i][j][k] = comm.Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k], - info.rank[i][j][k], tag[2-i][2-j][2-k] ); + MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype, + info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] ); } } } @@ -245,18 +263,19 @@ void fillHalo::fill( Array& data ) if ( !fill_pattern[i][j][k] ) continue; pack( data, i-1, j-1, k-1, send[i][j][k] ); - send_req[i][j][k] = comm.Isend( send[i][j][k], depth2*N_send_recv[i][j][k], - info.rank[i][j][k], tag[i][j][k] ); + MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype, + info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] ); } } } // Recv the dst data and unpack (we recive in reverse order to match the sends) + MPI_Status status; for (int i=2; i>=0; i--) { for (int j=2; j>=0; j--) { for (int k=2; k>=0; k--) { if ( !fill_pattern[i][j][k] ) continue; - comm.wait( recv_req[i][j][k] ); + MPI_Wait(&recv_req[i][j][k],&status); unpack( data, i-1, j-1, k-1, recv[i][j][k] ); } } @@ -267,7 +286,7 @@ void fillHalo::fill( Array& data ) for (int k=0; k<3; k++) { if ( !fill_pattern[i][j][k] ) continue; - comm.wait( send_req[i][j][k] ); + MPI_Wait(&send_req[i][j][k],&status); } } } diff --git a/common/Domain.cpp b/common/Domain.cpp index ff6c6b68..7da902c6 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -12,7 +12,7 @@ #include "common/Domain.h" #include "common/Array.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" // Inline function to read line without a return argument @@ -62,10 +62,11 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, NULL_USE( npy ); NULL_USE( npz ); // set up the neighbor ranks - int myrank = Comm.getRank(); + int myrank; + MPI_Comm_rank( Comm, &myrank ); rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); - Comm.barrier(); + MPI_Barrier(Comm); auto db = std::make_shared( ); db->putScalar( "BC", BC ); @@ -75,9 +76,10 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, db->putVector( "L", { lx, ly, lz } ); initialize( db ); } -Domain::Domain( std::shared_ptr db, const Utilities::MPI& Communicator): +Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): database(db), Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), + Comm(MPI_COMM_NULL), inlet_layers_x(0), inlet_layers_y(0), inlet_layers_z(0), outlet_layers_x(0), outlet_layers_y(0), outlet_layers_z(0), inlet_layers_phase(1),outlet_layers_phase(2), @@ -107,13 +109,14 @@ Domain::Domain( std::shared_ptr db, const Utilities::MPI& Communicator recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), id(NULL) { - Comm = Communicator.dup(); + MPI_Comm_dup(Communicator,&Comm); // set up the neighbor ranks - int myrank = Comm.getRank(); + int myrank; + MPI_Comm_rank( Comm, &myrank ); initialize( db ); rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); - Comm.barrier(); + MPI_Barrier(Comm); } Domain::~Domain() @@ -162,6 +165,10 @@ Domain::~Domain() delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; // Free id delete [] id; + // Free the communicator + if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { + MPI_Comm_free(&Comm); + } } void Domain::initialize( std::shared_ptr db ) @@ -212,7 +219,8 @@ void Domain::initialize( std::shared_ptr db ) Ny = ny+2; Nz = nz+2; // Initialize ranks - int myrank = Comm.getRank(); + int myrank; + MPI_Comm_rank( Comm, &myrank ); rank_info = RankInfoStruct(myrank,nproc[0],nproc[1],nproc[2]); // inlet layers only apply to lower part of domain if (rank_info.ix > 0) inlet_layers_x = 0; @@ -231,7 +239,8 @@ void Domain::initialize( std::shared_ptr db ) id = new signed char[N]; memset(id,0,N); BoundaryCondition = d_db->getScalar("BC"); - int nprocs = Comm.getSize(); + int nprocs; + MPI_Comm_size( Comm, &nprocs ); INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!"); } @@ -560,7 +569,7 @@ void Domain::Decomp( const std::string& Filename ) } else{ //printf("Sending data to process %i \n", rnk); - Comm.send(loc_id,N,rnk,15); + MPI_Send(loc_id,N,MPI_CHAR,rnk,15,Comm); } // Write the data for this rank data sprintf(LocalRankFilename,"ID.%05i",rnk+rank_offset); @@ -575,10 +584,10 @@ void Domain::Decomp( const std::string& Filename ) else{ // Recieve the subdomain from rank = 0 //printf("Ready to recieve data %i at process %i \n", N,rank); - Comm.recv(id,N,0,15); + MPI_Recv(id,N,MPI_CHAR,0,15,Comm,MPI_STATUS_IGNORE); } - Comm.barrier(); - + //Comm.barrier(); + MPI_Barrier(Comm); // Compute the porosity double sum; double sum_local=0.0; @@ -618,7 +627,8 @@ void Domain::Decomp( const std::string& Filename ) } } } - sum = Comm.sumReduce(sum_local); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm); + //sum = Comm.sumReduce(sum_local); porosity = sum*iVol_global; if (rank()==0) printf("Media porosity = %f \n",porosity); //......................................................... @@ -661,7 +671,7 @@ void Domain::AggregateLabels( const std::string& filename ){ } } } - Comm.barrier(); + MPI_Barrier(Comm); // populate the FullID if (rank() == 0){ @@ -687,7 +697,7 @@ void Domain::AggregateLabels( const std::string& filename ){ ipx = (rnk - ipz*npx*npy - ipy*npx); //printf("ipx=%i ipy=%i ipz=%i\n", ipx, ipy, ipz); int tag = 15+rnk; - Comm.recv(LocalID,local_size,rnk,tag); + MPI_Recv(LocalID,local_size,MPI_CHAR,rnk,tag,Comm,MPI_STATUS_IGNORE); for (int k=1; k db, const Utilities::MPI& Communicator); + Domain( std::shared_ptr db, MPI_Comm Communicator); //! Obsolete constructor Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, @@ -116,7 +116,7 @@ public: // Public variables (need to create accessors instead) double porosity; RankInfoStruct rank_info; - Utilities::MPI Comm; // MPI Communicator for this domain + MPI_Comm Comm; // MPI Communicator for this domain int BoundaryCondition; diff --git a/common/MPI.I b/common/MPI.I deleted file mode 100644 index 8cbc9c09..00000000 --- a/common/MPI.I +++ /dev/null @@ -1,1143 +0,0 @@ -// This file contains the default instantiations for templated operations -// Note: Intel compilers need definitions before all default instantions to compile correctly -#ifndef included_MPI_I -#define included_MPI_I - -#include "common/Utilities.h" - -#include - - -#define MPI_CLASS MPI -#define MPI_CLASS_ERROR ERROR -#define MPI_CLASS_ASSERT ASSERT - -#undef NULL_USE -#define NULL_USE( variable ) \ - do { \ - if ( 0 ) { \ - auto static t = (char *) &variable; \ - t++; \ - } \ - } while ( 0 ) - - -namespace Utilities { - - -// Function to test if a type is a std::pair -template -struct is_pair : std::false_type { -}; -template -struct is_pair> : std::true_type { -}; - - -// Function to test if a type can be passed by MPI -template -constexpr typename std::enable_if::value,bool>::type - is_mpi_copyable() -{ - return true; -} -template -constexpr typename std::enable_if::value&&is_pair::value,bool>::type - is_mpi_copyable() -{ - return is_mpi_copyable() && is_mpi_copyable(); -} -template -constexpr typename std::enable_if::value&&!is_pair::value,bool>::type - is_mpi_copyable() -{ - return false; -} - - -/************************************************************************ - * sumReduce * - ************************************************************************/ -template -inline TYPE MPI_CLASS::sumReduce( const TYPE value ) const -{ - if ( comm_size > 1 ) { - TYPE tmp = value; - call_sumReduce( &tmp, 1 ); - return tmp; - } else { - return value; - } -} -template -inline void MPI_CLASS::sumReduce( TYPE *x, const int n ) const -{ - if ( comm_size > 1 ) - call_sumReduce( x, n ); -} -template -inline void MPI_CLASS::sumReduce( const TYPE *x, TYPE *y, const int n ) const -{ - if ( comm_size > 1 ) { - call_sumReduce( x, y, n ); - } else { - for ( int i = 0; i < n; i++ ) - y[i] = x[i]; - } -} -// Define specializations of call_sumReduce(TYPE*, const int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_sumReduce( unsigned char *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( char *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( unsigned int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( unsigned long int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( long int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( size_t *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( float *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( double *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce>( std::complex *, const int ) const; -#endif -// Default instantiations of call_sumReduce(TYPE*, const int) -template -void MPI_CLASS::call_sumReduce( TYPE *, const int ) const -{ - char message[200]; - sprintf( message, "Default instantion of sumReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} -// Define specializations of call_sumReduce(const TYPE*, TYPE*, const int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_sumReduce( - const unsigned char *, unsigned char *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const char *, char *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( - const unsigned int *, unsigned int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const int *, int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( - const unsigned long int *, unsigned long int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const long int *, long int *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const size_t *, size_t *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const float *, float *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce( const double *, double *, const int ) const; -template<> -void MPI_CLASS::call_sumReduce>( - const std::complex *, std::complex *, const int ) const; -#endif -// Default instantiations of call_sumReduce(const TYPE*, TYPE*, const int) -template -void MPI_CLASS::call_sumReduce( const TYPE *x, TYPE *y, const int n ) const -{ - NULL_USE( x ); - NULL_USE( y ); - NULL_USE( n ); - char message[200]; - sprintf( message, "Default instantion of sumReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * minReduce * - ************************************************************************/ -template -inline TYPE MPI_CLASS::minReduce( const TYPE value ) const -{ - if ( comm_size > 1 ) { - TYPE tmp = value; - call_minReduce( &tmp, 1, nullptr ); - return tmp; - } else { - return value; - } -} -template -inline void MPI_CLASS::minReduce( TYPE *x, const int n, int *rank_of_min ) const -{ - if ( comm_size > 1 ) { - call_minReduce( x, n, rank_of_min ); - } else { - if ( rank_of_min != nullptr ) { - for ( int i = 0; i < n; i++ ) - rank_of_min[i] = 0; - } - } -} -template -inline void MPI_CLASS::minReduce( const TYPE *x, TYPE *y, const int n, int *rank_of_min ) const -{ - if ( comm_size > 1 ) { - call_minReduce( x, y, n, rank_of_min ); - } else { - for ( int i = 0; i < n; i++ ) { - y[i] = x[i]; - if ( rank_of_min != nullptr ) - rank_of_min[i] = 0; - } - } -} -// Define specializations of call_minReduce(TYPE*, const int, int*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_minReduce( unsigned char *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( char *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( unsigned int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( unsigned long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( - unsigned long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( size_t *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( float *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( double *, const int, int * ) const; -#endif -// Default instantiations of call_minReduce(TYPE*, const int, int*) -template -void MPI_CLASS::call_minReduce( TYPE *, const int, int * ) const -{ - char message[200]; - sprintf( message, "Default instantion of minReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} -// Define specializations of call_minReduce(const TYPE*, TYPE*, const int, int*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_minReduce( - const unsigned char *, unsigned char *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const char *, char *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( - const unsigned int *, unsigned int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const int *, int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( - const unsigned long int *, unsigned long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const long int *, long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( - const unsigned long long int *, unsigned long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( - const long long int *, long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const size_t *, size_t *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const float *, float *, const int, int * ) const; -template<> -void MPI_CLASS::call_minReduce( const double *, double *, const int, int * ) const; -#endif -// Default instantiations of call_minReduce(const TYPE*, TYPE*, const int, int*) -template -void MPI_CLASS::call_minReduce( const TYPE *, TYPE *, const int, int * ) const -{ - char message[200]; - sprintf( message, "Default instantion of minReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * maxReduce * - ************************************************************************/ -template -inline TYPE MPI_CLASS::maxReduce( const TYPE value ) const -{ - if ( comm_size > 1 ) { - TYPE tmp = value; - call_maxReduce( &tmp, 1, nullptr ); - return tmp; - } else { - return value; - } -} -template -inline void MPI_CLASS::maxReduce( TYPE *x, const int n, int *rank_of_max ) const -{ - if ( comm_size > 1 ) { - call_maxReduce( x, n, rank_of_max ); - } else { - if ( rank_of_max != nullptr ) { - for ( int i = 0; i < n; i++ ) - rank_of_max[i] = 0; - } - } -} -template -inline void MPI_CLASS::maxReduce( const TYPE *x, TYPE *y, const int n, int *rank_of_max ) const -{ - if ( comm_size > 1 ) { - call_maxReduce( x, y, n, rank_of_max ); - } else { - for ( int i = 0; i < n; i++ ) { - y[i] = x[i]; - if ( rank_of_max != nullptr ) - rank_of_max[i] = 0; - } - } -} -// Define specializations of call_maxReduce(TYPE*, const int, int*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_maxReduce( unsigned char *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( char *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( unsigned int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( unsigned long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( - unsigned long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( size_t *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( float *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( double *, const int, int * ) const; -#endif -// Default instantiations of call_maxReduce(TYPE*, const int, int*) -template -void MPI_CLASS::call_maxReduce( TYPE *, const int, int * ) const -{ - char message[200]; - sprintf( message, "Default instantion of maxReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} -// Define specializations of call_maxReduce(const TYPE*, TYPE*, const int, int*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_maxReduce( - const unsigned char *, unsigned char *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const char *, char *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( - const unsigned int *, unsigned int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const int *, int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( - const unsigned long int *, unsigned long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const long int *, long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( - const unsigned long long int *, unsigned long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( - const long long int *, long long int *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const size_t *, size_t *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const float *, float *, const int, int * ) const; -template<> -void MPI_CLASS::call_maxReduce( const double *, double *, const int, int * ) const; -#endif -// Default instantiations of call_maxReduce(const TYPE*, TYPE*, const int, int*) -template -void MPI_CLASS::call_maxReduce( const TYPE *, TYPE *, const int, int * ) const -{ - char message[200]; - sprintf( message, "Default instantion of maxReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * bcast * - ************************************************************************/ -// Define specializations of bcast(TYPE*, const int, const int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_bcast( unsigned char *, const int, const int ) const; -template<> -void MPI_CLASS::call_bcast( char *, const int, const int ) const; -template<> -void MPI_CLASS::call_bcast( unsigned int *, const int, const int ) const; -template<> -void MPI_CLASS::call_bcast( int *, const int, const int ) const; -template<> -void MPI_CLASS::call_bcast( float *, const int, const int ) const; -template<> -void MPI_CLASS::call_bcast( double *, const int, const int ) const; -#else -template<> -void MPI_CLASS::call_bcast( char *, const int, const int ) const; -#endif -// Default instantiations of bcast(TYPE*, const int, const int) -template -void MPI_CLASS::call_bcast( TYPE *x, const int n, const int root ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - call_bcast( (char *) x, (int) n * sizeof( TYPE ), root ); -} -// Specialization of bcast for std::string -template<> -inline std::string MPI_CLASS::bcast( const std::string &value, const int root ) const -{ - if ( comm_size == 1 ) - return value; - int length = static_cast( value.size() ); - call_bcast( &length, 1, root ); - if ( length == 0 ) - return std::string(); - char *str = new char[length + 1]; - if ( root == comm_rank ) { - for ( int i = 0; i < length; i++ ) - str[i] = value[i]; - } - call_bcast( str, length, root ); - str[length] = 0; - std::string result( str ); - delete[] str; - return result; -} -template<> -inline void MPI_CLASS::bcast( std::string *, const int, const int ) const -{ - MPI_CLASS_ERROR( "Cannot bcast an array of strings" ); -} -// Default implimentation of bcast -template -inline TYPE MPI_CLASS::bcast( const TYPE &value, const int root ) const -{ - if ( root >= comm_size ) - MPI_CLASS_ERROR( "root cannot be >= size in bcast" ); - if ( comm_size > 1 ) { - TYPE tmp = value; - call_bcast( &tmp, 1, root ); - return tmp; - } else { - return value; - } -} -template -inline void MPI_CLASS::bcast( TYPE *x, const int n, const int root ) const -{ - if ( root >= comm_size ) - MPI_CLASS_ERROR( "root cannot be >= size in bcast" ); - if ( comm_size > 1 ) - call_bcast( x, n, root ); -} - - -/************************************************************************ - * send * - ************************************************************************/ -// Define specializations of send(const TYPE*, const int, const int, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::send( const char *, const int, const int, int ) const; -template<> -void MPI_CLASS::send( const int *, int, const int, int ) const; -template<> -void MPI_CLASS::send( const float *, const int, const int, int ) const; -template<> -void MPI_CLASS::send( const double *, const int, const int, int ) const; -#else -template<> -void MPI_CLASS::send( const char *, const int, const int, int ) const; -#endif -// Default instantiations of send(const TYPE*, const int, const int, int) -template -inline void MPI_CLASS::send( - const TYPE *buf, const int length, const int recv_proc_number, int tag ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - send( (const char *) buf, length * sizeof( TYPE ), recv_proc_number, tag ); -} - - -/************************************************************************ - * Isend * - ************************************************************************/ -// Define specializations of Isend(const TYPE*, const int, const int, const int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -MPI_Request MPI_CLASS::Isend( const char *, const int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Isend( const int *, int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Isend( const float *, const int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Isend( const double *, const int, const int, const int ) const; -#else -template<> -MPI_Request MPI_CLASS::Isend( const char *, const int, const int, const int ) const; -#endif -// Default instantiations of Isend(const TYPE*, const int, const int, const int) -template -inline MPI_Request MPI_CLASS::Isend( - const TYPE *buf, const int length, const int recv_proc_number, const int tag ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - return Isend( (const char *) buf, length * sizeof( TYPE ), recv_proc_number, tag ); -} - - -/************************************************************************ - * recv * - ************************************************************************/ -// Define specializations of recv(TYPE*, int&, const int, const bool, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::recv( char *, int &, const int, const bool, int ) const; -template<> -void MPI_CLASS::recv( int *, int &, const int, const bool, int ) const; -template<> -void MPI_CLASS::recv( float *, int &, const int, const bool, int ) const; -template<> -void MPI_CLASS::recv( double *, int &, const int, const bool, int ) const; -#else -template<> -void MPI_CLASS::recv( char *, int &, const int, const bool, int ) const; -#endif -// Default instantiations of recv(TYPE*, int&, const int, const bool, int) -template -inline void MPI_CLASS::recv( - TYPE *buf, int &length, const int send_proc_number, const bool get_length, int tag ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - int size = length * sizeof( TYPE ); - recv( (char *) buf, size, send_proc_number, get_length, tag ); - if ( get_length ) { - MPI_CLASS_ASSERT( size % sizeof( TYPE ) == 0 ); - length = size / sizeof( TYPE ); - } -} - - -/************************************************************************ - * Irecv * - ************************************************************************/ -// Define specializations of recv(TYPE*, int&, const int, const bool, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -MPI_Request MPI_CLASS::Irecv( char *, const int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Irecv( int *, const int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Irecv( float *, const int, const int, const int ) const; -template<> -MPI_Request MPI_CLASS::Irecv( double *, const int, const int, const int ) const; -#else -template<> -MPI_Request MPI_CLASS::Irecv( char *, const int, const int, const int ) const; -#endif -// Default instantiations of recv(TYPE*, int&, const int, const bool, int) -template -inline MPI_Request MPI_CLASS::Irecv( - TYPE *buf, const int length, const int send_proc, const int tag ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - return Irecv( (char *) buf, length * sizeof( TYPE ), send_proc, tag ); -} - - -/************************************************************************ - * allGather * - ************************************************************************/ -template -std::vector MPI_CLASS::allGather( const TYPE &x ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - if ( getSize() <= 1 ) - return std::vector( 1, x ); - std::vector data( getSize() ); - allGather( x, data.data() ); - return data; -} -template -std::vector MPI_CLASS::allGather( const std::vector &x ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - if ( getSize() <= 1 ) - return x; - std::vector count = allGather( x.size() ); - std::vector disp( getSize(), 0 ); - size_t N = count[0]; - for ( size_t i = 1; i < count.size(); i++ ) { - disp[i] = disp[i - 1] + count[i - 1]; - N += count[i]; - } - std::vector data( N ); - allGather( x.data(), x.size(), data.data(), count.data(), disp.data(), true ); - return data; -} -// Specialization of MPI_CLASS::allGather for std::string -template<> -inline void MPI_CLASS::allGather( const std::string &x_in, std::string *x_out ) const -{ - // Get the bytes recvied per processor - std::vector recv_cnt( comm_size, 0 ); - allGather( (int) x_in.size() + 1, &recv_cnt[0] ); - std::vector recv_disp( comm_size, 0 ); - for ( int i = 1; i < comm_size; i++ ) - recv_disp[i] = recv_disp[i - 1] + recv_cnt[i - 1]; - // Call the vector form of allGather for the char arrays - char *recv_data = new char[recv_disp[comm_size - 1] + recv_cnt[comm_size - 1]]; - allGather( - x_in.c_str(), (int) x_in.size() + 1, recv_data, &recv_cnt[0], &recv_disp[0], true ); - for ( int i = 0; i < comm_size; i++ ) - x_out[i] = std::string( &recv_data[recv_disp[i]] ); - delete[] recv_data; -} -// Default instantiation of MPI_CLASS::allGather -template -inline void MPI_CLASS::allGather( const TYPE &x_in, TYPE *x_out ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - if ( comm_size > 1 ) { - // We can use the vector form of allGather with a char array to ge the data we want - call_allGather( x_in, x_out ); - } else { - // Single processor case - x_out[0] = x_in; - } -} -// Specialization of MPI_CLASS::allGather for std::string -template<> -inline int MPI_CLASS::allGather( - const std::string *, const int, std::string *, int *, int *, bool ) const -{ - MPI_CLASS_ERROR( "Cannot allGather an array of strings" ); - return 0; -} -// Define specializations of call_allGather(const TYPE, TYPE*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_allGather( const unsigned char &, unsigned char * ) const; -template<> -void MPI_CLASS::call_allGather( const char &, char * ) const; -template<> -void MPI_CLASS::call_allGather( const unsigned int &, unsigned int * ) const; -template<> -void MPI_CLASS::call_allGather( const int &, int * ) const; -template<> -void MPI_CLASS::call_allGather( - const unsigned long int &, unsigned long int * ) const; -template<> -void MPI_CLASS::call_allGather( const long int &, long int * ) const; -template<> -void MPI_CLASS::call_allGather( const float &, float * ) const; -template<> -void MPI_CLASS::call_allGather( const double &, double * ) const; -#endif -// Default instantiation of MPI_CLASS::allGather -template -int MPI_CLASS::allGather( const TYPE *send_data, const int send_cnt, TYPE *recv_data, int *recv_cnt, - int *recv_disp, bool known_recv ) const -{ - // Check the inputs - if ( known_recv && ( recv_cnt == nullptr || recv_disp == nullptr ) ) - MPI_CLASS_ERROR( "Error calling allGather" ); - // Check if we are dealing with a single processor - if ( comm_size == 1 ) { - if ( send_data == nullptr && send_cnt > 0 ) { - MPI_CLASS_ERROR( "send_data is null" ); - } else if ( !known_recv ) { - // We do not know the recieved sizes - for ( int i = 0; i < send_cnt; i++ ) - recv_data[i] = send_data[i]; - if ( recv_cnt != nullptr ) - recv_cnt[0] = send_cnt; - if ( recv_disp != nullptr ) - recv_disp[0] = 0; - } else { - // We know the recieved sizes - for ( int i = 0; i < send_cnt; i++ ) - recv_data[i + recv_disp[0]] = send_data[i]; - } - return send_cnt; - } - // Get the sizes of the recieved data (if necessary) - int *recv_cnt2 = recv_cnt; - int *recv_disp2 = recv_disp; - if ( !known_recv ) { - if ( recv_cnt == nullptr ) - recv_cnt2 = new int[comm_size]; - if ( recv_disp == nullptr ) - recv_disp2 = new int[comm_size]; - call_allGather( send_cnt, recv_cnt2 ); - recv_disp2[0] = 0; - for ( int i = 1; i < comm_size; i++ ) - recv_disp2[i] = recv_disp2[i - 1] + recv_cnt2[i - 1]; - } - int N_recv = 0; - for ( int i = 0; i < comm_size; i++ ) - N_recv += recv_cnt2[i]; - // Send/recv the data - call_allGather( send_data, send_cnt, recv_data, recv_cnt2, recv_disp2 ); - // Delete any temporary memory - if ( recv_cnt == nullptr ) - delete[] recv_cnt2; - if ( recv_disp == nullptr ) - delete[] recv_disp2; - return N_recv; -} -// Default instantiations of call_allGather(const TYPE, TYPE*) -template -void MPI_CLASS::call_allGather( const TYPE &x_in, TYPE *x_out ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - allGather( (const char *) &x_in, (int) sizeof( TYPE ), (char *) x_out ); -} -// Define specializations of call_allGather(const TYPE*, int, TYPE*, int*, int*) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_allGather( - const unsigned char *, int, unsigned char *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( const char *, int, char *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( - const unsigned int *, int, unsigned int *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( const int *, int, int *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( - const unsigned long int *, int, unsigned long int *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( const long int *, int, long int *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( const float *, int, float *, int *, int * ) const; -template<> -void MPI_CLASS::call_allGather( const double *, int, double *, int *, int * ) const; -#else -template<> -void MPI_CLASS::call_allGather( const char *, int, char *, int *, int * ) const; -#endif -// Default instantiations of int call_allGather(const TYPE*, int, TYPE*, int*) -template -void MPI_CLASS::call_allGather( - const TYPE *x_in, int size_in, TYPE *x_out, int *size_out, int *disp_out ) const -{ - int *size2 = new int[comm_size]; - int *disp2 = new int[comm_size]; - for ( int i = 0; i < comm_size; i++ ) { - size2[i] = size_out[i] * sizeof( TYPE ); - disp2[i] = disp_out[i] * sizeof( TYPE ); - } - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - call_allGather( - (const char *) x_in, (int) size_in * sizeof( TYPE ), (char *) x_out, size2, disp2 ); - delete[] size2; - delete[] disp2; -} - - -/************************************************************************ - * setGather * - ************************************************************************/ -template -inline void MPI_CLASS::setGather( std::set &set ) const -{ - std::vector send_buf( set.begin(), set.end() ); - std::vector recv_cnt( this->comm_size, 0 ); - this->allGather( (int) send_buf.size(), &recv_cnt[0] ); - std::vector recv_disp( this->comm_size, 0 ); - for ( int i = 1; i < this->comm_size; i++ ) - recv_disp[i] = recv_disp[i - 1] + recv_cnt[i - 1]; - size_t N_recv_tot = 0; - for ( int i = 0; i < this->comm_size; i++ ) - N_recv_tot += recv_cnt[i]; - if ( N_recv_tot == 0 ) - return; - std::vector recv_buf( N_recv_tot ); - TYPE *send_data = nullptr; - if ( send_buf.size() > 0 ) { - send_data = &send_buf[0]; - } - TYPE *recv_data = &recv_buf[0]; - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - this->allGather( - send_data, (int) send_buf.size(), recv_data, &recv_cnt[0], &recv_disp[0], true ); - for ( size_t i = 0; i < recv_buf.size(); i++ ) - set.insert( recv_buf[i] ); -} - - -/************************************************************************ - * mapGather * - ************************************************************************/ -template -inline void MPI_CLASS::mapGather( std::map &map ) const -{ - std::vector send_id; - std::vector send_data; - send_id.reserve( map.size() ); - send_data.reserve( map.size() ); - for ( auto it = map.begin(); it != map.end(); ++it ) { - send_id.push_back( it->first ); - send_data.push_back( it->second ); - } - int send_size = (int) send_id.size(); - std::vector recv_cnt( this->comm_size, 0 ); - this->allGather( send_size, &recv_cnt[0] ); - std::vector recv_disp( this->comm_size, 0 ); - for ( int i = 1; i < this->comm_size; i++ ) - recv_disp[i] = recv_disp[i - 1] + recv_cnt[i - 1]; - size_t N_recv_tot = 0; - for ( int i = 0; i < this->comm_size; i++ ) - N_recv_tot += recv_cnt[i]; - if ( N_recv_tot == 0 ) - return; - std::vector recv_id( N_recv_tot ); - std::vector recv_data( N_recv_tot ); - KEY *send_data1 = nullptr; - DATA *send_data2 = nullptr; - if ( send_id.size() > 0 ) { - send_data1 = &send_id[0]; - send_data2 = &send_data[0]; - } - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - this->allGather( send_data1, send_size, &recv_id[0], &recv_cnt[0], &recv_disp[0], true ); - this->allGather( - send_data2, send_size, &recv_data[0], &recv_cnt[0], &recv_disp[0], true ); - map = std::map(); - for ( size_t i = 0; i < N_recv_tot; i++ ) - map.insert( std::pair( recv_id[i], recv_data[i] ) ); -} - - -/************************************************************************ - * sumScan * - ************************************************************************/ -template -inline void MPI_CLASS::sumScan( const TYPE *x, TYPE *y, const int n ) const -{ - if ( comm_size > 1 ) { - call_sumScan( x, y, n ); - } else { - for ( int i = 0; i < n; i++ ) - y[i] = x[i]; - } -} -// Define specializations of call_sumScan(const TYPE*, TYPE*, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_sumScan( const unsigned char *, unsigned char *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const char *, char *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const unsigned int *, unsigned int *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const int *, int *, int ) const; -template<> -void MPI_CLASS::call_sumScan( - const unsigned long int *, unsigned long int *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const long int *, long int *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const size_t *, size_t *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const float *, float *, int ) const; -template<> -void MPI_CLASS::call_sumScan( const double *, double *, int ) const; -template<> -void MPI_CLASS::call_sumScan>( - const std::complex *, std::complex *, int ) const; -#endif -// Default instantiations of call_sumScan(const TYPE*, TYPE*, int) -template -void MPI_CLASS::call_sumScan( const TYPE *, TYPE *, int ) const -{ - char message[200]; - sprintf( message, "Default instantion of sumScan in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * minScan * - ************************************************************************/ -template -inline void MPI_CLASS::minScan( const TYPE *x, TYPE *y, const int n ) const -{ - if ( comm_size > 1 ) { - call_minScan( x, y, n ); - } else { - for ( int i = 0; i < n; i++ ) - y[i] = x[i]; - } -} -// Define specializations of call_minScan(const TYPE*, TYPE*, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_minScan( const unsigned char *, unsigned char *, int ) const; -template<> -void MPI_CLASS::call_minScan( const char *, char *, int ) const; -template<> -void MPI_CLASS::call_minScan( const unsigned int *, unsigned int *, int ) const; -template<> -void MPI_CLASS::call_minScan( const int *, int *, int ) const; -template<> -void MPI_CLASS::call_minScan( - const unsigned long int *, unsigned long int *, int ) const; -template<> -void MPI_CLASS::call_minScan( const long int *, long int *, int ) const; -template<> -void MPI_CLASS::call_minScan( const size_t *, size_t *, int ) const; -template<> -void MPI_CLASS::call_minScan( const float *, float *, int ) const; -template<> -void MPI_CLASS::call_minScan( const double *, double *, int ) const; -#endif -// Default instantiations of call_minScan(const TYPE*, TYPE*, int) -template -void MPI_CLASS::call_minScan( const TYPE *, TYPE *, int ) const -{ - char message[200]; - sprintf( message, "Default instantion of minScan in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * maxScan * - ************************************************************************/ -template -inline void MPI_CLASS::maxScan( const TYPE *x, TYPE *y, const int n ) const -{ - if ( comm_size > 1 ) { - call_maxScan( x, y, n ); - } else { - for ( int i = 0; i < n; i++ ) - y[i] = x[i]; - } -} -// Define specializations of call_maxScan(const TYPE*, TYPE*, int) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_maxScan( const unsigned char *, unsigned char *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const char *, char *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const unsigned int *, unsigned int *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const int *, int *, int ) const; -template<> -void MPI_CLASS::call_maxScan( - const unsigned long int *, unsigned long int *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const long int *, long int *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const size_t *, size_t *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const float *, float *, int ) const; -template<> -void MPI_CLASS::call_maxScan( const double *, double *, int ) const; -#endif -// Default instantiations of call_maxScan(const TYPE*, TYPE*, int) -template -void MPI_CLASS::call_maxScan( const TYPE *, TYPE *, int ) const -{ - char message[200]; - sprintf( message, "Default instantion of maxReduce in parallel is not supported (%s)", - typeid( TYPE ).name() ); - MPI_CLASS_ERROR( message ); -} - - -/************************************************************************ - * allToAll * - ************************************************************************/ -// Define specializations of allToAll(const int n, const char*, char* ) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::allToAll( - const int n, const unsigned char *, unsigned char * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const char *, char * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const unsigned int *, unsigned int * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const int *, int * ) const; -template<> -void MPI_CLASS::allToAll( - const int n, const unsigned long int *, unsigned long int * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const long int *, long int * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const float *, float * ) const; -template<> -void MPI_CLASS::allToAll( const int n, const double *, double * ) const; -#endif -// Default instantiations of allToAll(const int n, const char*, char* ) -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template -void MPI_CLASS::allToAll( const int n, const TYPE *send_data, TYPE *recv_data ) const -{ - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - allToAll( n * sizeof( TYPE ), (char *) send_data, (char *) recv_data ); -} -#else -template -void MPI_CLASS::allToAll( const int n, const TYPE *send_data, TYPE *recv_data ) const -{ - if ( comm_size != 1 ) - MPI_CLASS_ERROR( "Invalid size for allToAll" ); - for ( int i = 0; i < n; i++ ) - recv_data[i] = send_data[i]; -} -#endif - - -/************************************************************************ - * allToAll * - ************************************************************************/ -template -int MPI_CLASS::allToAll( const TYPE *send_data, const int send_cnt[], const int send_disp[], - TYPE *recv_data, int *recv_cnt, int *recv_disp, bool known_recv ) const -{ - int N_recieved = 0; - if ( comm_size == 1 ) { - // Special case for single-processor communicators - if ( known_recv ) { - if ( recv_cnt[0] != send_cnt[0] && send_cnt[0] > 0 ) - MPI_CLASS_ERROR( "Single processor send/recv are different sizes" ); - } else { - if ( recv_cnt != nullptr ) - recv_cnt[0] = send_cnt[0]; - if ( recv_disp != nullptr ) - recv_disp[0] = send_disp[0]; - } - for ( int i = 0; i < send_cnt[0]; i++ ) - recv_data[i + recv_disp[0]] = send_data[i + send_disp[0]]; - N_recieved = send_cnt[0]; - } else if ( known_recv ) { - // The recieve sizes are known - MPI_CLASS_ASSERT( recv_cnt != nullptr && recv_disp != nullptr ); - call_allToAll( send_data, send_cnt, send_disp, recv_data, recv_cnt, recv_disp ); - for ( int i = 0; i < comm_size; i++ ) - N_recieved += recv_cnt[i]; - } else { - // The recieve sizes are not known, we need to communicate that information first - int *recv_cnt2 = recv_cnt; - int *recv_disp2 = recv_disp; - if ( recv_cnt == nullptr ) - recv_cnt2 = new int[comm_size]; - if ( recv_disp == nullptr ) - recv_disp2 = new int[comm_size]; - // Communicate the size we will be recieving from each processor - allToAll( 1, send_cnt, recv_cnt2 ); - recv_disp2[0] = 0; - for ( int i = 1; i < comm_size; i++ ) - recv_disp2[i] = recv_disp2[i - 1] + recv_cnt2[i - 1]; - // Send the data - call_allToAll( send_data, send_cnt, send_disp, recv_data, recv_cnt2, recv_disp2 ); - for ( int i = 0; i < comm_size; i++ ) - N_recieved += recv_cnt2[i]; - if ( recv_cnt == nullptr ) - delete[] recv_cnt2; - if ( recv_disp == nullptr ) - delete[] recv_disp2; - } - return N_recieved; -} -// Define specializations of call_allToAll -#if defined( USE_MPI ) || defined( USE_EXT_MPI ) -template<> -void MPI_CLASS::call_allToAll( const unsigned char *, const int *, const int *, - unsigned char *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( - const char *, const int *, const int *, char *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( const unsigned int *, const int *, const int *, - unsigned int *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( - const int *, const int *, const int *, int *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( const unsigned long int *, const int *, - const int *, unsigned long int *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( - const long int *, const int *, const int *, long int *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( - const float *, const int *, const int *, float *, const int *, const int * ) const; -template<> -void MPI_CLASS::call_allToAll( - const double *, const int *, const int *, double *, const int *, const int * ) const; -#else -template<> -void MPI_CLASS::call_allToAll( - const char *, const int *, const int *, char *, const int *, const int * ) const; -#endif -// Default instantiations of call_allToAll -template -void MPI_CLASS::call_allToAll( const TYPE *send_data, const int send_cnt[], const int send_disp[], - TYPE *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - int *send_cnt2 = new int[comm_size]; - int *recv_cnt2 = new int[comm_size]; - int *send_disp2 = new int[comm_size]; - int *recv_disp2 = new int[comm_size]; - for ( int i = 0; i < comm_size; i++ ) { - send_cnt2[i] = send_cnt[i] * sizeof( TYPE ); - send_disp2[i] = send_disp[i] * sizeof( TYPE ); - recv_cnt2[i] = recv_cnt[i] * sizeof( TYPE ); - recv_disp2[i] = recv_disp[i] * sizeof( TYPE ); - } - static_assert( is_mpi_copyable(), "Object is not trivially copyable" ); - call_allToAll( - (char *) send_data, send_cnt2, send_disp2, (char *) recv_data, recv_cnt2, recv_disp2 ); - delete[] send_cnt2; - delete[] recv_cnt2; - delete[] send_disp2; - delete[] recv_disp2; -} - - -} // namespace Utilities - -#endif diff --git a/common/MPI.cpp b/common/MPI.cpp deleted file mode 100644 index 73932d03..00000000 --- a/common/MPI.cpp +++ /dev/null @@ -1,3758 +0,0 @@ -// This file impliments a wrapper class for MPI functions - -#include "common/MPI.h" -#include "common/Utilities.h" - -#include "ProfilerApp.h" -#include "StackTrace/ErrorHandlers.h" -#include "StackTrace/StackTrace.h" - -// Include all other headers -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -// Include OS specific headers -#undef USE_WINDOWS -#undef USE_LINUX -#undef USE_MAC -#if defined( WIN32 ) || defined( _WIN32 ) || defined( WIN64 ) || defined( _WIN64 ) -// We are using windows -#define USE_WINDOWS -#include -#include -#define sched_yield() Sleep( 0 ) -#elif defined( __APPLE__ ) -// Using MAC -#define USE_MAC -#include -#elif defined( __linux ) || defined( __linux__ ) || defined( __unix ) || defined( __posix ) -// We are using linux -#define USE_LINUX -#include -#include -#else -#error Unknown OS -#endif - - -// Convience defines -#define MPI_ERROR ERROR -#define MPI_ASSERT ASSERT -#define MPI_INSIST INSIST -#define MPI_WARNING WARNING -#define MPI_CLASS_COMM_NULL MPI_COMM_NULL -#define MPI_CLASS_COMM_SELF MPI_COMM_SELF -#define MPI_CLASS_COMM_WORLD MPI_COMM_WORLD - - -// Global variable to track create new unique comms (dup and split) -#ifndef USE_MPI -MPI_Comm uniqueGlobalComm = 11; -#endif - - -#if defined( USE_SAMRAI ) && defined( USE_PETSC ) && !defined( USE_MPI ) -int MPI_REQUEST_NULL = 3; -int MPI_ERR_IN_STATUS = 4; -#endif - - -namespace Utilities { - - -// Some special structs to work with MPI -#ifdef USE_MPI -struct IntIntStruct { - int j; - int i; -}; -struct LongIntStruct { - long int j; - int i; -}; -struct FloatIntStruct { - float f; - int i; -}; -struct DoubleIntStruct { - double d; - int i; -}; -#endif - - -// Initialized the static member variables -volatile unsigned int MPI_CLASS::N_MPI_Comm_created = 0; -volatile unsigned int MPI_CLASS::N_MPI_Comm_destroyed = 0; -short MPI_CLASS::profile_level = 127; - - -// Define a type for use with size_t -#ifdef USE_MPI -static MPI_Datatype MPI_SIZE_T = 0x0; -static MPI_Datatype getSizeTDataType() -{ - int size_int, size_long, size_longlong, size_longlong2; - MPI_Type_size( MPI_UNSIGNED, &size_int ); - MPI_Type_size( MPI_UNSIGNED_LONG, &size_long ); - MPI_Type_size( MPI_UNSIGNED_LONG_LONG, &size_longlong ); - MPI_Type_size( MPI_LONG_LONG_INT, &size_longlong2 ); - if ( sizeof( size_t ) == size_int ) { - return MPI_UNSIGNED; - } else if ( sizeof( size_t ) == size_long ) { - return MPI_UNSIGNED_LONG; - } else if ( sizeof( size_t ) == size_longlong ) { - return MPI_UNSIGNED_LONG_LONG; - } else if ( sizeof( size_t ) == size_longlong2 ) { - MPI_WARNING( "Using signed long long datatype for size_t in MPI" ); - return MPI_LONG_LONG_INT; // Note: this is not unsigned - } else { - MPI_ERROR( "No suitable datatype found" ); - } - return 0; -} -#endif - - -// Static data for asyncronous communication without MPI -// Note: these routines may not be thread-safe yet -#ifndef USE_MPI -static const int mpi_max_tag = 0x003FFFFF; -struct Isendrecv_struct { - const char *data; // Pointer to data - int status; // Status: 1-sending, 2-recieving -}; -std::map global_isendrecv_list; -static MPI_Request getRequest( MPI_Comm comm, int tag ) -{ - MPI_ASSERT( tag >= 0 && tag <= mpi_max_tag ); - // Use hashing function: 2^64*0.5*(sqrt(5)-1) - uint64_t a = static_cast( comm ) * 0x9E3779B97F4A7C15; - uint64_t b = static_cast( tag ) * 0x9E3779B97F4A7C15; - uint64_t hash = a ^ b; - MPI_Request request; - memcpy( &request, &hash, sizeof( MPI_Request ) ); - return request; -} -#endif - - -// Check the mpi error code -#ifdef USE_MPI -inline void check_MPI( int error ) -{ - if ( error != MPI_SUCCESS ) - MPI_ERROR( "Error calling MPI routine" ); -} -#endif - - -/****************************************************************** - * Some helper functions to convert between signed/unsigned types * - ******************************************************************/ -DISABLE_WARNINGS -static inline constexpr unsigned int offset_int() -{ - return ~static_cast( std::numeric_limits::min() ) + 1; -} -static inline constexpr unsigned long int offset_long() -{ - return ~static_cast( std::numeric_limits::min() ) + 1; -} -static inline constexpr unsigned long long int offset_long_long() -{ - return ~static_cast( std::numeric_limits::min() ) + 1; -} -ENABLE_WARNINGS -static inline unsigned int signed_to_unsigned( int x ) -{ - const auto offset = offset_int(); - return ( x >= 0 ) ? static_cast( x ) + offset : - offset - static_cast( -x ); -} -static inline unsigned long int signed_to_unsigned( long int x ) -{ - const auto offset = offset_long(); - return ( x >= 0 ) ? static_cast( x ) + offset : - offset - static_cast( -x ); -} -static inline unsigned long long int signed_to_unsigned( long long int x ) -{ - const auto offset = offset_long_long(); - return ( x >= 0 ) ? static_cast( x ) + offset : - offset - static_cast( -x ); -} -static inline int unsigned_to_signed( unsigned int x ) -{ - const auto offset = offset_int(); - return ( x >= offset ) ? static_cast( x - offset ) : -static_cast( offset - x ); -} -static inline long int unsigned_to_signed( unsigned long int x ) -{ - const auto offset = offset_long(); - return ( x >= offset ) ? static_cast( x - offset ) : - -static_cast( offset - x ); -} -static inline long long int unsigned_to_signed( unsigned long long int x ) -{ - const auto offset = offset_long_long(); - return ( x >= offset ) ? static_cast( x - offset ) : - -static_cast( offset - x ); -} - - -/************************************************************************ - * Get the MPI version * - ************************************************************************/ -std::array MPI_CLASS::version() -{ -#ifdef USE_MPI - int MPI_version; - int MPI_subversion; - MPI_Get_version( &MPI_version, &MPI_subversion ); - return { MPI_version, MPI_subversion }; -#else - return { 0, 0 }; -#endif -} -std::string MPI_CLASS::info() -{ -#ifdef USE_MPI -#if MPI_VERSION >= 3 - int MPI_version_length = 0; - char MPI_version_string[MPI_MAX_LIBRARY_VERSION_STRING]; - MPI_Get_library_version( MPI_version_string, &MPI_version_length ); - if ( MPI_version_length > 0 ) { - std::string MPI_info( MPI_version_string, MPI_version_length ); - size_t pos = MPI_info.find( '\n' ); - while ( pos != std::string::npos ) { - MPI_info.insert( pos + 1, " " ); - pos = MPI_info.find( '\n', pos + 1 ); - } - return MPI_info; - } -#endif - auto tmp = version(); - return std::to_string( tmp[0] ) + "." + std::to_string( tmp[0] ); -#else - return std::string(); -#endif -} - - -/************************************************************************ - * Functions to get/set the process affinities * - ************************************************************************/ -int MPI_CLASS::getNumberOfProcessors() { return std::thread::hardware_concurrency(); } -std::vector MPI_CLASS::getProcessAffinity() -{ - std::vector procs; -#ifdef USE_LINUX - cpu_set_t mask; - int error = sched_getaffinity( getpid(), sizeof( cpu_set_t ), &mask ); - if ( error != 0 ) - MPI_ERROR( "Error getting process affinity" ); - for ( int i = 0; i < (int) sizeof( cpu_set_t ) * CHAR_BIT; i++ ) { - if ( CPU_ISSET( i, &mask ) ) - procs.push_back( i ); - } -#elif defined( USE_MAC ) - // MAC does not support getting or setting the affinity - printf( "Warning: MAC does not support getting the process affinity\n" ); - procs.clear(); -#elif defined( USE_WINDOWS ) - HANDLE hProc = GetCurrentProcess(); - size_t procMask; - size_t sysMask; - PDWORD_PTR procMaskPtr = reinterpret_cast( &procMask ); - PDWORD_PTR sysMaskPtr = reinterpret_cast( &sysMask ); - GetProcessAffinityMask( hProc, procMaskPtr, sysMaskPtr ); - for ( int i = 0; i < (int) sizeof( size_t ) * CHAR_BIT; i++ ) { - if ( ( procMask & 0x1 ) != 0 ) - procs.push_back( i ); - procMask >>= 1; - } -#else -#error Unknown OS -#endif - return procs; -} -void MPI_CLASS::setProcessAffinity( const std::vector &procs ) -{ -#ifdef USE_LINUX - cpu_set_t mask; - CPU_ZERO( &mask ); - for ( auto cpu : procs ) - CPU_SET( cpu, &mask ); - int error = sched_setaffinity( getpid(), sizeof( cpu_set_t ), &mask ); - if ( error != 0 ) - MPI_ERROR( "Error setting process affinity" ); -#elif defined( USE_MAC ) - // MAC does not support getting or setting the affinity - NULL_USE( procs ); -#elif defined( USE_WINDOWS ) - DWORD mask = 0; - for ( size_t i = 0; i < procs.size(); i++ ) - mask |= ( (DWORD) 1 ) << procs[i]; - HANDLE hProc = GetCurrentProcess(); - SetProcessAffinityMask( hProc, mask ); -#else -#error Unknown OS -#endif -} - - -/************************************************************************ - * Function to check if MPI is active * - ************************************************************************/ -bool MPI_CLASS::MPI_active() -{ -#ifdef USE_MPI - int initialized = 0, finalized = 0; - MPI_Initialized( &initialized ); - MPI_Finalized( &finalized ); - return initialized != 0 && finalized == 0; -#else - return true; -#endif -} -MPI_CLASS::ThreadSupport MPI_CLASS::queryThreadSupport() -{ -#ifdef USE_MPI - int provided = 0; - MPI_Query_thread( &provided ); - if ( provided == MPI_THREAD_SINGLE ) - return ThreadSupport::SINGLE; - if ( provided == MPI_THREAD_FUNNELED ) - return ThreadSupport::FUNNELED; - if ( provided == MPI_THREAD_SERIALIZED ) - return ThreadSupport::SERIALIZED; - if ( provided == MPI_THREAD_MULTIPLE ) - return ThreadSupport::MULTIPLE; - return ThreadSupport::SINGLE; -#else - return ThreadSupport::MULTIPLE; -#endif -} - - -/************************************************************************ - * Function to perform a load balance of the given processes * - ************************************************************************/ -void MPI_CLASS::balanceProcesses( const MPI_CLASS &globalComm, const int method, - const std::vector &procs, const int N_min_in, const int N_max_in ) -{ - // Build the list of processors to use - std::vector cpus = procs; - if ( cpus.empty() ) { - for ( int i = 0; i < getNumberOfProcessors(); i++ ) - cpus.push_back( i ); - } - // Handle the "easy cases" - if ( method == 1 ) { - // Trivial case where we do not need any communication - setProcessAffinity( cpus ); - return; - } - // Get the sub-communicator for the current node - MPI_CLASS nodeComm = globalComm.splitByNode(); - int N_min = std::min( std::max( N_min_in, 1 ), cpus.size() ); - int N_max = N_max_in; - if ( N_max == -1 ) - N_max = cpus.size(); - N_max = std::min( N_max, cpus.size() ); - MPI_ASSERT( N_max >= N_min ); - // Perform the load balance within the node - if ( method == 2 ) { - int N_proc = cpus.size() / nodeComm.getSize(); - N_proc = std::max( N_proc, N_min ); - N_proc = std::min( N_proc, N_max ); - std::vector cpus2( N_proc, -1 ); - for ( int i = 0; i < N_proc; i++ ) - cpus2[i] = cpus[( nodeComm.getRank() * N_proc + i ) % cpus.size()]; - setProcessAffinity( cpus2 ); - } else { - MPI_ERROR( "Unknown method for load balance" ); - } -} - - -/************************************************************************ - * Empty constructor * - ************************************************************************/ -MPI_CLASS::MPI_CLASS() -{ -// Initialize the data members to a defaul communicator of self -#ifdef USE_MPI - communicator = MPI_COMM_NULL; - d_maxTag = 0x7FFFFFFF; -#else - communicator = MPI_CLASS_COMM_NULL; - d_maxTag = mpi_max_tag; -#endif - d_ranks = nullptr; - d_count = nullptr; - d_manage = false; - comm_rank = 0; - comm_size = 1; - d_isNull = true; - d_currentTag = nullptr; - d_call_abort = true; - tmp_alignment = -1; -} - - -/************************************************************************ - * Empty deconstructor * - ************************************************************************/ -MPI_CLASS::~MPI_CLASS() { reset(); } -void MPI_CLASS::reset() -{ - // Decrement the count if used - int count = -1; - if ( d_count != nullptr ) - count = --( *d_count ); - if ( count == 0 ) { - // We are holding that last reference to the MPI_Comm object, we need to free it - if ( d_manage ) { -#ifdef USE_MPI - MPI_Comm_set_errhandler( communicator, MPI_ERRORS_ARE_FATAL ); - int err = MPI_Comm_free( &communicator ); - if ( err != MPI_SUCCESS ) - MPI_ERROR( "Problem free'ing MPI_Comm object" ); - communicator = MPI_CLASS_COMM_NULL; - ++N_MPI_Comm_destroyed; -#endif - } - if ( d_ranks != nullptr ) - delete[] d_ranks; - delete d_count; - } - if ( d_currentTag == nullptr ) { - // No tag index - } else if ( d_currentTag[1] > 1 ) { - --( d_currentTag[1] ); - } else { - delete[] d_currentTag; - } - d_manage = false; - d_count = nullptr; - d_ranks = nullptr; - comm_rank = 0; - comm_size = 1; - d_maxTag = 0; - d_isNull = true; - d_currentTag = nullptr; - d_call_abort = true; -} - - -/************************************************************************ - * Copy constructors * - ************************************************************************/ -MPI_CLASS::MPI_CLASS( const MPI_CLASS &comm ) - : communicator( comm.communicator ), - d_isNull( comm.d_isNull ), - d_manage( comm.d_manage ), - comm_rank( comm.comm_rank ), - comm_size( comm.comm_size ), - d_ranks( comm.d_ranks ), - d_maxTag( comm.d_maxTag ), - d_currentTag( comm.d_currentTag ) -{ - // Initialize the data members to the existing comm object - if ( d_currentTag != nullptr ) - ++d_currentTag[1]; - d_call_abort = comm.d_call_abort; - // Set and increment the count - d_count = comm.d_count; - if ( d_count != nullptr ) - ++( *d_count ); - tmp_alignment = -1; -} -MPI_CLASS::MPI_CLASS( MPI_CLASS &&rhs ) : MPI_CLASS() -{ - std::swap( communicator, rhs.communicator ); - std::swap( d_isNull, rhs.d_isNull ); - std::swap( d_manage, rhs.d_manage ); - std::swap( d_call_abort, rhs.d_call_abort ); - std::swap( profile_level, rhs.profile_level ); - std::swap( comm_rank, rhs.comm_rank ); - std::swap( comm_size, rhs.comm_size ); - std::swap( d_ranks, rhs.d_ranks ); - std::swap( d_maxTag, rhs.d_maxTag ); - std::swap( d_currentTag, rhs.d_currentTag ); - std::swap( d_count, rhs.d_count ); - std::swap( tmp_alignment, rhs.tmp_alignment ); -} - - -/************************************************************************ - * Assignment operators * - ************************************************************************/ -MPI_CLASS &MPI_CLASS::operator=( const MPI_CLASS &comm ) -{ - if ( this == &comm ) // protect against invalid self-assignment - return *this; - // Destroy the previous object - this->reset(); - // Initialize the data members to the existing object - this->communicator = comm.communicator; - this->comm_rank = comm.comm_rank; - this->comm_size = comm.comm_size; - this->d_ranks = comm.d_ranks; - this->d_isNull = comm.d_isNull; - this->d_manage = comm.d_manage; - this->d_maxTag = comm.d_maxTag; - this->d_call_abort = comm.d_call_abort; - this->d_currentTag = comm.d_currentTag; - if ( this->d_currentTag != nullptr ) - ++( this->d_currentTag[1] ); - // Set and increment the count - this->d_count = comm.d_count; - if ( this->d_count != nullptr ) - ++( *d_count ); - this->tmp_alignment = -1; - return *this; -} -MPI_CLASS &MPI_CLASS::operator=( MPI_CLASS &&rhs ) -{ - if ( this == &rhs ) // protect against invalid self-assignment - return *this; - std::swap( communicator, rhs.communicator ); - std::swap( d_isNull, rhs.d_isNull ); - std::swap( d_manage, rhs.d_manage ); - std::swap( d_call_abort, rhs.d_call_abort ); - std::swap( profile_level, rhs.profile_level ); - std::swap( comm_rank, rhs.comm_rank ); - std::swap( comm_size, rhs.comm_size ); - std::swap( d_ranks, rhs.d_ranks ); - std::swap( d_maxTag, rhs.d_maxTag ); - std::swap( d_currentTag, rhs.d_currentTag ); - std::swap( d_count, rhs.d_count ); - std::swap( tmp_alignment, rhs.tmp_alignment ); - return *this; -} - - -/************************************************************************ - * Constructor from existing MPI communicator * - ************************************************************************/ -int d_global_currentTag_world1[2] = { 1, 1 }; -int d_global_currentTag_world2[2] = { 1, 1 }; -int d_global_currentTag_self[2] = { 1, 1 }; -#ifdef USE_MPI -std::atomic_int d_global_count_world1 = { 1 }; -std::atomic_int d_global_count_world2 = { 1 }; -std::atomic_int d_global_count_self = { 1 }; -#endif -MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage ) -{ - d_count = nullptr; - d_ranks = nullptr; - d_manage = false; - tmp_alignment = -1; - // Check if we are using our version of comm_world - if ( comm == MPI_CLASS_COMM_WORLD ) { - communicator = MPI_COMM_WORLD; - } else if ( comm == MPI_CLASS_COMM_SELF ) { - communicator = MPI_COMM_SELF; - } else if ( comm == MPI_CLASS_COMM_NULL ) { - communicator = MPI_COMM_NULL; - } else { - communicator = comm; - } -#ifdef USE_MPI - // We are using MPI, use the MPI communicator to initialize the data - if ( communicator != MPI_COMM_NULL ) { - // Set the MPI_SIZE_T datatype if it has not been set - if ( MPI_SIZE_T == 0x0 ) - MPI_SIZE_T = getSizeTDataType(); - // Attach the error handler - StackTrace::setMPIErrorHandler( communicator ); - // Get the communicator properties - MPI_Comm_rank( communicator, &comm_rank ); - MPI_Comm_size( communicator, &comm_size ); - int flag, *val; - int ierr = MPI_Comm_get_attr( communicator, MPI_TAG_UB, &val, &flag ); - MPI_ASSERT( ierr == MPI_SUCCESS ); - if ( flag == 0 ) { - d_maxTag = 0x7FFFFFFF; // The tag is not a valid attribute (set to 2^31-1) - } else { - d_maxTag = *val; - if ( d_maxTag < 0 ) { - d_maxTag = 0x7FFFFFFF; - } // The maximum tag is > a signed int (set to 2^31-1) - MPI_INSIST( d_maxTag >= 0x7FFF, "maximum tag size is < MPI standard" ); - } - } else { - comm_rank = 1; - comm_size = 0; - d_maxTag = 0x7FFFFFFF; - } - d_isNull = communicator == MPI_COMM_NULL; - if ( manage && communicator != MPI_COMM_NULL && communicator != MPI_COMM_SELF && - communicator != MPI_COMM_WORLD ) - d_manage = true; - // Create the count (Note: we do not need to worry about thread safety) - if ( communicator == MPI_CLASS_COMM_WORLD ) { - d_count = &d_global_count_world1; - ++( *d_count ); - } else if ( communicator == MPI_COMM_WORLD ) { - d_count = &d_global_count_world2; - ++( *d_count ); - } else if ( communicator == MPI_COMM_SELF ) { - d_count = &d_global_count_self; - ++( *d_count ); - } else if ( communicator == MPI_COMM_NULL ) { - d_count = nullptr; - } else { - d_count = new std::atomic_int; - *d_count = 1; - } - if ( d_manage ) - ++N_MPI_Comm_created; - // Create d_ranks - if ( comm_size > 1 ) { - d_ranks = new int[comm_size]; - d_ranks[0] = -1; - } -#else - // We are not using MPI, intialize based on the communicator - NULL_USE( manage ); - comm_rank = 0; - comm_size = 1; - d_maxTag = mpi_max_tag; - d_isNull = communicator == MPI_COMM_NULL; - if ( d_isNull ) - comm_size = 0; -#endif - if ( communicator == MPI_CLASS_COMM_WORLD ) { - d_currentTag = d_global_currentTag_world1; - ++( this->d_currentTag[1] ); - } else if ( communicator == MPI_COMM_WORLD ) { - d_currentTag = d_global_currentTag_world2; - ++( this->d_currentTag[1] ); - } else if ( communicator == MPI_COMM_SELF ) { - d_currentTag = d_global_currentTag_self; - ++( this->d_currentTag[1] ); - } else if ( communicator == MPI_COMM_NULL ) { - d_currentTag = nullptr; - } else { - d_currentTag = new int[2]; - d_currentTag[0] = ( d_maxTag <= 0x10000 ) ? 1 : 0x1FFF; - d_currentTag[1] = 1; - } - d_call_abort = true; -} - - -/************************************************************************ - * Return the ranks of the communicator in the global comm * - ************************************************************************/ -std::vector MPI_CLASS::globalRanks() const -{ - // Get my global rank if it has not been set - static int myGlobalRank = -1; - if ( myGlobalRank == -1 ) { -#ifdef USE_MPI - if ( MPI_active() ) - MPI_Comm_rank( MPI_CLASS_COMM_WORLD, &myGlobalRank ); -#else - myGlobalRank = 0; -#endif - } - // Check if we are dealing with a serial or null communicator - if ( comm_size == 1 ) - return std::vector( 1, myGlobalRank ); - if ( d_ranks == nullptr || communicator == MPI_COMM_NULL ) - return std::vector(); - // Fill d_ranks if necessary - if ( d_ranks[0] == -1 ) { - if ( communicator == MPI_CLASS_COMM_WORLD ) { - for ( int i = 0; i < comm_size; i++ ) - d_ranks[i] = i; - } else { - - MPI_ASSERT( myGlobalRank != -1 ); - this->allGather( myGlobalRank, d_ranks ); - } - } - // Return d_ranks - return std::vector( d_ranks, d_ranks + comm_size ); -} - - -/************************************************************************ - * Generate a random number * - ************************************************************************/ -size_t MPI_CLASS::rand() const -{ - size_t val = 0; - if ( getRank() == 0 ) { - static std::random_device rd; - static std::mt19937 gen( rd() ); - static std::uniform_int_distribution dist; - val = dist( gen ); - } - val = bcast( val, 0 ); - return val; -} - - -/************************************************************************ - * Intersect two communicators * - ************************************************************************/ -#ifdef USE_MPI -static inline void MPI_Group_free2( MPI_Group *group ) -{ - if ( *group != MPI_GROUP_EMPTY ) { - // MPICH is fine with free'ing an empty group, OpenMPI crashes - MPI_Group_free( group ); - } -} -MPI_CLASS MPI_CLASS::intersect( const MPI_CLASS &comm1, const MPI_CLASS &comm2 ) -{ - MPI_Group group1 = MPI_GROUP_EMPTY, group2 = MPI_GROUP_EMPTY; - if ( !comm1.isNull() ) { - MPI_Group_free2( &group1 ); - MPI_Comm_group( comm1.communicator, &group1 ); - } - if ( !comm2.isNull() ) { - MPI_Group_free2( &group2 ); - MPI_Comm_group( comm2.communicator, &group2 ); - } - MPI_Group group12; - MPI_Group_intersection( group1, group2, &group12 ); - int compare1, compare2; - MPI_Group_compare( group1, group12, &compare1 ); - MPI_Group_compare( group2, group12, &compare2 ); - MPI_CLASS new_comm( MPI_CLASS_COMM_NULL ); - int size; - MPI_Group_size( group12, &size ); - if ( compare1 != MPI_UNEQUAL && size != 0 ) { - // The intersection matches comm1 - new_comm = comm1; - } else if ( compare2 != MPI_UNEQUAL && size != 0 ) { - // The intersection matches comm2 - new_comm = comm2; - } else if ( comm1.isNull() ) { - // comm1 is null, we can return safely (comm1 is needed for communication) - } else { - // The intersection is smaller than comm1 or comm2 - // Check if the new comm is nullptr for all processors - int max_size = 0; - MPI_Allreduce( &size, &max_size, 1, MPI_INT, MPI_MAX, comm1.communicator ); - if ( max_size == 0 ) { - // We are dealing with completely disjoint sets - new_comm = MPI_CLASS( MPI_CLASS_COMM_NULL, false ); - } else { - // Create the new comm - // Note: OpenMPI crashes if the intersection group is EMPTY for any processors - // We will set it to SELF for the EMPTY processors, then create a nullptr comm later - if ( group12 == MPI_GROUP_EMPTY ) { - MPI_Group_free2( &group12 ); - MPI_Comm_group( MPI_COMM_SELF, &group12 ); - } - MPI_Comm new_MPI_comm; - MPI_Comm_create( comm1.communicator, group12, &new_MPI_comm ); - if ( size > 0 ) { - // This is the valid case where we create a new intersection comm - new_comm = MPI_CLASS( new_MPI_comm, true ); - } else { - // We actually want a null comm for this communicator - new_comm = MPI_CLASS( MPI_CLASS_COMM_NULL, false ); - MPI_Comm_free( &new_MPI_comm ); - } - } - } - MPI_Group_free2( &group1 ); - MPI_Group_free2( &group2 ); - MPI_Group_free2( &group12 ); - return new_comm; -} -#else -MPI_CLASS MPI_CLASS::intersect( const MPI_CLASS &comm1, const MPI_CLASS &comm2 ) -{ - if ( comm1.isNull() || comm2.isNull() ) - return MPI_CLASS( MPI_CLASS_COMM_NULL, false ); - MPI_ASSERT( comm1.comm_size == 1 && comm2.comm_size == 1 ); - return comm1; -} -#endif - - -/************************************************************************ - * Split a comm * - ************************************************************************/ -MPI_CLASS MPI_CLASS::split( int color, int key ) const -{ - if ( d_isNull ) { - return MPI_CLASS( MPI_CLASS_COMM_NULL ); - } else if ( comm_size == 1 ) { - if ( color == -1 ) - return MPI_CLASS( MPI_CLASS_COMM_NULL ); - return dup(); - } - MPI_Comm new_MPI_comm = MPI_CLASS_COMM_NULL; -#ifdef USE_MPI - // USE MPI to split the communicator - if ( color == -1 ) { - check_MPI( MPI_Comm_split( communicator, MPI_UNDEFINED, key, &new_MPI_comm ) ); - } else { - check_MPI( MPI_Comm_split( communicator, color, key, &new_MPI_comm ) ); - } -#endif - // Create the new object - NULL_USE( key ); - MPI_CLASS new_comm( new_MPI_comm, true ); - new_comm.d_call_abort = d_call_abort; - return new_comm; -} -MPI_CLASS MPI_CLASS::splitByNode( int key ) const -{ - // Check if we are dealing with a single processor (trivial case) - if ( comm_size == 1 ) - return this->split( 0, 0 ); - // Get the node name - std::string name = MPI_CLASS::getNodeName(); - // Gather the names from all ranks - std::vector list( comm_size ); - allGather( name, &list[0] ); - // Create the colors - std::vector color( comm_size, -1 ); - color[0] = 0; - for ( int i = 1; i < comm_size; i++ ) { - const std::string tmp1 = list[i]; - for ( int j = 0; j < i; j++ ) { - const std::string tmp2 = list[j]; - if ( tmp1 == tmp2 ) { - color[i] = color[j]; - break; - } - color[i] = color[i - 1] + 1; - } - } - MPI_CLASS new_comm = this->split( color[comm_rank], key ); - return new_comm; -} - - -/************************************************************************ - * Duplicate an exisiting comm object * - ************************************************************************/ -MPI_CLASS MPI_CLASS::dup() const -{ - if ( d_isNull ) - return MPI_CLASS( MPI_CLASS_COMM_NULL ); - MPI_Comm new_MPI_comm = communicator; -#if defined( USE_MPI ) || defined( USE_PETSC ) - // USE MPI to duplicate the communicator - MPI_Comm_dup( communicator, &new_MPI_comm ); -#else - new_MPI_comm = uniqueGlobalComm; - uniqueGlobalComm++; -#endif - // Create the new comm object - MPI_CLASS new_comm( new_MPI_comm, true ); - new_comm.d_isNull = d_isNull; - new_comm.d_call_abort = d_call_abort; - return new_comm; -} - - -/************************************************************************ - * Get the node name * - ************************************************************************/ -std::string MPI_CLASS::getNodeName() -{ -#ifdef USE_MPI - int length; - char name[MPI_MAX_PROCESSOR_NAME + 1]; - memset( name, 0, MPI_MAX_PROCESSOR_NAME + 1 ); - MPI_Get_processor_name( name, &length ); - return std::string( name ); -#else - return "Node0"; -#endif -} - - -/************************************************************************ - * Overload operator == * - ************************************************************************/ -bool MPI_CLASS::operator==( const MPI_CLASS &comm ) const -{ - return communicator == comm.communicator; -} - - -/************************************************************************ - * Overload operator != * - ************************************************************************/ -bool MPI_CLASS::operator!=( const MPI_CLASS &comm ) const -{ - return communicator != comm.communicator; -} - - -/************************************************************************ - * Overload operator < * - ************************************************************************/ -bool MPI_CLASS::operator<( const MPI_CLASS &comm ) const -{ - MPI_ASSERT( !this->d_isNull && !comm.d_isNull ); - bool flag = true; - // First check if either communicator is NULL - if ( this->d_isNull ) - return false; - if ( comm.d_isNull ) - flag = false; - // Use compare to check if the comms are equal - if ( compare( comm ) != 0 ) - return false; - // Check that the size of the other communicator is > the current communicator size - if ( comm_size >= comm.comm_size ) - flag = false; -// Check the union of the communicator groups -// this is < comm iff this group is a subgroup of comm's group -#ifdef USE_MPI - MPI_Group group1 = MPI_GROUP_EMPTY, group2 = MPI_GROUP_EMPTY, group12 = MPI_GROUP_EMPTY; - if ( !d_isNull ) - MPI_Comm_group( communicator, &group1 ); - if ( !comm.d_isNull ) - MPI_Comm_group( comm.communicator, &group2 ); - MPI_Group_union( group1, group2, &group12 ); - int compare; - MPI_Group_compare( group2, group12, &compare ); - if ( compare == MPI_UNEQUAL ) - flag = false; - MPI_Group_free( &group1 ); - MPI_Group_free( &group2 ); - MPI_Group_free( &group12 ); -#endif - // Perform a global reduce of the flag (equivalent to all operation) - return allReduce( flag ); -} - - -/************************************************************************ - * Overload operator <= * - ************************************************************************/ -bool MPI_CLASS::operator<=( const MPI_CLASS &comm ) const -{ - MPI_ASSERT( !this->d_isNull && !comm.d_isNull ); - bool flag = true; - // First check if either communicator is NULL - if ( this->d_isNull ) - return false; - if ( comm.d_isNull ) - flag = false; -#ifdef USE_MPI - int world_size = 0; - MPI_Comm_size( MPI_COMM_WORLD, &world_size ); - if ( comm.getSize() == world_size ) - return true; - if ( getSize() == 1 && !comm.d_isNull ) - return true; -#endif - // Use compare to check if the comms are equal - if ( compare( comm ) != 0 ) - return true; - // Check that the size of the other communicator is > the current communicator size - // this is <= comm iff this group is a subgroup of comm's group - if ( comm_size > comm.comm_size ) - flag = false; -// Check the unnion of the communicator groups -#ifdef USE_MPI - MPI_Group group1, group2, group12; - MPI_Comm_group( communicator, &group1 ); - MPI_Comm_group( comm.communicator, &group2 ); - MPI_Group_union( group1, group2, &group12 ); - int compare; - MPI_Group_compare( group2, group12, &compare ); - if ( compare == MPI_UNEQUAL ) - flag = false; - MPI_Group_free( &group1 ); - MPI_Group_free( &group2 ); - MPI_Group_free( &group12 ); -#endif - // Perform a global reduce of the flag (equivalent to all operation) - return allReduce( flag ); -} - - -/************************************************************************ - * Overload operator > * - ************************************************************************/ -bool MPI_CLASS::operator>( const MPI_CLASS &comm ) const -{ - bool flag = true; - // First check if either communicator is NULL - if ( this->d_isNull ) - return false; - if ( comm.d_isNull ) - flag = false; - // Use compare to check if the comms are equal - if ( compare( comm ) != 0 ) - return false; - // Check that the size of the other communicator is > the current communicator size - if ( comm_size <= comm.comm_size ) - flag = false; -// Check the unnion of the communicator groups -// this is > comm iff comm's group is a subgroup of this group -#ifdef USE_MPI - MPI_Group group1 = MPI_GROUP_EMPTY, group2 = MPI_GROUP_EMPTY, group12 = MPI_GROUP_EMPTY; - if ( !d_isNull ) - MPI_Comm_group( communicator, &group1 ); - if ( !comm.d_isNull ) - MPI_Comm_group( comm.communicator, &group2 ); - MPI_Group_union( group1, group2, &group12 ); - int compare; - MPI_Group_compare( group1, group12, &compare ); - if ( compare == MPI_UNEQUAL ) - flag = false; - MPI_Group_free( &group1 ); - MPI_Group_free( &group2 ); - MPI_Group_free( &group12 ); -#endif - // Perform a global reduce of the flag (equivalent to all operation) - return allReduce( flag ); -} - - -/************************************************************************ - * Overload operator >= * - ************************************************************************/ -bool MPI_CLASS::operator>=( const MPI_CLASS &comm ) const -{ - bool flag = true; - // First check if either communicator is NULL - if ( this->d_isNull ) - return false; - if ( comm.d_isNull ) - flag = false; -#ifdef USE_MPI - int world_size = 0; - MPI_Comm_size( MPI_COMM_WORLD, &world_size ); - if ( getSize() == world_size ) - return true; - if ( comm.getSize() == 1 && !comm.d_isNull ) - return true; -#endif - // Use compare to check if the comms are equal - if ( compare( comm ) != 0 ) - return true; - // Check that the size of the other communicator is > the current communicator size - if ( comm_size < comm.comm_size ) - flag = false; -// Check the unnion of the communicator groups -// this is >= comm iff comm's group is a subgroup of this group -#ifdef USE_MPI - MPI_Group group1 = MPI_GROUP_EMPTY, group2 = MPI_GROUP_EMPTY, group12 = MPI_GROUP_EMPTY; - if ( !d_isNull ) - MPI_Comm_group( communicator, &group1 ); - if ( !comm.d_isNull ) - MPI_Comm_group( comm.communicator, &group2 ); - MPI_Group_union( group1, group2, &group12 ); - int compare; - MPI_Group_compare( group1, group12, &compare ); - if ( compare == MPI_UNEQUAL ) - flag = false; - MPI_Group_free( &group1 ); - MPI_Group_free( &group2 ); - MPI_Group_free( &group12 ); -#endif - // Perform a global reduce of the flag (equivalent to all operation) - return allReduce( flag ); -} - - -/************************************************************************ - * Compare two comm objects * - ************************************************************************/ -int MPI_CLASS::compare( const MPI_CLASS &comm ) const -{ - if ( communicator == comm.communicator ) - return 1; -#ifdef USE_MPI - if ( d_isNull || comm.d_isNull ) - return 0; - int result; - check_MPI( MPI_Comm_compare( communicator, comm.communicator, &result ) ); - if ( result == MPI_IDENT ) - return 2; - else if ( result == MPI_CONGRUENT ) - return 3; - else if ( result == MPI_SIMILAR ) - return 4; - else if ( result == MPI_UNEQUAL ) - return 0; - MPI_ERROR( "Unknown results from comm compare" ); -#else - if ( comm.communicator == MPI_COMM_NULL || communicator == MPI_COMM_NULL ) - return 0; - else - return 3; -#endif - return 0; -} - - -/************************************************************************ - * Abort the program. * - ************************************************************************/ -void MPI_CLASS::setCallAbortInSerialInsteadOfExit( bool flag ) { d_call_abort = flag; } -void MPI_CLASS::abort() const -{ -#ifdef USE_MPI - MPI_Comm comm = communicator; - if ( comm == MPI_COMM_NULL ) - comm = MPI_COMM_WORLD; - if ( !MPI_active() ) { - // MPI is not availible - exit( -1 ); - } else if ( comm_size > 1 ) { - MPI_Abort( comm, -1 ); - } else if ( d_call_abort ) { - MPI_Abort( comm, -1 ); - } else { - exit( -1 ); - } -#else - exit( -1 ); -#endif -} - - -/************************************************************************ - * newTag * - ************************************************************************/ -int MPI_CLASS::newTag() -{ -#ifdef USE_MPI - // Syncronize the processes to ensure all ranks enter this call - // Needed so the count will match - barrier(); - // Return and increment the tag - int tag = ( *d_currentTag )++; - MPI_INSIST( tag <= d_maxTag, "Maximum number of tags exceeded\n" ); - return tag; -#else - static int globalCurrentTag = 1; - return globalCurrentTag++; -#endif -} - - -/************************************************************************ - * allReduce * - ************************************************************************/ -bool MPI_CLASS::allReduce( const bool value ) const -{ - bool ret = value; - if ( comm_size > 1 ) { -#ifdef USE_MPI - MPI_Allreduce( - (void *) &value, (void *) &ret, 1, MPI_UNSIGNED_CHAR, MPI_MIN, communicator ); -#else - MPI_ERROR( "This shouldn't be possible" ); -#endif - } - return ret; -} - - -/************************************************************************ - * anyReduce * - ************************************************************************/ -bool MPI_CLASS::anyReduce( const bool value ) const -{ - bool ret = value; - if ( comm_size > 1 ) { -#ifdef USE_MPI - MPI_Allreduce( - (void *) &value, (void *) &ret, 1, MPI_UNSIGNED_CHAR, MPI_MAX, communicator ); -#else - MPI_ERROR( "This shouldn't be possible" ); -#endif - } - return ret; -} - - -/************************************************************************ - * call_sumReduce * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_sumReduce( - const unsigned char *send, unsigned char *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( unsigned char *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new unsigned char[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_CHAR, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// char -template<> -void MPI_CLASS::call_sumReduce( const char *send, char *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( char *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new char[n]; - MPI_Allreduce( send, recv, n, MPI_SIGNED_CHAR, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_sumReduce( - const unsigned int *send, unsigned int *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( unsigned int *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new unsigned int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// int -template<> -void MPI_CLASS::call_sumReduce( const int *send, int *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_INT, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( int *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new int[n]; - MPI_Allreduce( send, recv, n, MPI_INT, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_sumReduce( const long int *send, long int *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_LONG, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( long int *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new long int[n]; - MPI_Allreduce( send, recv, n, MPI_LONG, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_sumReduce( - const unsigned long *send, unsigned long *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( unsigned long *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new unsigned long int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_LONG, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// size_t -#ifdef USE_WINDOWS -template<> -void MPI_CLASS::call_sumReduce( const size_t *send, size_t *recv, const int n ) const -{ - MPI_ASSERT( MPI_SIZE_T != 0 ); - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_SIZE_T, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( size_t *x, const int n ) const -{ - MPI_ASSERT( MPI_SIZE_T != 0 ); - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new size_t[n]; - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_SIZE_T, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -#endif -// float -template<> -void MPI_CLASS::call_sumReduce( const float *send, float *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_FLOAT, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( float *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new float[n]; - MPI_Allreduce( send, recv, n, MPI_FLOAT, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// double -template<> -void MPI_CLASS::call_sumReduce( const double *send, double *recv, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_DOUBLE, MPI_SUM, communicator ); - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce( double *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = x; - auto recv = new double[n]; - MPI_Allreduce( send, recv, n, MPI_DOUBLE, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -// std::complex -template<> -void MPI_CLASS::call_sumReduce>( - const std::complex *x, std::complex *y, const int n ) const -{ - PROFILE_START( "sumReduce1", profile_level ); - auto send = new double[2 * n]; - auto recv = new double[2 * n]; - for ( int i = 0; i < n; i++ ) { - send[2 * i + 0] = real( x[i] ); - send[2 * i + 1] = imag( x[i] ); - } - MPI_Allreduce( (void *) send, (void *) recv, 2 * n, MPI_DOUBLE, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - y[i] = std::complex( recv[2 * i + 0], recv[2 * i + 1] ); - delete[] send; - delete[] recv; - PROFILE_STOP( "sumReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_sumReduce>( std::complex *x, const int n ) const -{ - PROFILE_START( "sumReduce2", profile_level ); - auto send = new double[2 * n]; - auto recv = new double[2 * n]; - for ( int i = 0; i < n; i++ ) { - send[2 * i + 0] = real( x[i] ); - send[2 * i + 1] = imag( x[i] ); - } - MPI_Allreduce( send, recv, 2 * n, MPI_DOUBLE, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = std::complex( recv[2 * i + 0], recv[2 * i + 1] ); - delete[] send; - delete[] recv; - PROFILE_STOP( "sumReduce2", profile_level ); -} -#endif - - -/************************************************************************ - * call_minReduce * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_minReduce( - const unsigned char *send, unsigned char *recv, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_MIN, communicator ); - PROFILE_STOP( "minReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = send[i]; - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_minReduce( - unsigned char *x, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce2", profile_level ); - auto send = x; - auto recv = new unsigned char[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_CHAR, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "minReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = x[i]; - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -// char -template<> -void MPI_CLASS::call_minReduce( - const char *send, char *recv, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_MIN, communicator ); - PROFILE_STOP( "minReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = send[i]; - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_minReduce( char *x, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce2", profile_level ); - auto send = x; - auto recv = new char[n]; - MPI_Allreduce( send, recv, n, MPI_SIGNED_CHAR, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "minReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = x[i]; - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -// unsigned int -template<> -void MPI_CLASS::call_minReduce( - const unsigned int *send, unsigned int *recv, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_MIN, communicator ); - PROFILE_STOP( "minReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( send[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_minReduce( - unsigned int *x, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce2", profile_level ); - auto send = x; - auto recv = new unsigned int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "minReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( x[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -// int -template<> -void MPI_CLASS::call_minReduce( - const int *x, int *y, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_INT, MPI_MIN, communicator ); - } else { - auto recv = new IntIntStruct[n]; - auto send = new IntIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_2INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].j; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( int *x, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce2", profile_level ); - if ( comm_rank_of_min == nullptr ) { - auto send = x; - auto recv = new int[n]; - MPI_Allreduce( send, recv, n, MPI_INT, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new IntIntStruct[n]; - auto send = new IntIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_2INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].j; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce2", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_minReduce( const unsigned long int *send, - unsigned long int *recv, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_MIN, communicator ); - PROFILE_STOP( "minReduce1", profile_level ); - } else { - auto tmp = new long int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( send[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_minReduce( - unsigned long int *x, const int n, int *comm_rank_of_min ) const -{ - if ( comm_rank_of_min == nullptr ) { - PROFILE_START( "minReduce2", profile_level ); - auto send = x; - auto recv = new unsigned long int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_LONG, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "minReduce2", profile_level ); - } else { - auto tmp = new long int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( x[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -// long int -template<> -void MPI_CLASS::call_minReduce( - const long int *x, long int *y, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG, MPI_MIN, communicator ); - } else { - auto recv = new LongIntStruct[n]; - auto send = new LongIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_LONG_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].j; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( long int *x, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce2", profile_level ); - if ( comm_rank_of_min == nullptr ) { - auto send = x; - auto recv = new long int[n]; - MPI_Allreduce( send, recv, n, MPI_LONG, MPI_MIN, communicator ); - for ( long int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new LongIntStruct[n]; - auto send = new LongIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_LONG_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].j; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce2", profile_level ); -} -// unsigned long long int -template<> -void MPI_CLASS::call_minReduce( const unsigned long long int *send, - unsigned long long int *recv, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - auto x = new long long int[n]; - auto y = new long long int[n]; - for ( int i = 0; i < n; i++ ) - x[i] = unsigned_to_signed( send[i] ); - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG_LONG_INT, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( y[i] ); - delete[] x; - delete[] y; - } else { - printf( "minReduce will use double\n" ); - auto tmp = new double[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = static_cast( send[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( - unsigned long long int *x, const int n, int *comm_rank_of_min ) const -{ - auto recv = new unsigned long long int[n]; - call_minReduce( x, recv, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; -} -// long long int -template<> -void MPI_CLASS::call_minReduce( - const long long int *x, long long int *y, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG_LONG_INT, MPI_MIN, communicator ); - } else { - printf( "minReduce will use double\n" ); - auto tmp = new double[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = static_cast( x[i] ); - call_minReduce( tmp, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - y[i] = static_cast( tmp[i] ); - delete[] tmp; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( - long long int *x, const int n, int *comm_rank_of_min ) const -{ - auto recv = new long long int[n]; - call_minReduce( x, recv, n, comm_rank_of_min ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( recv[i] ); - delete[] recv; -} -// float -template<> -void MPI_CLASS::call_minReduce( - const float *x, float *y, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_INT, MPI_MIN, communicator ); - } else { - auto recv = new FloatIntStruct[n]; - auto send = new FloatIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].f = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_FLOAT_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].f; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( float *x, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce2", profile_level ); - if ( comm_rank_of_min == nullptr ) { - auto send = x; - auto recv = new float[n]; - MPI_Allreduce( send, recv, n, MPI_FLOAT, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new FloatIntStruct[n]; - auto send = new FloatIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].f = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_FLOAT_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].f; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce2", profile_level ); -} -// double -template<> -void MPI_CLASS::call_minReduce( - const double *x, double *y, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce1", profile_level ); - if ( comm_rank_of_min == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_DOUBLE, MPI_MIN, communicator ); - } else { - auto recv = new DoubleIntStruct[n]; - auto send = new DoubleIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].d = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_DOUBLE_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].d; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_minReduce( double *x, const int n, int *comm_rank_of_min ) const -{ - PROFILE_START( "minReduce2", profile_level ); - if ( comm_rank_of_min == nullptr ) { - auto send = x; - auto recv = new double[n]; - MPI_Allreduce( send, recv, n, MPI_DOUBLE, MPI_MIN, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new DoubleIntStruct[n]; - auto send = new DoubleIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].d = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_DOUBLE_INT, MPI_MINLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].d; - comm_rank_of_min[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "minReduce2", profile_level ); -} -#endif - - -/************************************************************************ - * call_maxReduce * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_maxReduce( - const unsigned char *send, unsigned char *recv, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_MAX, communicator ); - PROFILE_STOP( "maxReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = send[i]; - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_maxReduce( - unsigned char *x, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce2", profile_level ); - auto send = x; - auto recv = new unsigned char[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_CHAR, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "maxReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = x[i]; - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -// char -template<> -void MPI_CLASS::call_maxReduce( - const char *send, char *recv, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_MAX, communicator ); - PROFILE_STOP( "maxReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = send[i]; - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_maxReduce( char *x, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce2", profile_level ); - auto send = x; - auto recv = new char[n]; - MPI_Allreduce( send, recv, n, MPI_SIGNED_CHAR, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "maxReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = x[i]; - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = static_cast( tmp[i] ); - delete[] tmp; - } -} -// unsigned int -template<> -void MPI_CLASS::call_maxReduce( - const unsigned int *send, unsigned int *recv, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_MAX, communicator ); - PROFILE_STOP( "maxReduce1", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( send[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_maxReduce( - unsigned int *x, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce2", profile_level ); - auto send = x; - auto recv = new unsigned int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "maxReduce2", profile_level ); - } else { - auto tmp = new int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( x[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -// int -template<> -void MPI_CLASS::call_maxReduce( - const int *x, int *y, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_INT, MPI_MAX, communicator ); - } else { - auto recv = new IntIntStruct[n]; - auto send = new IntIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_2INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].j; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( int *x, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce2", profile_level ); - if ( comm_rank_of_max == nullptr ) { - int *send = x; - auto recv = new int[n]; - MPI_Allreduce( send, recv, n, MPI_INT, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new IntIntStruct[n]; - auto send = new IntIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_2INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].j; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce2", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_maxReduce( - const long int *x, long int *y, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG, MPI_MAX, communicator ); - } else { - auto recv = new LongIntStruct[n]; - auto send = new LongIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_LONG_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].j; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( long int *x, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce2", profile_level ); - if ( comm_rank_of_max == nullptr ) { - auto send = x; - auto recv = new long int[n]; - MPI_Allreduce( send, recv, n, MPI_LONG, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new LongIntStruct[n]; - auto send = new LongIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].j = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_LONG_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].j; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce2", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_maxReduce( const unsigned long int *send, - unsigned long int *recv, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce1", profile_level ); - MPI_Allreduce( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_MAX, communicator ); - PROFILE_STOP( "maxReduce1", profile_level ); - } else { - auto tmp = new long int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( send[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -template<> -void MPI_CLASS::call_maxReduce( - unsigned long int *x, const int n, int *comm_rank_of_max ) const -{ - if ( comm_rank_of_max == nullptr ) { - PROFILE_START( "maxReduce2", profile_level ); - auto send = x; - auto recv = new unsigned long int[n]; - MPI_Allreduce( send, recv, n, MPI_UNSIGNED_LONG, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - PROFILE_STOP( "maxReduce2", profile_level ); - } else { - auto tmp = new long int[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = unsigned_to_signed( x[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( tmp[i] ); - delete[] tmp; - } -} -// unsigned long long int -template<> -void MPI_CLASS::call_maxReduce( const unsigned long long int *send, - unsigned long long int *recv, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - auto x = new long long int[n]; - auto y = new long long int[n]; - for ( int i = 0; i < n; i++ ) - x[i] = unsigned_to_signed( send[i] ); - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG_LONG_INT, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - recv[i] = signed_to_unsigned( y[i] ); - delete[] x; - delete[] y; - } else { - printf( "maxReduce will use double\n" ); - auto tmp = new double[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = static_cast( send[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - recv[i] = static_cast( tmp[i] ); - delete[] tmp; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( - unsigned long long int *x, const int n, int *comm_rank_of_max ) const -{ - auto recv = new unsigned long long int[n]; - call_maxReduce( x, recv, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; -} -// long long int -template<> -void MPI_CLASS::call_maxReduce( - const long long int *x, long long int *y, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_LONG_LONG_INT, MPI_MAX, communicator ); - } else { - printf( "maxReduce will use double\n" ); - auto tmp = new double[n]; - for ( int i = 0; i < n; i++ ) - tmp[i] = static_cast( x[i] ); - call_maxReduce( tmp, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - y[i] = static_cast( tmp[i] ); - delete[] tmp; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( - long long int *x, const int n, int *comm_rank_of_max ) const -{ - auto recv = new long long int[n]; - call_maxReduce( x, recv, n, comm_rank_of_max ); - for ( int i = 0; i < n; i++ ) - x[i] = signed_to_unsigned( recv[i] ); - delete[] recv; -} -// float -template<> -void MPI_CLASS::call_maxReduce( - const float *x, float *y, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_FLOAT, MPI_MAX, communicator ); - } else { - auto recv = new FloatIntStruct[n]; - auto send = new FloatIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].f = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_FLOAT_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].f; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( float *x, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce2", profile_level ); - if ( comm_rank_of_max == nullptr ) { - auto send = x; - auto recv = new float[n]; - MPI_Allreduce( send, recv, n, MPI_FLOAT, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new FloatIntStruct[n]; - auto send = new FloatIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].f = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_FLOAT_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].f; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce2", profile_level ); -} -// double -template<> -void MPI_CLASS::call_maxReduce( - const double *x, double *y, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce1", profile_level ); - if ( comm_rank_of_max == nullptr ) { - MPI_Allreduce( (void *) x, (void *) y, n, MPI_DOUBLE, MPI_MAX, communicator ); - } else { - auto recv = new DoubleIntStruct[n]; - auto send = new DoubleIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].d = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_DOUBLE_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - y[i] = recv[i].d; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce1", profile_level ); -} -template<> -void MPI_CLASS::call_maxReduce( double *x, const int n, int *comm_rank_of_max ) const -{ - PROFILE_START( "maxReduce2", profile_level ); - if ( comm_rank_of_max == nullptr ) { - auto send = x; - auto recv = new double[n]; - MPI_Allreduce( send, recv, n, MPI_DOUBLE, MPI_MAX, communicator ); - for ( int i = 0; i < n; i++ ) - x[i] = recv[i]; - delete[] recv; - } else { - auto recv = new DoubleIntStruct[n]; - auto send = new DoubleIntStruct[n]; - for ( int i = 0; i < n; ++i ) { - send[i].d = x[i]; - send[i].i = comm_rank; - } - MPI_Allreduce( send, recv, n, MPI_DOUBLE_INT, MPI_MAXLOC, communicator ); - for ( int i = 0; i < n; ++i ) { - x[i] = recv[i].d; - comm_rank_of_max[i] = recv[i].i; - } - delete[] recv; - delete[] send; - } - PROFILE_STOP( "maxReduce2", profile_level ); -} -#endif - - -/************************************************************************ - * bcast * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// char -template<> -void MPI_CLASS::call_bcast( unsigned char *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_UNSIGNED_CHAR, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -template<> -void MPI_CLASS::call_bcast( char *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_CHAR, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -// int -template<> -void MPI_CLASS::call_bcast( unsigned int *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_UNSIGNED, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -template<> -void MPI_CLASS::call_bcast( int *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_INT, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -// float -template<> -void MPI_CLASS::call_bcast( float *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_FLOAT, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -// double -template<> -void MPI_CLASS::call_bcast( double *x, const int n, const int root ) const -{ - PROFILE_START( "bcast", profile_level ); - MPI_Bcast( x, n, MPI_DOUBLE, root, communicator ); - PROFILE_STOP( "bcast", profile_level ); -} -#else -// We need a concrete instantiation of bcast(x,n,root); -template<> -void MPI_CLASS::call_bcast( char *, const int, const int ) const -{ -} -#endif - - -/************************************************************************ - * Perform a global barrier across all processors. * - ************************************************************************/ -void MPI_CLASS::barrier() const -{ -#ifdef USE_MPI - MPI_Barrier( communicator ); -#endif -} - - -/************************************************************************ - * Send data array to another processor. * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// char -template<> -void MPI_CLASS::send( - const char *buf, const int length, const int recv_proc_number, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - // Send the data - PROFILE_START( "send", profile_level ); - MPI_Send( (void *) buf, length, MPI_CHAR, recv_proc_number, tag, communicator ); - PROFILE_STOP( "send", profile_level ); -} -// int -template<> -void MPI_CLASS::send( - const int *buf, const int length, const int recv_proc_number, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - // Send the data - PROFILE_START( "send", profile_level ); - MPI_Send( (void *) buf, length, MPI_INT, recv_proc_number, tag, communicator ); - PROFILE_STOP( "send", profile_level ); -} -// float -template<> -void MPI_CLASS::send( - const float *buf, const int length, const int recv_proc_number, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - // Send the data - PROFILE_START( "send", profile_level ); - MPI_Send( (void *) buf, length, MPI_FLOAT, recv_proc_number, tag, communicator ); - PROFILE_STOP( "send", profile_level ); -} -// double -template<> -void MPI_CLASS::send( - const double *buf, const int length, const int recv_proc_number, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - // Send the data - PROFILE_START( "send", profile_level ); - MPI_Send( (void *) buf, length, MPI_DOUBLE, recv_proc_number, tag, communicator ); - PROFILE_STOP( "send", profile_level ); -} -#else -// We need a concrete instantiation of send for use without MPI -template<> -void MPI_CLASS::send( const char *buf, const int length, const int, int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - PROFILE_START( "send", profile_level ); - auto id = getRequest( communicator, tag ); - auto it = global_isendrecv_list.find( id ); - MPI_INSIST( it == global_isendrecv_list.end(), - "send must be paired with a previous call to irecv in serial" ); - MPI_ASSERT( it->second.status == 2 ); - memcpy( (char *) it->second.data, buf, length ); - global_isendrecv_list.erase( it ); - PROFILE_START( "send", profile_level ); -} -#endif - - -/************************************************************************ - * Non-blocking send data array to another processor. * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// char -template<> -MPI_Request MPI_CLASS::Isend( - const char *buf, const int length, const int recv_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Isend", profile_level ); - MPI_Isend( (void *) buf, length, MPI_CHAR, recv_proc, tag, communicator, &request ); - PROFILE_STOP( "Isend", profile_level ); - return request; -} -// int -template<> -MPI_Request MPI_CLASS::Isend( - const int *buf, const int length, const int recv_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Isend", profile_level ); - MPI_Isend( (void *) buf, length, MPI_INT, recv_proc, tag, communicator, &request ); - PROFILE_STOP( "Isend", profile_level ); - return request; -} -// float -template<> -MPI_Request MPI_CLASS::Isend( - const float *buf, const int length, const int recv_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Isend", profile_level ); - MPI_Isend( (void *) buf, length, MPI_FLOAT, recv_proc, tag, communicator, &request ); - PROFILE_STOP( "Isend", profile_level ); - return request; -} -// double -template<> -MPI_Request MPI_CLASS::Isend( - const double *buf, const int length, const int recv_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Isend", profile_level ); - MPI_Isend( (void *) buf, length, MPI_DOUBLE, recv_proc, tag, communicator, &request ); - PROFILE_STOP( "Isend", profile_level ); - return request; -} -#else -// We need a concrete instantiation of send for use without mpi -template<> -MPI_Request MPI_CLASS::Isend( - const char *buf, const int length, const int, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - PROFILE_START( "Isend", profile_level ); - auto id = getRequest( communicator, tag ); - auto it = global_isendrecv_list.find( id ); - if ( it == global_isendrecv_list.end() ) { - // We are calling isend first - Isendrecv_struct data; - data.data = buf; - data.status = 1; - global_isendrecv_list.insert( std::pair( id, data ) ); - } else { - // We called irecv first - MPI_ASSERT( it->second.status == 2 ); - memcpy( (char *) it->second.data, buf, length ); - global_isendrecv_list.erase( it ); - } - PROFILE_STOP( "Isend", profile_level ); - return id; -} -#endif - - -/************************************************************************ - * Send byte array to another processor. * - ************************************************************************/ -void MPI_CLASS::sendBytes( - const void *buf, const int number_bytes, const int recv_proc_number, int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - send( (const char *) buf, number_bytes, recv_proc_number, tag ); -} - - -/************************************************************************ - * Non-blocking send byte array to another processor. * - ************************************************************************/ -MPI_Request MPI_CLASS::IsendBytes( - const void *buf, const int number_bytes, const int recv_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - return Isend( (const char *) buf, number_bytes, recv_proc, tag ); -} - - -/************************************************************************ - * Recieve data array to another processor. * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// char -template<> -void MPI_CLASS::recv( - char *buf, int &length, const int send_proc_number, const bool get_length, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - PROFILE_START( "recv", profile_level ); - // Get the recieve length if necessary - if ( get_length ) { - int bytes = this->probe( send_proc_number, tag ); - int recv_length = bytes / sizeof( char ); - MPI_INSIST( length >= recv_length, "Recived length is larger than allocated array" ); - length = recv_length; - } - // Send the data - MPI_Status status; - MPI_Recv( (void *) buf, length, MPI_CHAR, send_proc_number, tag, communicator, &status ); - PROFILE_STOP( "recv", profile_level ); -} -// int -template<> -void MPI_CLASS::recv( - int *buf, int &length, const int send_proc_number, const bool get_length, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - PROFILE_START( "recv", profile_level ); - // Get the recieve length if necessary - if ( get_length ) { - int bytes = this->probe( send_proc_number, tag ); - int recv_length = bytes / sizeof( int ); - MPI_INSIST( length >= recv_length, "Recived length is larger than allocated array" ); - length = recv_length; - } - // Send the data - MPI_Status status; - MPI_Recv( (void *) buf, length, MPI_INT, send_proc_number, tag, communicator, &status ); - PROFILE_STOP( "recv", profile_level ); -} -// float -template<> -void MPI_CLASS::recv( - float *buf, int &length, const int send_proc_number, const bool get_length, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - PROFILE_START( "recv", profile_level ); - // Get the recieve length if necessary - if ( get_length ) { - int bytes = this->probe( send_proc_number, tag ); - int recv_length = bytes / sizeof( float ); - MPI_INSIST( length >= recv_length, "Recived length is larger than allocated array" ); - length = recv_length; - } - // Send the data - MPI_Status status; - MPI_Recv( (void *) buf, length, MPI_FLOAT, send_proc_number, tag, communicator, &status ); - PROFILE_STOP( "recv", profile_level ); -} -// double -template<> -void MPI_CLASS::recv( - double *buf, int &length, const int send_proc_number, const bool get_length, int tag ) const -{ - // Set the tag to 0 if it is < 0 - tag = ( tag >= 0 ) ? tag : 0; - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - PROFILE_START( "recv", profile_level ); - // Get the recieve length if necessary - if ( get_length ) { - int bytes = this->probe( send_proc_number, tag ); - int recv_length = bytes / sizeof( double ); - MPI_INSIST( length >= recv_length, "Recived length is larger than allocated array" ); - length = recv_length; - } - // Send the data - MPI_Status status; - MPI_Recv( (void *) buf, length, MPI_DOUBLE, send_proc_number, tag, communicator, &status ); - PROFILE_STOP( "recv", profile_level ); -} -#else -// We need a concrete instantiation of recv for use without mpi -template<> -void MPI_CLASS::recv( char *buf, int &length, const int, const bool, int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - PROFILE_START( "recv", profile_level ); - auto id = getRequest( communicator, tag ); - auto it = global_isendrecv_list.find( id ); - MPI_INSIST( it != global_isendrecv_list.end(), - "recv must be paired with a previous call to isend in serial" ); - MPI_ASSERT( it->second.status == 1 ); - memcpy( buf, it->second.data, length ); - global_isendrecv_list.erase( it ); - PROFILE_STOP( "recv", profile_level ); -} -#endif - - -/************************************************************************ - * Non-blocking recieve data array to another processor. * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// char -template<> -MPI_Request MPI_CLASS::Irecv( - char *buf, const int length, const int send_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Irecv", profile_level ); - MPI_Irecv( (void *) buf, length, MPI_CHAR, send_proc, tag, communicator, &request ); - PROFILE_STOP( "Irecv", profile_level ); - return request; -} -// int -template<> -MPI_Request MPI_CLASS::Irecv( - int *buf, const int length, const int send_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Irecv", profile_level ); - MPI_Irecv( (void *) buf, length, MPI_INT, send_proc, tag, communicator, &request ); - PROFILE_STOP( "Irecv", profile_level ); - return request; -} -// float -template<> -MPI_Request MPI_CLASS::Irecv( - float *buf, const int length, const int send_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Irecv", profile_level ); - MPI_Irecv( (void *) buf, length, MPI_FLOAT, send_proc, tag, communicator, &request ); - PROFILE_STOP( "Irecv", profile_level ); - return request; -} -// double -template<> -MPI_Request MPI_CLASS::Irecv( - double *buf, const int length, const int send_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Request request; - PROFILE_START( "Irecv", profile_level ); - MPI_Irecv( (void *) buf, length, MPI_DOUBLE, send_proc, tag, communicator, &request ); - PROFILE_STOP( "Irecv", profile_level ); - return request; -} -#else -// We need a concrete instantiation of irecv for use without mpi -template<> -MPI_Request MPI_CLASS::Irecv( char *buf, const int length, const int, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - PROFILE_START( "Irecv", profile_level ); - auto id = getRequest( communicator, tag ); - auto it = global_isendrecv_list.find( id ); - if ( it == global_isendrecv_list.end() ) { - // We are calling Irecv first - Isendrecv_struct data; - data.data = buf; - data.status = 2; - global_isendrecv_list.insert( std::pair( id, data ) ); - } else { - // We called Isend first - MPI_ASSERT( it->second.status == 1 ); - memcpy( buf, it->second.data, length ); - global_isendrecv_list.erase( it ); - } - PROFILE_STOP( "Irecv", profile_level ); - return id; -} -#endif - - -/************************************************************************ - * Recieve byte array to another processor. * - ************************************************************************/ -void MPI_CLASS::recvBytes( void *buf, int &number_bytes, const int send_proc, int tag ) const -{ - recv( (char *) buf, number_bytes, send_proc, false, tag ); -} - - -/************************************************************************ - * Recieve byte array to another processor. * - ************************************************************************/ -MPI_Request MPI_CLASS::IrecvBytes( - void *buf, const int number_bytes, const int send_proc, const int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - return Irecv( (char *) buf, number_bytes, send_proc, tag ); -} - - -/************************************************************************ - * allGather * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_allGather( - const unsigned char &x_in, unsigned char *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( - (void *) &x_in, 1, MPI_UNSIGNED_CHAR, (void *) x_out, 1, MPI_UNSIGNED_CHAR, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( const unsigned char *x_in, int size_in, - unsigned char *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_CHAR, (void *) x_out, size_out, disp_out, MPI_CHAR, - communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// char -template<> -void MPI_CLASS::call_allGather( const char &x_in, char *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_CHAR, (void *) x_out, 1, MPI_CHAR, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const char *x_in, int size_in, char *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_CHAR, (void *) x_out, size_out, disp_out, MPI_CHAR, - communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_allGather( const unsigned int &x_in, unsigned int *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_UNSIGNED, (void *) x_out, 1, MPI_UNSIGNED, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const unsigned int *x_in, int size_in, unsigned int *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_UNSIGNED, (void *) x_out, size_out, disp_out, - MPI_UNSIGNED, communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// int -template<> -void MPI_CLASS::call_allGather( const int &x_in, int *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_INT, (void *) x_out, 1, MPI_INT, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const int *x_in, int size_in, int *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_INT, (void *) x_out, size_out, disp_out, MPI_INT, - communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_allGather( - const unsigned long int &x_in, unsigned long int *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( - (void *) &x_in, 1, MPI_UNSIGNED_LONG, (void *) x_out, 1, MPI_UNSIGNED_LONG, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( const unsigned long int *x_in, int size_in, - unsigned long int *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_UNSIGNED_LONG, (void *) x_out, size_out, disp_out, - MPI_UNSIGNED_LONG, communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_allGather( const long int &x_in, long int *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_LONG, (void *) x_out, 1, MPI_LONG, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const long int *x_in, int size_in, long int *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_LONG, (void *) x_out, size_out, disp_out, MPI_LONG, - communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// float -template<> -void MPI_CLASS::call_allGather( const float &x_in, float *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_FLOAT, (void *) x_out, 1, MPI_FLOAT, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const float *x_in, int size_in, float *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_FLOAT, (void *) x_out, size_out, disp_out, - MPI_FLOAT, communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -// double -template<> -void MPI_CLASS::call_allGather( const double &x_in, double *x_out ) const -{ - PROFILE_START( "allGather", profile_level ); - MPI_Allgather( (void *) &x_in, 1, MPI_DOUBLE, (void *) x_out, 1, MPI_DOUBLE, communicator ); - PROFILE_STOP( "allGather", profile_level ); -} -template<> -void MPI_CLASS::call_allGather( - const double *x_in, int size_in, double *x_out, int *size_out, int *disp_out ) const -{ - PROFILE_START( "allGatherv", profile_level ); - MPI_Allgatherv( (void *) x_in, size_in, MPI_DOUBLE, (void *) x_out, size_out, disp_out, - MPI_DOUBLE, communicator ); - PROFILE_STOP( "allGatherv", profile_level ); -} -#else -// We need a concrete instantiation of call_allGather(x_in,size_in,x_out,size_out) -template<> -void MPI_CLASS::call_allGather( const char *, int, char *, int *, int * ) const -{ - MPI_ERROR( "Internal error in communicator (allGather) " ); -} -#endif - - -/************************************************************************ - * allToAll * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -template<> -void MPI_CLASS::allToAll( - const int n, const unsigned char *send, unsigned char *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( - (void *) send, n, MPI_UNSIGNED_CHAR, (void *) recv, n, MPI_UNSIGNED_CHAR, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( const int n, const char *send, char *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_CHAR, (void *) recv, n, MPI_CHAR, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( - const int n, const unsigned int *send, unsigned int *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_UNSIGNED, (void *) recv, n, MPI_UNSIGNED, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( const int n, const int *send, int *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_INT, (void *) recv, n, MPI_INT, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( - const int n, const unsigned long int *send, unsigned long int *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( - (void *) send, n, MPI_UNSIGNED_LONG, (void *) recv, n, MPI_UNSIGNED_LONG, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( const int n, const long int *send, long int *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_LONG, (void *) recv, n, MPI_LONG, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( const int n, const float *send, float *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_FLOAT, (void *) recv, n, MPI_FLOAT, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -template<> -void MPI_CLASS::allToAll( const int n, const double *send, double *recv ) const -{ - PROFILE_START( "allToAll", profile_level ); - MPI_Alltoall( (void *) send, n, MPI_DOUBLE, (void *) recv, n, MPI_DOUBLE, communicator ); - PROFILE_STOP( "allToAll", profile_level ); -} -#endif - - -/************************************************************************ - * call_allToAll * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_allToAll( const unsigned char *send_data, const int send_cnt[], - const int send_disp[], unsigned char *recv_data, const int *recv_cnt, - const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_UNSIGNED_CHAR, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_UNSIGNED_CHAR, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// char -template<> -void MPI_CLASS::call_allToAll( const char *send_data, const int send_cnt[], - const int send_disp[], char *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_CHAR, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_CHAR, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_allToAll( const unsigned int *send_data, const int send_cnt[], - const int send_disp[], unsigned int *recv_data, const int *recv_cnt, - const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_UNSIGNED, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_UNSIGNED, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// int -template<> -void MPI_CLASS::call_allToAll( const int *send_data, const int send_cnt[], - const int send_disp[], int *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_INT, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_INT, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_allToAll( const unsigned long int *send_data, - const int send_cnt[], const int send_disp[], unsigned long int *recv_data, const int *recv_cnt, - const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_UNSIGNED_LONG, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_UNSIGNED_LONG, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_allToAll( const long int *send_data, const int send_cnt[], - const int send_disp[], long int *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_LONG, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_LONG, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// float -template<> -void MPI_CLASS::call_allToAll( const float *send_data, const int send_cnt[], - const int send_disp[], float *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_FLOAT, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_FLOAT, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -// double -template<> -void MPI_CLASS::call_allToAll( const double *send_data, const int send_cnt[], - const int send_disp[], double *recv_data, const int *recv_cnt, const int *recv_disp ) const -{ - PROFILE_START( "allToAllv", profile_level ); - MPI_Alltoallv( (void *) send_data, (int *) send_cnt, (int *) send_disp, MPI_DOUBLE, - (void *) recv_data, (int *) recv_cnt, (int *) recv_disp, MPI_DOUBLE, communicator ); - PROFILE_STOP( "allToAllv", profile_level ); -} -#else -// Default instatiation of unsigned char -template<> -void MPI_CLASS::call_allToAll( - const char *, const int[], const int[], char *, const int *, const int * ) const -{ - MPI_ERROR( "Should not reach this point" ); -} -#endif - - -/************************************************************************ - * call_sumScan * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_sumScan( - const unsigned char *send, unsigned char *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// char -template<> -void MPI_CLASS::call_sumScan( const char *send, char *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_sumScan( - const unsigned int *send, unsigned int *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// int -template<> -void MPI_CLASS::call_sumScan( const int *send, int *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_INT, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_sumScan( const long int *send, long int *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_LONG, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_sumScan( - const unsigned long *send, unsigned long *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// size_t -#ifdef USE_WINDOWS -template<> -void MPI_CLASS::call_sumScan( const size_t *send, size_t *recv, int n ) const -{ - MPI_ASSERT( MPI_SIZE_T != 0 ); - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIZE_T, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -#endif -// float -template<> -void MPI_CLASS::call_sumScan( const float *send, float *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_FLOAT, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// double -template<> -void MPI_CLASS::call_sumScan( const double *send, double *recv, int n ) const -{ - PROFILE_START( "sumScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_DOUBLE, MPI_SUM, communicator ); - PROFILE_STOP( "sumScan", profile_level ); -} -// std::complex -template<> -void MPI_CLASS::call_sumScan>( - const std::complex *x, std::complex *y, int n ) const -{ - auto send = new double[2 * n]; - auto recv = new double[2 * n]; - for ( int i = 0; i < n; i++ ) { - send[2 * i + 0] = real( x[i] ); - send[2 * i + 1] = imag( x[i] ); - } - MPI_Scan( (void *) send, (void *) recv, 2 * n, MPI_DOUBLE, MPI_SUM, communicator ); - for ( int i = 0; i < n; i++ ) - y[i] = std::complex( recv[2 * i + 0], recv[2 * i + 1] ); - delete[] send; - delete[] recv; -} -#endif - - -/************************************************************************ - * call_minScan * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_minScan( - const unsigned char *send, unsigned char *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// char -template<> -void MPI_CLASS::call_minScan( const char *send, char *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_minScan( - const unsigned int *send, unsigned int *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// int -template<> -void MPI_CLASS::call_minScan( const int *send, int *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_INT, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_minScan( - const unsigned long int *send, unsigned long int *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_minScan( const long int *send, long int *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_LONG, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// size_t -#ifdef USE_WINDOWS -template<> -void MPI_CLASS::call_minScan( const size_t *send, size_t *recv, int n ) const -{ - MPI_ASSERT( MPI_SIZE_T != 0 ); - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIZE_T, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -#endif -// float -template<> -void MPI_CLASS::call_minScan( const float *send, float *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_FLOAT, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -// double -template<> -void MPI_CLASS::call_minScan( const double *send, double *recv, int n ) const -{ - PROFILE_START( "minScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_DOUBLE, MPI_MIN, communicator ); - PROFILE_STOP( "minScan", profile_level ); -} -#endif - - -/************************************************************************ - * call_maxScan * - * Note: these specializations are only called when using MPI. * - ************************************************************************/ -#ifdef USE_MPI -// unsigned char -template<> -void MPI_CLASS::call_maxScan( - const unsigned char *send, unsigned char *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_CHAR, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// char -template<> -void MPI_CLASS::call_maxScan( const char *send, char *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIGNED_CHAR, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// unsigned int -template<> -void MPI_CLASS::call_maxScan( - const unsigned int *send, unsigned int *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// int -template<> -void MPI_CLASS::call_maxScan( const int *send, int *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_INT, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// long int -template<> -void MPI_CLASS::call_maxScan( const long int *send, long int *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_LONG, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// unsigned long int -template<> -void MPI_CLASS::call_maxScan( - const unsigned long int *send, unsigned long int *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_UNSIGNED_LONG, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// size_t -#ifdef USE_WINDOWS -template<> -void MPI_CLASS::call_maxScan( const size_t *send, size_t *recv, int n ) const -{ - MPI_ASSERT( MPI_SIZE_T != 0 ); - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_SIZE_T, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -#endif -// float -template<> -void MPI_CLASS::call_maxScan( const float *send, float *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_INT, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -// double -template<> -void MPI_CLASS::call_maxScan( const double *send, double *recv, int n ) const -{ - PROFILE_START( "maxScan", profile_level ); - MPI_Scan( (void *) send, (void *) recv, n, MPI_DOUBLE, MPI_MAX, communicator ); - PROFILE_STOP( "maxScan", profile_level ); -} -#endif - - -/************************************************************************ - * Communicate ranks for communication * - ************************************************************************/ -std::vector MPI_CLASS::commRanks( const std::vector &ranks ) const -{ -#ifdef USE_MPI - // Get a byte array with the ranks to communicate - auto data1 = new char[comm_size]; - auto data2 = new char[comm_size]; - memset( data1, 0, comm_size ); - memset( data2, 0, comm_size ); - for ( auto &rank : ranks ) - data1[rank] = 1; - MPI_Alltoall( data1, 1, MPI_CHAR, data2, 1, MPI_CHAR, communicator ); - int N = 0; - for ( int i = 0; i < comm_size; i++ ) - N += data2[i]; - std::vector ranks_out; - ranks_out.reserve( N ); - for ( int i = 0; i < comm_size; i++ ) { - if ( data2[i] ) - ranks_out.push_back( i ); - } - delete[] data1; - delete[] data2; - return ranks_out; -#else - return ranks; -#endif -} - - -/************************************************************************ - * Wait functions * - ************************************************************************/ -#ifdef USE_MPI -void MPI_CLASS::wait( MPI_Request request ) -{ - PROFILE_START( "wait", profile_level ); - MPI_Status status; - int flag = 0; - int err = MPI_Test( &request, &flag, &status ); - MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid - while ( !flag ) { - // Put the current thread to sleep to allow other threads to run - sched_yield(); - // Check if the request has finished - MPI_Test( &request, &flag, &status ); - } - PROFILE_STOP( "wait", profile_level ); -} -int MPI_CLASS::waitAny( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return -1; - PROFILE_START( "waitAny", profile_level ); - int index = -1; - int flag = 0; - auto status = new MPI_Status[count]; - int err = MPI_Testany( count, request, &index, &flag, status ); - MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid - while ( !flag ) { - // Put the current thread to sleep to allow other threads to run - sched_yield(); - // Check if the request has finished - MPI_Testany( count, request, &index, &flag, status ); - } - MPI_ASSERT( index >= 0 ); // Check that the index is valid - delete[] status; - PROFILE_STOP( "waitAny", profile_level ); - return index; -} -void MPI_CLASS::waitAll( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return; - PROFILE_START( "waitAll", profile_level ); - int flag = 0; - auto status = new MPI_Status[count]; - int err = MPI_Testall( count, request, &flag, status ); - MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid - while ( !flag ) { - // Put the current thread to sleep to allow other threads to run - sched_yield(); - // Check if the request has finished - MPI_Testall( count, request, &flag, status ); - } - PROFILE_STOP( "waitAll", profile_level ); - delete[] status; -} -std::vector MPI_CLASS::waitSome( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return std::vector(); - PROFILE_START( "waitSome", profile_level ); - std::vector indicies( count, -1 ); - auto *status = new MPI_Status[count]; - int outcount = 0; - int err = MPI_Testsome( count, request, &outcount, &indicies[0], status ); - MPI_ASSERT( err == MPI_SUCCESS ); // Check that the first call is valid - MPI_ASSERT( outcount != MPI_UNDEFINED ); // Check that the first call is valid - while ( outcount == 0 ) { - // Put the current thread to sleep to allow other threads to run - sched_yield(); - // Check if the request has finished - MPI_Testsome( count, request, &outcount, &indicies[0], status ); - } - indicies.resize( outcount ); - delete[] status; - PROFILE_STOP( "waitSome", profile_level ); - return indicies; -} -#else -void MPI_CLASS::wait( MPI_Request request ) -{ - PROFILE_START( "wait", profile_level ); - while ( 1 ) { - // Check if the request is in our list - if ( global_isendrecv_list.find( request ) == global_isendrecv_list.end() ) - break; - // Put the current thread to sleep to allow other threads to run - sched_yield(); - } - PROFILE_STOP( "wait", profile_level ); -} -int MPI_CLASS::waitAny( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return -1; - PROFILE_START( "waitAny", profile_level ); - int index = 0; - while ( 1 ) { - // Check if the request is in our list - bool found_any = false; - for ( int i = 0; i < count; i++ ) { - if ( global_isendrecv_list.find( request[i] ) == global_isendrecv_list.end() ) { - found_any = true; - index = i; - } - } - if ( found_any ) - break; - // Put the current thread to sleep to allow other threads to run - sched_yield(); - } - PROFILE_STOP( "waitAny", profile_level ); - return index; -} -void MPI_CLASS::waitAll( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return; - PROFILE_START( "waitAll", profile_level ); - while ( 1 ) { - // Check if the request is in our list - bool found_all = true; - for ( int i = 0; i < count; i++ ) { - if ( global_isendrecv_list.find( request[i] ) != global_isendrecv_list.end() ) - found_all = false; - } - if ( found_all ) - break; - // Put the current thread to sleep to allow other threads to run - sched_yield(); - } - PROFILE_STOP( "waitAll", profile_level ); -} -std::vector MPI_CLASS::waitSome( int count, MPI_Request *request ) -{ - if ( count == 0 ) - return std::vector(); - PROFILE_START( "waitSome", profile_level ); - std::vector indicies; - while ( 1 ) { - // Check if the request is in our list - for ( int i = 0; i < count; i++ ) { - if ( global_isendrecv_list.find( request[i] ) == global_isendrecv_list.end() ) - indicies.push_back( i ); - } - if ( !indicies.empty() ) - break; - // Put the current thread to sleep to allow other threads to run - sched_yield(); - } - PROFILE_STOP( "waitSome", profile_level ); - return indicies; -} -#endif - - -/************************************************************************ - * Probe functions * - ************************************************************************/ -#ifdef USE_MPI -int MPI_CLASS::Iprobe( int source, int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Status status; - int flag = 0; - MPI_Iprobe( source, tag, communicator, &flag, &status ); - if ( flag == 0 ) - return -1; - int count; - MPI_Get_count( &status, MPI_BYTE, &count ); - MPI_ASSERT( count >= 0 ); - return count; -} -int MPI_CLASS::probe( int source, int tag ) const -{ - MPI_INSIST( tag <= d_maxTag, "Maximum tag value exceeded" ); - MPI_INSIST( tag >= 0, "tag must be >= 0" ); - MPI_Status status; - MPI_Probe( source, tag, communicator, &status ); - int count; - MPI_Get_count( &status, MPI_BYTE, &count ); - MPI_ASSERT( count >= 0 ); - return count; -} -#else -int MPI_CLASS::Iprobe( int, int ) const -{ - MPI_ERROR( "Not implimented for serial codes (Iprobe)" ); - return 0; -} -int MPI_CLASS::probe( int, int ) const -{ - MPI_ERROR( "Not implimented for serial codes (probe)" ); - return 0; -} -#endif - - -/************************************************************************ - * Timer functions * - ************************************************************************/ -#ifdef USE_MPI -double MPI_CLASS::time() { return MPI_Wtime(); } -double MPI_CLASS::tick() { return MPI_Wtick(); } -#else -double MPI_CLASS::time() -{ - auto t = std::chrono::system_clock::now(); - auto ns = std::chrono::duration_cast( t.time_since_epoch() ); - return 1e-9 * ns.count(); -} -double MPI_CLASS::tick() -{ - auto period = std::chrono::system_clock::period(); - return static_cast( period.num ) / static_cast( period.den ); -} -#endif - - -/************************************************************************ - * Serialize a block of code across MPI processes * - ************************************************************************/ -void MPI_CLASS::serializeStart() -{ -#ifdef USE_MPI - using namespace std::chrono_literals; - if ( comm_rank == 0 ) { - // Start rank 0 immediately - } else { - // Wait for a message from the previous rank - MPI_Request request; - MPI_Status status; - int flag = false, buf = 0; - MPI_Irecv( &buf, 1, MPI_INT, comm_rank - 1, 5627, MPI_COMM_WORLD, &request ); - while ( !flag ) { - MPI_Test( &request, &flag, &status ); - std::this_thread::sleep_for( 50ms ); - } - } -#endif -} -void MPI_CLASS::serializeStop() -{ -#ifdef USE_MPI - using namespace std::chrono_literals; - if ( comm_rank < comm_size - 1 ) { - // Send flag to next rank - MPI_Send( &comm_rank, 1, MPI_INT, comm_rank + 1, 5627, MPI_COMM_WORLD ); - // Wait for final finished flag - int flag = false, buf = 0; - MPI_Request request; - MPI_Status status; - MPI_Irecv( &buf, 1, MPI_INT, comm_size - 1, 5627, MPI_COMM_WORLD, &request ); - while ( !flag ) { - MPI_Test( &request, &flag, &status ); - std::this_thread::sleep_for( 50ms ); - } - } else { - // Send final flag to all ranks - for ( int i = 0; i < comm_size - 1; i++ ) - MPI_Send( &comm_rank, 1, MPI_INT, i, 5627, MPI_COMM_WORLD ); - } -#endif -} - - -/**************************************************************************** - * Function to start/stop MPI * - ****************************************************************************/ -#ifdef USE_EXT_MPI -static bool called_MPI_Init = false; -#endif -bool MPI_CLASS::MPI_Active() -{ -#ifdef USE_EXT_MPI - int MPI_initialized, MPI_finialized; - MPI_Initialized( &MPI_initialized ); - MPI_Finalized( &MPI_finialized ); - return MPI_initialized != 0 && MPI_finialized == 0; -#else - return false; -#endif -} -void MPI_CLASS::start_MPI( int argc, char *argv[], int profile_level ) -{ - changeProfileLevel( profile_level ); - NULL_USE( argc ); - NULL_USE( argv ); -#ifdef USE_EXT_MPI - if ( MPI_Active() ) { - called_MPI_Init = false; - } else { - int provided; - int result = MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided ); - if ( result != MPI_SUCCESS ) - MPI_ERROR( "Unable to initialize MPI" ); - if ( provided < MPI_THREAD_MULTIPLE ) - std::cerr << "Warning: Failed to start MPI with MPI_THREAD_MULTIPLE\n"; - called_MPI_Init = true; - } -#endif -} -void MPI_CLASS::stop_MPI() -{ -#ifdef USE_EXT_MPI - int finalized; - MPI_Finalized( &finalized ); - if ( called_MPI_Init && !finalized ) { - MPI_Barrier( MPI_COMM_WORLD ); - MPI_Finalize(); - called_MPI_Init = true; - } -#endif -} - - -} // namespace Utilities - diff --git a/common/MPI.h b/common/MPI.h deleted file mode 100644 index e3fd3e13..00000000 --- a/common/MPI.h +++ /dev/null @@ -1,1152 +0,0 @@ -// This file includes a wrapper class for MPI functions -// Note this is a modified version of the MPI class for the Advanced Multi-Physics Package -// Used with permission - -/* - -Copyright (c) 2012 UT-Battelle, LLC - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -Collection of administrative costs for redistribution of the source code or binary form is allowed. However, collection of a royalty or other fee in excess of good faith amount for cost recovery for such redistribution is prohibited. - -*/ - -#ifndef included_LBPM_MPI -#define included_LBPM_MPI - - -#include -#include -#include -#include -#include -#include -#include - - -// Include mpi.h (or define MPI objects) -// clang-format off -#ifdef USE_MPI - #include "mpi.h" -#else - typedef int MPI_Comm; - typedef int MPI_Request; - typedef int MPI_Status; - typedef void *MPI_Errhandler; - enum MPI_TYPES { MPI_INT, MPI_FLOAT, MPI_DOUBLE }; - #define MPI_COMM_WORLD ( (MPI_Comm) 0xF4000010 ) - #define MPI_COMM_SELF ( (MPI_Comm) 0xF4000001 ) - #define MPI_COMM_NULL ( (MPI_Comm) 0xF4000000 ) -#endif -// clang-format on - - -namespace Utilities { - - -/** - * \class MPI - * - * @brief Provides C++ wrapper around MPI routines. - * - * Class MPI groups common MPI routines into one globally-accessible - * location. It provides small, simple routines that are common in MPI code. - * In some cases, the calling syntax has been simplified for convenience. - * Moreover, there is no reason to include the preprocessor ifdef/endif - * guards around these calls, since the MPI libraries are not called in - * these routines if the MPI libraries are not being used (e.g., when - * writing serial code). - * Note: Many of the communication routines are templated on type. When using - * unknown types the reduce calls will fail, the send and gather calls should - * succeed provided that the size of the data type object is a fixed size on - * all processors. sizeof(type) must be the same for all elements and processors. - */ -class MPI final -{ -public: - enum class ThreadSupport : int { SINGLE, FUNNELED, SERIALIZED, MULTIPLE }; - -public: // Constructors - /** - *\brief Is MPI active - *\details This returns true if MPI is initailized and not finalized - */ - static bool MPI_active(); - - /** - *\brief Empty constructor - *\details This creates an empty constructor that does not contain an MPI communicator. - */ - MPI(); - - - //! Empty destructor - ~MPI(); - - - /** - * \brief Constructor from existing MPI communicator - * \details This constructor creates a new communicator from an existing MPI communicator. - * This does not create a new internal MPI_Comm, but uses the existing comm. - * Note that by default, this will not free the MPI_Comm object and the user is - * responsible - * for free'ing the MPI_Comm when it is no longer used. This behavior is controlled by the - * optional manage argument. - * \param comm Existing MPI communicator - * \param manage Do we want to manage the comm (free the MPI_Comm when this object leaves - * scope) - */ - MPI( MPI_Comm comm, bool manage = false ); - - - /** - * \brief Constructor from existing communicator - * \details This constructor creates a new communicator from an existing communicator. - * This does not create a new internal MPI_Comm, but uses the existing comm. - * \param comm Existing communicator - */ - MPI( const MPI &comm ); - - - /*! - * Move constructor - * @param rhs Communicator to copy - */ - MPI( MPI &&rhs ); - - - /** - * \brief Assignment operator - * \details This operator overloads the assignment to correctly copy an communicator - * \param comm Existing MPI object - */ - MPI &operator=( const MPI &comm ); - - - /*! - * Move assignment operator - * @param rhs Communicator to copy - */ - MPI &operator=( MPI &&rhs ); - - - /** - * \brief Reset the object - * \details This resets the object to the empty state without an MPI_Comm - */ - void reset(); - - -public: // Member functions - /** - * \brief Get the node name - * \details This function returns a unique name for each node. - * It is a wrapper for MPI_Get_processor_name. - */ - static std::string getNodeName(); - - - //! Function to return the number of processors available - static int getNumberOfProcessors(); - - - //! Function to return the affinity of the current process - static std::vector getProcessAffinity(); - - - //! Function to set the affinity of the current process - static void setProcessAffinity( const std::vector &procs ); - - - /** - * \brief Load balance the processes within a node - * \details This function will redistribute the processes within a node using the - * process affinities to achieve the desired load balance. - * Note: this is a global operation on the given comm, and it is STRONGLY - * recommended to use COMM_WORLD. - * \param comm The communicator to use (Default is COMM_WORLD) - * \param method The desired load balance method to use: - * 1: Adjust the affinities so all processes share the given processors. - * This effectively allows the OS to handle the load balancing - * by migrating the processes as necessary. This is recommended - * for most users and use cases. (default) - * 2: Adjust the affinities so that the fewest number of processes overlap. - * This will try to give each process a unique set of processors while - * ensuring that each process has at least N_min processes. - * \param procs An optional list of processors to use. By default, setting this to an - * empty vector will use all available processors on the given node. - * \param N_min The minimum number of processors for any process (-1 indicates all available - * processors). - * \param N_max The maximum number of processors for any process (-1 indicates all available - * processors). - * - */ - static void balanceProcesses( const MPI &comm = MPI( MPI_COMM_WORLD ), const int method = 1, - const std::vector &procs = std::vector(), const int N_min = 1, - const int N_max = -1 ); - - - //! Query the level of thread support - static ThreadSupport queryThreadSupport(); - - - /** - * \brief Generate a random number - * \details This generates a random number that is consistent across the comm - */ - size_t rand() const; - - - /** - * \brief Split an existing communicator - * \details This creates a new communicator by splitting an existing communicator. - * See MPI_Comm_split for information on how the underlying split will occur. - * Note: the underlying MPI_Comm object will be free'd automatically when it is no longer - * used by any MPI objects. - * \param color Control of subset assignment (nonnegative integer). - * Processes with the same color are in the same new communicator . - * -1: processor will not be a member of any object (NULL object will be returned) - * \param key Control of rank assignment (integer). - * Note that, for a fixed color, the keys need not be unique. The processes will - * be sorted - * in ascending order according to this key, then all the processes in a given - * color will - * have the relative rank order as they did in their parent group. (See - * MPI_Comm_split) - */ - MPI split( int color, int key = -1 ) const; - - - /** - * \brief Split an existing communicator by node - * \details This creates a new communicator by splitting an existing communicator - * by the node. This will result in a separate MPI_Comm for each physical node. - * Internally this will use MPI_Get_processor_name to identify the nodes. - * Note: the underlying MPI_Comm object will be free'd automatically when it is no longer - * used by any MPI objects) - * \param key Control of rank assignment (integer). - * Note that, for a fixed color, the keys need not be unique. The processes will - * be sorted - * in ascending order according to this key, then all the processes in a given - * color will - * have the relative rank order as they did in their parent group. (See - * MPI_Comm_split) - */ - MPI splitByNode( int key = -1 ) const; - - - /** - * \brief Duplicate an existing communicator - * \details This creates a new communicator by duplicating an existing communicator. - * The resulting communicator will exist over the same processes, but have a different - * context. - * Note: the underlying MPI_Comm object will be free'd automatically when it is no longer - * used by any MPI objects. - */ - MPI dup() const; - - - /** - * \brief Create a communicator from the intersection of two communicators - * \details This creates a new communicator by intersecting two existing communicators. - * Any processors that do not contain the both communicators will receive a NULL communicator. - * There are 3 possible cases: - * The communicators are disjoint (a null communicator will be returned on all processors). - * One communicator is a sub communicator of another. This will require communication on - * the smaller communicator only. - * The communicators partially overlap. This will require communication on the first - * communicator. - */ - static MPI intersect( const MPI &comm1, const MPI &comm2 ); - - - /** - * Check if the current communicator is NULL - */ - bool isNull() const { return d_isNull; } - - - /** - * \brief Return the global ranks for the comm - * \details This returns a vector which contains the global ranks for each - * member of the communicator. The global ranks are defined according to WORLD comm. - */ - std::vector globalRanks() const; - - - /** - * Get the current MPI communicator. - * Note: The underlying MPI_Comm object may be free'd by the object when it is no - * longer used by any communicators. If the user has made a copy using the - * getCommunicator routine, then it may be free'd without user knowledge. The - * user is responsible for checking if the communicator is valid, or keeping a - * copy of the communicator that provided the MPI_Communicator. - */ - const MPI_Comm &getCommunicator() const { return communicator; } - - - /** - * \brief Overload operator == - * \details Overload operator comm1 == comm2. Two MPI objects are == if they share the same - * communicator. - * Note: this is a local operation. - */ - bool operator==( const MPI & ) const; - - - /** - * \brief Overload operator != - * \details Overload operator comm1 != comm2. Two MPI objects are != if they - * do not share the same communicator. - * Note: this is a local operation. - */ - bool operator!=( const MPI & ) const; - - - /** - * \brief Overload operator < - * \details Overload operator comm1 < comm2. One MPI object is < another iff all the - * processors in the first object are also in the second. Additionally, the second - * object must contain at least one processor that is not in the first object. - * This is a collective operation, based on the first communicator. - * As a result all processors on the first communicator will return the same value, - * while any processors that are not on the first communicator will return an unknown value. - * Additionally, all processors on the first object MUST call this routine and will be - * synchronized through this call (there is an internalallReduce). - */ - bool operator<( const MPI & ) const; - - - /** - * \brief Overload operator <= - * \details Overload operator comm1 <= comm2. One MPI object is <= another iff all the - * processors in the first object are also in the second. This is a collective operation, - * based on the first communicator. As a result all processors on the first communicator - * will return the same value, while any processors that are not on the first communicator - * will return an unknown value. Additionally, all processors on the first object MUST - * call this routine and will be synchronized through this call (there is an internal - * allReduce). - */ - bool operator<=( const MPI & ) const; - - - /** - * \brief Overload operator > - * \details Overload operator comm1 > comm2. One MPI object is > another iff all the - * processors in the second object are also in the first. Additionally, the first object - * must contain at least one processor that is not in the second object. - * This is a collective operation, based on the first communicator. - * As a result all processors on the first communicator will return the same value, - * while any processors that are not on the first communicator will return an unknown value. - * Additionally, all processors on the first object MUST call this routine and will be - * synchronized through this call (there is an internal allReduce). - */ - bool operator>( const MPI & ) const; - - - /** - * \brief Overload operator >= - * \details Overload operator comm1 >= comm2. One MPI object is > another iff all the - * processors in the second object are also in the first. Additionally, the first object - * must contain at least one processor that is not in the second object. - * This is a collective operation, based on the first communicator. - * As a result all processors on the first communicator will return the same value, while any - * processors that are not on the first communicator will return an unknown value. - * Additionally, all processors on the first object MUST call this routine and will be - * synchronized through this call (there is an internal allReduce). - */ - bool operator>=( const MPI & ) const; - - - /** - * \brief Compare to another communicator - * \details This compares the current communicator to another communicator. - * This returns 1 if the two communicators are equal (they share the same MPI communicator), - * 2 if the contexts and groups are the same, 3 if different contexts but identical groups, - * 4 if different contexts but similar groups, and 0 otherwise. - * Note: this is a local operation. - */ - int compare( const MPI & ) const; - - - /** - * Return the processor rank (identifier) from 0 through the number of - * processors minus one. - */ - int getRank() const { return comm_rank; } - - - /** - * Return the number of processors. - */ - int getSize() const { return comm_size; } - - - /** - * Return the maximum tag - */ - int maxTag() const { return d_maxTag; } - - - /** - * \brief Return a new tag - * \details This routine will return an unused tag for communication. - * Note that this tag may match a user tag, but this function will - * not return two duplicate tags. This is a global operation. - */ - int newTag(); - - - /** - * Call MPI_Abort or exit depending on whether running with one or more - * processes and value set by function above, if called. The default is - * to call exit(-1) if running with one processor and to call MPI_Abort() - * otherwise. This function avoids having to guard abort calls in - * application code. - */ - void abort() const; - - - /** - * Set boolean flag indicating whether exit or abort is called when running - * with one processor. Calling this function influences the behavior of - * calls to abort(). By default, the flag is true meaning that - * abort() will be called. Passing false means exit(-1) will be called. - */ - void setCallAbortInSerialInsteadOfExit( bool flag = true ); - - - /** - * \brief Boolean all reduce - * \details This function performs a boolean all reduce across all processors. - * It returns true iff all processor are true; - * \param value The input value for the all reduce - */ - bool allReduce( const bool value ) const; - - - /** - * \brief Boolean any reduce - * \details This function performs a boolean any reduce across all processors. - * It returns true if any processor is true; - * \param value The input value for the all reduce - */ - bool anyReduce( const bool value ) const; - - - /** - * \brief Sum Reduce - * \details This function performs a sum all reduce across all processor. - * It returns the sum across all processors; - * \param value The input value for the all reduce - */ - template - type sumReduce( const type value ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array sum Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise sum is returned in the same array. - * \param x The input/output array for the reduce - * \param n The number of values in the array (must match on all nodes) - */ - template - void sumReduce( type *x, const int n = 1 ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array sum Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise sum is returned in the same array. - * \param x The input array for the reduce - * \param y The output array for the reduce - * \param n The number of values in the array (must match on all nodes) - */ - template - void sumReduce( const type *x, type *y, const int n = 1 ) const; - - - /** - * \brief Min Reduce - * \details This function performs a min all reduce across all processor. - * It returns the minimum value across all processors; - * \param value The input value for the all reduce - */ - template - type minReduce( const type value ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array min Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise minimum is returned in the same array. - * - * If a 'rank_of_min' argument is provided, it will set the array to the - * rank of process holding the minimum value. Like the double argument, - * the size of the supplied 'rank_of_min' array should be n. - * \param x The input/output array for the reduce - * \param n The number of values in the array (must match on all nodes) - * \param rank_of_min Optional array indicating the rank of the processor containing the - * minimum value - */ - template - void minReduce( type *x, const int n = 1, int *rank_of_min = nullptr ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array min Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise minimum is returned in the same array. - * - * If a 'rank_of_min' argument is provided, it will set the array to the - * rank of process holding the minimum value. Like the double argument, - * the size of the supplied 'rank_of_min' array should be n. - * \param x The input array for the reduce - * \param y The output array for the reduce - * \param n The number of values in the array (must match on all nodes) - * \param rank_of_min Optional array indicating the rank of the processor containing the - * minimum value - */ - template - void minReduce( const type *x, type *y, const int n = 1, int *rank_of_min = nullptr ) const; - - - /** - * \brief Max Reduce - * \details This function performs a max all reduce across all processor. - * It returns the maximum value across all processors; - * \param value The input value for the all reduce - */ - template - type maxReduce( const type value ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array max Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise maximum is returned in the same array. - * - * If a 'rank_of_min' argument is provided, it will set the array to the - * rank of process holding the minimum value. Like the double argument, - * the size of the supplied 'rank_of_min' array should be n. - * \param x The input/output array for the reduce - * \param n The number of values in the array (must match on all nodes) - * \param rank_of_max Optional array indicating the rank of the processor containing the - * minimum value - */ - template - void maxReduce( type *x, const int n = 1, int *rank_of_max = nullptr ) const; - - - /** - * \brief Sum Reduce - * \details Perform an array max Reduce across all nodes. Each - * processor contributes an array of values, and the - * element-wise maximum is returned in the same array. - * - * If a 'rank_of_min' argument is provided, it will set the array to the - * rank of process holding the minimum value. Like the double argument, - * the size of the supplied 'rank_of_min' array should be n. - * \param x The input array for the reduce - * \param y The output array for the reduce - * \param n The number of values in the array (must match on all nodes) - * \param rank_of_max Optional array indicating the rank of the processor containing the - * minimum value - */ - template - void maxReduce( const type *x, type *y, const int n = 1, int *rank_of_max = nullptr ) const; - - - /** - * \brief Scan Sum Reduce - * \details Computes the sum scan (partial reductions) of data on a collection of processes. - * See MPI_Scan for more information. - * \param x The input array for the scan - * \param y The output array for the scan - * \param n The number of values in the array (must match on all nodes) - */ - template - void sumScan( const type *x, type *y, const int n = 1 ) const; - - - /** - * \brief Scan Min Reduce - * \details Computes the min scan (partial reductions) of data on a collection of processes. - * See MPI_Scan for more information. - * \param x The input array for the scan - * \param y The output array for the scan - * \param n The number of values in the array (must match on all nodes) - */ - template - void minScan( const type *x, type *y, const int n = 1 ) const; - - - /** - * \brief Scan Max Reduce - * \details Computes the max scan (partial reductions) of data on a collection of processes. - * See MPI_Scan for more information. - * \param x The input array for the scan - * \param y The output array for the scan - * \param n The number of values in the array (must match on all nodes) - */ - template - void maxScan( const type *x, type *y, const int n = 1 ) const; - - - /** - * \brief Broadcast - * \details This function broadcasts a value from root to all processors - * \param value The input value for the broadcast. - * \param root The processor performing the broadcast - */ - template - type bcast( const type &value, const int root ) const; - - - /** - * \brief Broadcast - * \details This function broadcasts an array from root to all processors - * \param value The input/output array for the broadcast - * \param n The number of values in the array (must match on all nodes) - * \param root The processor performing the broadcast - */ - template - void bcast( type *value, const int n, const int root ) const; - - - /** - * Perform a global barrier across all processors. - */ - void barrier() const; - - - /*! - * @brief This function sends an MPI message with an array to another processor. - * - * If the receiving processor knows in advance the length - * of the array, use "send_length = false;" otherwise, - * this processor will first send the length of the array, - * then send the data. This call must be paired with a - * matching call to recv. - * - * @param buf Pointer to array buffer with length integers. - * @param length Number of integers in buf that we want to send. - * @param recv Receiving processor number. - * @param tag Optional integer argument specifying an integer tag - * to be sent with this message. Default tag is 0. - * The matching recv must share this tag. - */ - template - void send( const type *buf, const int length, const int recv, int tag = 0 ) const; - - - /*! - * @brief This function sends an MPI message with an array of bytes - * (MPI_BYTES) to receiving_proc_number. - * - * This call must be paired with a matching call to recvBytes. - * - * @param buf Void pointer to an array of number_bytes bytes to send. - * @param N_bytes Integer number of bytes to send. - * @param recv Receiving processor number. - * @param tag Optional integer argument specifying an integer tag - * to be sent with this message. Default tag is 0. - * The matching recv must share this tag. - */ - void sendBytes( const void *buf, const int N_bytes, const int recv, int tag = 0 ) const; - - - /*! - * @brief This function sends an MPI message with an array - * to another processor using a non-blocking call. - * The receiving processor must know the length of the array. - * This call must be paired with a matching call to Irecv. - * - * @param buf Pointer to array buffer with length integers. - * @param length Number of integers in buf that we want to send. - * @param recv_proc Receiving processor number. - * @param tag Integer argument specifying an integer tag - * to be sent with this message. - */ - template - MPI_Request Isend( - const type *buf, const int length, const int recv_proc, const int tag ) const; - - - /*! - * @brief This function sends an MPI message with an array of bytes - * (MPI_BYTES) to receiving_proc_number using a non-blocking call. - * The receiving processor must know the number of bytes to receive. - * This call must be paired with a matching call to IrecvBytes. - * - * @param buf Void pointer to an array of number_bytes bytes to send. - * @param N_bytes Integer number of bytes to send. - * @param recv_proc Receiving processor number. - * @param tag Integer argument specifying an integer tag - * to be sent with this message. - */ - MPI_Request IsendBytes( - const void *buf, const int N_bytes, const int recv_proc, const int tag ) const; - - - /*! - * @brief This function receives an MPI message with a data - * array from another processor. - * - * If this processor knows in advance the length of the array, - * use "get_length = false;" otherwise we will get the return size. - * This call must be paired with a matching call to send. - * - * @param buf Pointer to integer array buffer with capacity of length integers. - * @param length If get_length==true: The number of elements to be received, otherwise - * the maximum number of values that can be stored in buf. - * On output the number of received elements. - * @param send Processor number of sender. - * @param tag Optional integer argument specifying a tag which must be matched - * by the tag of the incoming message. Default tag is 0. - */ - template - inline void recv( type *buf, int length, const int send, int tag ) const - { - int length2 = length; - recv( buf, length2, send, false, tag ); - } - - - /*! - * @brief This function receives an MPI message with a data - * array from another processor. - * - * If this processor knows in advance the length of the array, - * use "get_length = false;" otherwise we will get the return size. - * This call must be paired with a matching call to send. - * - * @param buf Pointer to integer array buffer with capacity of length integers. - * @param length If get_length==true: The number of elements to be received, otherwise - * the maximum number of values that can be stored in buf. - * On output the number of received elements. - * @param send Processor number of sender. - * @param get_length Optional boolean argument specifying if we first - * need to check the message size to get the size of the array. - * Default value is true. - * @param tag Optional integer argument specifying a tag which must be matched - * by the tag of the incoming message. Default tag is 0. - */ - template - void recv( type *buf, int &length, const int send, const bool get_length, int tag ) const; - - - /*! - * @brief This function receives an MPI message with an array of - * max size number_bytes (MPI_BYTES) from any processor. - * - * This call must be paired with a matching call to sendBytes. - * - * @param buf Void pointer to a buffer of size number_bytes bytes. - * @param N_bytes Integer number specifying size of buf in bytes. - * @param send Integer number specifying size of buf in bytes. - * @param tag Optional integer argument specifying a tag which - * must be matched by the tag of the incoming message. Default - * tag is 0. - */ - void recvBytes( void *buf, int &N_bytes, const int send, int tag = 0 ) const; - - - /*! - * @brief This function receives an MPI message with a data - * array from another processor using a non-blocking call. - * - * @param buf Pointer to integer array buffer with capacity of length integers. - * @param length Maximum number of values that can be stored in buf. - * @param send_proc Processor number of sender. - * @param tag Optional integer argument specifying a tag which must - * be matched by the tag of the incoming message. - */ - template - MPI_Request Irecv( type *buf, const int length, const int send_proc, const int tag ) const; - - - /*! - * @brief This function receives an MPI message with an array of - * max size number_bytes (MPI_BYTES) from any processor. - * - * This call must be paired with a matching call to sendBytes. - * - * @param buf Void pointer to a buffer of size number_bytes bytes. - * @param N_bytes Integer number specifying size of buf in bytes. - * @param send_proc Processor number of sender. - * @param tag Integer argument specifying a tag which must - * be matched by the tag of the incoming message. - */ - MPI_Request IrecvBytes( - void *buf, const int N_bytes, const int send_proc, const int tag ) const; - - - /*! - * Each processor sends every other processor a single value. - * @param[in] x Input value for allGather - * @return Output array for allGather - */ - template - std::vector allGather( const type &x ) const; - - - /*! - * Each processor sends every other processor an array - * @param[in] x Input array for allGather - * @return Output array for allGather - */ - template - std::vector allGather( const std::vector &x_in ) const; - - - /*! - * Each processor sends every other processor a single value. - * The x_out array should be preallocated to a length equal - * to the number of processors. - * @param x_in Input value for allGather - * @param x_out Output array for allGather (must be preallocated to the size of the - * communicator) - */ - template - void allGather( const type &x_in, type *x_out ) const; - - - /*! - * Each processor sends an array of data to all other processors. - * Each processor receives the values from all processors and gathers them - * to a single array. If successful, the total number of received - * elements will be returned. - * @param send_data Input array - * @param send_cnt The number of values to send - * @param recv_data Output array of received values - * @param recv_cnt The number of values to receive from each processor (N). - * If known, this should be provided as an input. Otherwise - * it is an optional output that will return the number of - * received values from each processor. - * @param recv_disp The displacement (relative to the start of the array) - * from which to store the data received from processor i. - * If known, this should be provided as an input. Otherwise - * it is an optional output that will return the starting location - * (relative to the start of the array) for the received data from - * processor i. - * @param known_recv Are the received counts and displacements known. - * If the received sizes are known, then they must be provided, - * and an extra communication step is not necessary. If the received - * sizes are not known, then an extra communication step will occur - * internally - * and the sizes and displacements will be returned (if desired). - */ - template - int allGather( const type *send_data, const int send_cnt, type *recv_data, - int *recv_cnt = nullptr, int *recv_disp = nullptr, bool known_recv = false ) const; - - - /*! - * This function combines sets from different processors to create a single master set - * @param set Input/Output std::set for the gather. - */ - template - void setGather( std::set &set ) const; - - - /*! - * This function combines std::maps from different processors to create a single master std::map - * If two or more ranks share the same key, the lowest rank will be used - * @param map Input/Output std::map for the gather. - */ - template - void mapGather( std::map &map ) const; - - - /*! - * Each processor sends an array of n values to each processor. - * Each processor sends an array of n values to each processor. - * The jth block of data is sent from processor i to processor j and placed - * in the ith block on the receiving processor. In the variable - * description, N is the size of the communicator. Note that this is a - * blocking global communication. - * @param n The number of elements in each data block to send. - * @param send_data Input array (nxN) - * @param recv_data Output array of received values (nxN) - */ - template - void allToAll( const int n, const type *send_data, type *recv_data ) const; - - - /*! - * Each processor sends an array of data to the different processors. - * Each processor may send any size array to any processor. In the variable - * description, N is the size of the communicator. Note that this is a - * blocking global communication. If successful, the total number of received - * elements will be returned. - * @param send_data Input array - * @param send_cnt The number of values to send to each processor (N) - * @param send_disp The displacement (relative to the start of the array) - * from which to send to processor i - * @param recv_data Output array of received values - * @param recv_cnt The number of values to receive from each processor (N). - * If known, this should be provided as an input. Otherwise - * it is an optional output that will return the number of - * received values from each processor. - * @param recv_disp The displacement (relative to the start of the array) - * from which to send to processor i. - * If known, this should be provided as an input. Otherwise - * it is an optional output that will return the starting location - * (relative to the start of the array) for the received data from - * processor i. - * @param known_recv Are the received counts and displacements known. - * If the received sizes are known, then they must be provided, - * and an extra communication step is not necessary. If the received - * sizes are not know, then an extra communication step will occur - * internally - * and the sizes and displacements will be returned (if desired). - */ - template - int allToAll( const type *send_data, const int send_cnt[], const int send_disp[], - type *recv_data, int *recv_cnt = nullptr, int *recv_disp = nullptr, - bool known_recv = false ) const; - - - /*! - * \brief Send a list of proccesor ids to communicate - * \details This function communicates a list of proccesors to communicate. - * Given a list of ranks that we want to send/receieve data to/from, this routine - * will communicate that set to the other ranks returning the list of processors - * that want to communication with the current rank. - * Note: this routine will involved global communication - * \param ranks List of ranks that the current rank wants to communicate with - * \return List of ranks that want to communicate with the current processor - */ - std::vector commRanks( const std::vector &ranks ) const; - - - /*! - * \brief Wait for a communication to finish - * \details Wait for a communication to finish. - * Note: this does not require a communicator. - * \param request Communication request to wait for (returned for Isend or Irecv) - */ - static void wait( MPI_Request request ); - - - /*! - * \brief Wait for any communication to finish. - * \details This function waits for any of the given communication requests to finish. - * It returns the index of the communication request that finished. - * Note: this does not require a communicator. - * \param count Number of communications to check - * \param request Array of communication requests to wait for (returned for Isend or Irecv) - */ - static int waitAny( int count, MPI_Request *request ); - - - /*! - * \brief Wait for all communications to finish. - * \details This function waits for all of the given communication requests to finish. - * Note: this does not require a communicator. - * \param count Number of communications to check - * \param request Array of communication requests to wait for (returned for Isend or Irecv) - */ - static void waitAll( int count, MPI_Request *request ); - - - /*! - * \brief Wait for some communications to finish. - * \details This function waits for one (or more) communications to finish. - * It returns an array of the indicies that have finished. - * Note: this does not require a communicator. - * \param count Number of communications to check - * \param request Array of communication requests to wait for (returned for Isend or Irecv) - */ - static std::vector waitSome( int count, MPI_Request *request ); - - - /*! - * \brief Nonblocking test for a message - * \details This function performs a non-blocking test for a message. - * It will return the number of bytes in the message if a message with - * the specified source and tag (on the current communicator) is available. - * Otherwise it will return -1. - * \param source source rank (-1: any source) - * \param tag tag (-1: any tag) - */ - int Iprobe( int source = -1, int tag = -1 ) const; - - - /*! - * \brief Blocking test for a message - * \details This function performs a blocking test for a message. - * It will return the number of bytes in the message when a message with - * the specified source and tag (on the current communicator) is available - * \param source source rank (-1: any source) - * \param tag tag (-1: any tag) - */ - int probe( int source = -1, int tag = -1 ) const; - - - /*! - * \brief Start a serial region - * \details This function will serialize MPI processes so that they run - * one at a time. A call to serializeStart must be followed by a call - * to serializeStop after the commands to be executed. - * Note: the ranks will be run in order. - */ - void serializeStart(); - - - /*! - * \brief Stop a serial region - * \details Stop a serial region. See serializeStart for more information. - */ - void serializeStop(); - - - /*! - * \brief Elapsed time - * \details This function returns the elapsed time on the calling processor - * since an arbitrary point in the past (seconds). It is a wrapper to MPI_Wtime. - * See "tick" for the timer resolution in seconds. - * The time may or may not be synchronized across processors depending on the MPI - * implementation. Refer to MPI documentation for the desired platform for more information. - */ - static double time(); - - - /*! - * \brief Timer resolution - * \details This function returns the timer resolution used by "time" - */ - static double tick(); - - - /*! - * \brief Change the level of the internal timers - * \details This function changes the level of the timers used to profile MPI - * \param level New level of the timers - */ - static void changeProfileLevel( int level ) { profile_level = level; } - - - //! Return the total number of MPI_Comm objects that have been created - static size_t MPI_Comm_created() { return N_MPI_Comm_created; } - - //! Return the total number of MPI_Comm objects that have been destroyed - static size_t MPI_Comm_destroyed() { return N_MPI_Comm_destroyed; } - - //! Return details about MPI - static std::string info(); - - //! Return the MPI version number { major, minor } - static std::array version(); - - //! Check if MPI is active - static bool MPI_Active(); - - //! Start MPI - static void start_MPI( int argc_in, char *argv_in[], int profile_level = 0 ); - - //! Stop MPI - static void stop_MPI(); - - -private: // Private helper functions for templated MPI operations; - template - void call_sumReduce( type *x, const int n = 1 ) const; - template - void call_sumReduce( const type *x, type *y, const int n = 1 ) const; - template - void call_minReduce( type *x, const int n = 1, int *rank_of_min = nullptr ) const; - template - void call_minReduce( - const type *x, type *y, const int n = 1, int *rank_of_min = nullptr ) const; - template - void call_maxReduce( type *x, const int n = 1, int *rank_of_max = nullptr ) const; - template - void call_maxReduce( - const type *x, type *y, const int n = 1, int *rank_of_max = nullptr ) const; - template - void call_bcast( type *x, const int n, const int root ) const; - template - void call_allGather( const type &x_in, type *x_out ) const; - template - void call_allGather( - const type *x_in, int size_in, type *x_out, int *size_out, int *disp_out ) const; - template - void call_sumScan( const type *x, type *y, int n = 1 ) const; - template - void call_minScan( const type *x, type *y, int n = 1 ) const; - template - void call_maxScan( const type *x, type *y, int n = 1 ) const; - template - void call_allToAll( const type *send_data, const int send_cnt[], const int send_disp[], - type *recv_data, const int *recv_cnt, const int *recv_disp ) const; - - -private: // data members - // The internal MPI communicator - MPI_Comm communicator; - - // Is the communicator NULL - bool d_isNull; - - // Do we want to manage this communicator - bool d_manage; - - // Do we want to call MPI_abort instead of exit - bool d_call_abort; - - // The level for the profiles of MPI - static short profile_level; - - // The rank and size of the communicator - int comm_rank, comm_size; - - // The ranks of the comm in the global comm - mutable int *volatile d_ranks; - - // Some attributes - int d_maxTag; - int *volatile d_currentTag; - - /* How many objects share the same underlying MPI communicator. - * When the count goes to 0, the MPI comm will be free'd (assuming it was created - * by an communicator). This may not be perfect, but is likely to be good enough. - * Note that for thread safety, any access to this variable should be blocked for thread safety. - * The value of count MUST be volatile to ensure the correct value is always used. - */ - std::atomic_int *volatile d_count; - - // Add a variable for data alignment (necessary for some Intel builds) - double tmp_alignment; - - /* We want to keep track of how many MPI_Comm objects we have created over time. - * Like the count, for thread safety this should be blocked, however the most likely error - * caused by not blocking is a slight error in the MPI count. Since this is just for reference - * we do not need to block (recognizing that the value may not be 100% accurate). - */ - static volatile unsigned int N_MPI_Comm_created; - static volatile unsigned int N_MPI_Comm_destroyed; -}; - - -} // namespace Utilities - - -// Include the default instantiations -// \cond HIDDEN_SYMBOLS -#include "common/MPI.I" -// \endcond - - -#endif diff --git a/common/MPI_Helpers.cpp b/common/MPI_Helpers.cpp new file mode 100644 index 00000000..736a2f02 --- /dev/null +++ b/common/MPI_Helpers.cpp @@ -0,0 +1,266 @@ +#include "common/MPI_Helpers.h" +#include "common/Utilities.h" + + +/******************************************************** +* Return the MPI data type * +********************************************************/ +template<> MPI_Datatype getMPItype() { + return MPI_CHAR; +} +template<> MPI_Datatype getMPItype() { + return MPI_UNSIGNED_CHAR; +} +template<> MPI_Datatype getMPItype() { + return MPI_INT; +} +template<> MPI_Datatype getMPItype() { + return MPI_LONG; +} +template<> MPI_Datatype getMPItype() { + return MPI_UNSIGNED_LONG; +} +template<> MPI_Datatype getMPItype() { + return MPI_LONG_LONG; +} +template<> MPI_Datatype getMPItype() { + return MPI_FLOAT; +} +template<> MPI_Datatype getMPItype() { + return MPI_DOUBLE; +} + + +/******************************************************** +* Concrete implimentations for packing/unpacking * +********************************************************/ +// unsigned char +template<> +size_t packsize( const unsigned char& ) +{ + return sizeof(unsigned char); +} +template<> +void pack( const unsigned char& rhs, char *buffer ) +{ + memcpy(buffer,&rhs,sizeof(unsigned char)); +} +template<> +void unpack( unsigned char& data, const char *buffer ) +{ + memcpy(&data,buffer,sizeof(unsigned char)); +} +// char +template<> +size_t packsize( const char& ) +{ + return sizeof(char); +} +template<> +void pack( const char& rhs, char *buffer ) +{ + memcpy(buffer,&rhs,sizeof(char)); +} +template<> +void unpack( char& data, const char *buffer ) +{ + memcpy(&data,buffer,sizeof(char)); +} +// int +template<> +size_t packsize( const int& ) +{ + return sizeof(int); +} +template<> +void pack( const int& rhs, char *buffer ) +{ + memcpy(buffer,&rhs,sizeof(int)); +} +template<> +void unpack( int& data, const char *buffer ) +{ + memcpy(&data,buffer,sizeof(int)); +} +// unsigned int +template<> +size_t packsize( const unsigned int& ) +{ + return sizeof(unsigned int); +} +template<> +void pack( const unsigned int& rhs, char *buffer ) +{ + memcpy(buffer,&rhs,sizeof(int)); +} +template<> +void unpack( unsigned int& data, const char *buffer ) +{ + memcpy(&data,buffer,sizeof(int)); +} +// size_t +template<> +size_t packsize( const size_t& ) +{ + return sizeof(size_t); +} +template<> +void pack( const size_t& rhs, char *buffer ) +{ + memcpy(buffer,&rhs,sizeof(size_t)); +} +template<> +void unpack( size_t& data, const char *buffer ) +{ + memcpy(&data,buffer,sizeof(size_t)); +} +// std::string +template<> +size_t packsize( const std::string& rhs ) +{ + return rhs.size()+1; +} +template<> +void pack( const std::string& rhs, char *buffer ) +{ + memcpy(buffer,rhs.c_str(),rhs.size()+1); +} +template<> +void unpack( std::string& data, const char *buffer ) +{ + data = std::string(buffer); +} + + +/******************************************************** +* Fake MPI routines * +********************************************************/ +#ifndef USE_MPI +int MPI_Init(int*,char***) +{ + return 0; +} +int MPI_Init_thread(int*,char***, int required, int *provided ) +{ + *provided = required; + return 0; +} +int MPI_Finalize() +{ + return 0; +} +int MPI_Comm_size( MPI_Comm, int *size ) +{ + *size = 1; + return 0; +} +int MPI_Comm_rank( MPI_Comm, int *rank ) +{ + *rank = 0; + return 0; +} +int MPI_Barrier( MPI_Comm ) +{ + return 0; +} +int MPI_Waitall( int, MPI_Request[], MPI_Status[] ) +{ + return 0; +} +int MPI_Wait( MPI_Request*, MPI_Status* ) +{ + return 0; +} +int MPI_Bcast( void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm ) +{ + return 0; +} +int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, + MPI_Comm comm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, + MPI_Comm comm, MPI_Status *status) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, + MPI_Comm comm, MPI_Request *request) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source, + int tag, MPI_Comm comm, MPI_Request *request) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, int recvcount, MPI_Datatype recvtype, + MPI_Comm comm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, MPI_Comm comm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int sendtag, + void *recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int recvtag, + MPI_Comm comm, MPI_Status *status) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, MPI_Comm comm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Comm_group(MPI_Comm comm, MPI_Group *group) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) +{ + ERROR("Not implimented yet"); + return 0; +} +int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) +{ + *newcomm = comm; + return 0; +} +double MPI_Wtime( void ) +{ + return 0.0; +} +int MPI_Comm_free(MPI_Comm *group) +{ + return 0; +} +int MPI_Group_free(MPI_Group *group) +{ + return 0; +} +#endif + + diff --git a/common/MPI_Helpers.h b/common/MPI_Helpers.h new file mode 100644 index 00000000..1d20318e --- /dev/null +++ b/common/MPI_Helpers.h @@ -0,0 +1,239 @@ +// This file contains wrappers for MPI routines and functions to pack/unpack data structures +#ifndef MPI_WRAPPERS_INC +#define MPI_WRAPPERS_INC + +#include +#include +#include +#include + +#ifdef USE_MPI + // Inlcude MPI + #include "mpi.h" +#else + // Create fake MPI types + typedef int MPI_Comm; + typedef int MPI_Request; + typedef int MPI_Status; + #define MPI_COMM_WORLD 0 + #define MPI_COMM_SELF 0 + #define MPI_COMM_NULL -1 + #define MPI_GROUP_NULL -2 + #define MPI_STATUS_IGNORE NULL + enum MPI_Datatype { MPI_LOGICAL, MPI_CHAR, MPI_UNSIGNED_CHAR, MPI_INT, + MPI_UNSIGNED, MPI_LONG, MPI_UNSIGNED_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE }; + enum MPI_Op { MPI_MIN, MPI_MAX, MPI_SUM }; + typedef int MPI_Group; + #define MPI_THREAD_SINGLE 0 + #define MPI_THREAD_FUNNELED 1 + #define MPI_THREAD_SERIALIZED 2 + #define MPI_THREAD_MULTIPLE 3 + // Fake MPI functions + int MPI_Init(int*,char***); + int MPI_Init_thread( int *argc, char ***argv, int required, int *provided ); + int MPI_Finalize(); + int MPI_Comm_size( MPI_Comm, int *size ); + int MPI_Comm_rank( MPI_Comm, int *rank ); + int MPI_Barrier(MPI_Comm); + int MPI_Wait(MPI_Request*,MPI_Status*); + int MPI_Waitall(int,MPI_Request[],MPI_Status[]); + int MPI_Bcast(void*,int,MPI_Datatype,int,MPI_Comm); + int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, + MPI_Comm comm); + int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, + MPI_Comm comm, MPI_Status *status); + int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, + MPI_Comm comm, MPI_Request *request); + int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source, + int tag, MPI_Comm comm, MPI_Request *request); + int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); + int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, int recvcount, MPI_Datatype recvtype, + MPI_Comm comm); + int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, MPI_Comm comm); + int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int sendtag, + void *recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int recvtag, + MPI_Comm comm, MPI_Status *status); + int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, MPI_Comm comm); + double MPI_Wtime( void ); + int MPI_Comm_group(MPI_Comm comm, MPI_Group *group); + int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm); + int MPI_Comm_free(MPI_Comm *group); + int MPI_Group_free(MPI_Group *group); + int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); +#endif + + +//! Get the size of the MPI_Comm +// Note: this is a thread and interrupt safe function +inline int comm_size( MPI_Comm comm ) { + int size = 1; + MPI_Comm_size( comm, &size ); + return size; +} + + +//! Get the rank of the MPI_Comm +// Note: this is a thread and interrupt safe function +inline int comm_rank( MPI_Comm comm ) { + int rank = 1; + MPI_Comm_rank( comm, &rank ); + return rank; +} + + +//! Get the size of MPI_COMM_WORLD +inline int MPI_WORLD_SIZE( ) { + return comm_size( MPI_COMM_WORLD ); +} + +//! Get the size of MPI_COMM_WORLD +inline int MPI_WORLD_RANK( ) { + return comm_rank( MPI_COMM_WORLD ); +} + +//! Return the appropriate MPI datatype for a class +template +MPI_Datatype getMPItype(); + + +//! Template function to return the buffer size required to pack a class +template +size_t packsize( const TYPE& rhs ); + +//! Template function to pack a class to a buffer +template +void pack( const TYPE& rhs, char *buffer ); + +//! Template function to unpack a class from a buffer +template +void unpack( TYPE& data, const char *buffer ); + + +//! Template function to return the buffer size required to pack a std::vector +template +size_t packsize( const std::vector& rhs ); + +//! Template function to pack a class to a buffer +template +void pack( const std::vector& rhs, char *buffer ); + +//! Template function to pack a class to a buffer +template +void unpack( std::vector& data, const char *buffer ); + + +//! Template function to return the buffer size required to pack a std::pair +template +size_t packsize( const std::pair& rhs ); + +//! Template function to pack a class to a buffer +template +void pack( const std::pair& rhs, char *buffer ); + +//! Template function to pack a class to a buffer +template +void unpack( std::pair& data, const char *buffer ); + + +//! Template function to return the buffer size required to pack a std::map +template +size_t packsize( const std::map& rhs ); + +//! Template function to pack a class to a buffer +template +void pack( const std::map& rhs, char *buffer ); + +//! Template function to pack a class to a buffer +template +void unpack( std::map& data, const char *buffer ); + + +//! Template function to return the buffer size required to pack a std::set +template +size_t packsize( const std::set& rhs ); + +//! Template function to pack a class to a buffer +template +void pack( const std::set& rhs, char *buffer ); + +//! Template function to pack a class to a buffer +template +void unpack( std::set& data, const char *buffer ); + + + +// Helper functions +inline double sumReduce( MPI_Comm comm, double x ) +{ + double y = 0; + MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,comm); + return y; +} +inline float sumReduce( MPI_Comm comm, float x ) +{ + float y = 0; + MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_SUM,comm); + return y; +} +inline int sumReduce( MPI_Comm comm, int x ) +{ + int y = 0; + MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm); + return y; +} +inline long long sumReduce( MPI_Comm comm, long long x ) +{ + long long y = 0; + MPI_Allreduce(&x,&y,1,MPI_LONG_LONG,MPI_SUM,comm); + return y; +} +inline bool sumReduce( MPI_Comm comm, bool x ) +{ + int y = sumReduce( comm, x?1:0 ); + return y>0; +} +inline std::vector sumReduce( MPI_Comm comm, const std::vector& x ) +{ + auto y = x; + MPI_Allreduce(x.data(),y.data(),x.size(),MPI_FLOAT,MPI_SUM,comm); + return y; +} +inline std::vector sumReduce( MPI_Comm comm, const std::vector& x ) +{ + auto y = x; + MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm); + return y; +} +inline double maxReduce( MPI_Comm comm, double x ) +{ + double y = 0; + MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_MAX,comm); + return y; +} +inline float maxReduce( MPI_Comm comm, float x ) +{ + float y = 0; + MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm); + return y; +} +inline int maxReduce( MPI_Comm comm, int x ) +{ + int y = 0; + MPI_Allreduce(&x,&y,1,MPI_INT,MPI_MAX,comm); + return y; +} + + +#endif + + +#include "common/MPI_Helpers.hpp" + + diff --git a/IO/PackData.hpp b/common/MPI_Helpers.hpp similarity index 95% rename from IO/PackData.hpp rename to common/MPI_Helpers.hpp index 006cdf73..85261cf1 100644 --- a/IO/PackData.hpp +++ b/common/MPI_Helpers.hpp @@ -1,9 +1,8 @@ -// This file functions to pack/unpack data structures -#ifndef included_PackData_hpp -#define included_PackData_hpp - -#include "IO/PackData.h" +// This file contains wrappers for MPI routines and functions to pack/unpack data structures +#ifndef MPI_WRAPPERS_HPP +#define MPI_WRAPPERS_HPP +#include "common/MPI_Helpers.h" #include #include #include diff --git a/common/ReadMicroCT.cpp b/common/ReadMicroCT.cpp index 2209e712..79ef241e 100644 --- a/common/ReadMicroCT.cpp +++ b/common/ReadMicroCT.cpp @@ -64,11 +64,11 @@ Array readMicroCT( const std::string& filename ) // Read the compressed micro CT data and distribute -Array readMicroCT( const Database& domain, const Utilities::MPI& comm ) +Array readMicroCT( const Database& domain, MPI_Comm comm ) { // Get the local problem info auto n = domain.getVector( "n" ); - int rank = comm.getRank(); + int rank = comm_rank(MPI_COMM_WORLD); auto nproc = domain.getVector( "nproc" ); RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] ); diff --git a/common/ReadMicroCT.h b/common/ReadMicroCT.h index c8acc379..f232740e 100644 --- a/common/ReadMicroCT.h +++ b/common/ReadMicroCT.h @@ -5,12 +5,11 @@ #include "common/Array.h" #include "common/Communication.h" #include "common/Database.h" -#include "common/MPI.h" Array readMicroCT( const std::string& filename ); -Array readMicroCT( const Database& domain, const Utilities::MPI& comm ); +Array readMicroCT( const Database& domain, MPI_Comm comm ); #endif diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6f2966e7..e8a75994 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -5,7 +5,9 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ Lock=false; // unlock the communicator //...................................................................................... // Create a separate copy of the communicator for the device - MPI_COMM_SCALBL = Dm->Comm.dup(); + //MPI_Comm_group(Dm->Comm,&Group); + //MPI_Comm_create(Dm->Comm,Group,&MPI_COMM_SCALBL); + MPI_Comm_dup(Dm->Comm,&MPI_COMM_SCALBL); //...................................................................................... // Copy the domain size and communication information directly from Dm Nx = Dm->Nx; @@ -213,7 +215,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int)); //...................................................................................... - MPI_COMM_SCALBL.barrier(); + MPI_Barrier(MPI_COMM_SCALBL); //................................................................................... // Set up the recieve distribution lists @@ -286,7 +288,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ //................................................................................... //...................................................................................... - MPI_COMM_SCALBL.barrier(); + MPI_Barrier(MPI_COMM_SCALBL); ScaLBL_DeviceBarrier(); //...................................................................................... SendCount = sendCount_x+sendCount_X+sendCount_y+sendCount_Y+sendCount_z+sendCount_Z+ @@ -867,8 +869,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,dist,N); ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,dist,N); - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 5*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 5*recvCount_X,rank_X,recvtag); + MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); //...Packing for X face(1,7,9,11,13)................................ ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,dist,N); ScaLBL_D3Q19_Pack(7,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,dist,N); @@ -876,8 +878,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,dist,N); ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,dist,N); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 5*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 5*recvCount_x,rank_x,recvtag); + MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); //...Packing for y face(4,8,9,16,18)................................. ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,dist,N); ScaLBL_D3Q19_Pack(8,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,dist,N); @@ -885,8 +887,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,dist,N); ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,dist,N); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 5*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 5*recvCount_Y,rank_Y,recvtag); + MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,dist,N); ScaLBL_D3Q19_Pack(7,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); @@ -894,8 +896,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,dist,N); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 5*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 5*recvCount_y,rank_y,recvtag); + MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); //...Packing for z face(6,12,13,16,17)................................ ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,dist,N); ScaLBL_D3Q19_Pack(12,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,dist,N); @@ -903,8 +905,8 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,dist,N); ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,dist,N); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 5*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 5*recvCount_Z,rank_Z,recvtag); + MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); //...Packing for Z face(5,11,14,15,18)................................ ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,dist,N); @@ -913,57 +915,57 @@ void ScaLBL_Communicator::SendD3Q19AA(double *dist){ ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N); ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,dist,N); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 5*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 5*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); //...Pack the xy edge (8)................................ ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,dist,N); - req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]); //...Pack the Xy edge (9)................................ ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,dist,N); - req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]); //...Pack the xY edge (10)................................ ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,dist,N); - req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]); //...Pack the XY edge (7)................................ ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,dist,N); - req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]); //...Pack the xz edge (12)................................ ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,dist,N); - req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]); //...Pack the xZ edge (14)................................ ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,dist,N); - req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]); //...Pack the Xz edge (13)................................ ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,dist,N); - req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]); //...Pack the XZ edge (11)................................ ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,dist,N); - req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]); //...Pack the yz edge (16)................................ ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,dist,N); - req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]); //...Pack the yZ edge (18)................................ ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,dist,N); - req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); //...Pack the Yz edge (17)................................ ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,dist,N); - req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); //...Pack the YZ edge (15)................................ ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,dist,N); - req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]); //................................................................................... } @@ -973,8 +975,8 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1057,8 +1059,8 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ // Recieves halo and incorporates into D3Q19 based stencil gradient computation //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1151,36 +1153,36 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N); ScaLBL_D3Q19_Pack(2,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,Bq,N); - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); + MPI_Isend(sendbuf_x, 2*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 2*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); //...Packing for X face(1,7,9,11,13)................................ ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N); ScaLBL_D3Q19_Pack(1,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,Bq,N); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); + MPI_Isend(sendbuf_X, 2*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 2*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); //...Packing for y face(4,8,9,16,18)................................. ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N); ScaLBL_D3Q19_Pack(4,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,Bq,N); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N); ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); + MPI_Isend(sendbuf_Y, 2*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 2*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); //...Packing for z face(6,12,13,16,17)................................ ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N); ScaLBL_D3Q19_Pack(6,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,Bq,N); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); + MPI_Isend(sendbuf_z, 2*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 2*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); //...Packing for Z face(5,11,14,15,18)................................ ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N); @@ -1188,8 +1190,8 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ //................................................................................... // Send all the distributions - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_Z, 2*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 2*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); } @@ -1199,8 +1201,8 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(6,req1); - MPI_COMM_SCALBL.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1291,18 +1293,18 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){ //................................................................................... // Send all the distributions - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 3*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 3*recvCount_X,rank_X,recvtag); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 3*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 3*recvCount_x,rank_x,recvtag); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 3*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 3*recvCount_Y,rank_Y,recvtag); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 3*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 3*recvCount_y,rank_y,recvtag); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 3*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 3*recvCount_Z,rank_Z,recvtag); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 3*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 3*recvCount_z,rank_z,recvtag); + MPI_Isend(sendbuf_x, 3*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, 3*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); + MPI_Isend(sendbuf_X, 3*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, 3*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); + MPI_Isend(sendbuf_y, 3*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 3*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); + MPI_Isend(sendbuf_Y, 3*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, 3*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); + MPI_Isend(sendbuf_z, 3*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, 3*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); + MPI_Isend(sendbuf_Z, 3*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, 3*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); } @@ -1312,8 +1314,8 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 //................................................................................... // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(6,req1); - MPI_COMM_SCALBL.waitAll(6,req2); + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... @@ -1407,49 +1409,49 @@ void ScaLBL_Communicator::SendHalo(double *data){ // Send / Recv all the phase indcator field values //................................................................................... - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); - req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, sendCount_xy,rank_xy,sendtag); - req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, recvCount_XY,rank_XY,recvtag); - req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, sendCount_XY,rank_XY,sendtag); - req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, recvCount_xy,rank_xy,recvtag); - req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, sendCount_Xy,rank_Xy,sendtag); - req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, recvCount_xY,rank_xY,recvtag); - req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, sendCount_xY,rank_xY,sendtag); - req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, recvCount_Xy,rank_Xy,recvtag); - req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, sendCount_xz,rank_xz,sendtag); - req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, recvCount_XZ,rank_XZ,recvtag); - req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, sendCount_XZ,rank_XZ,sendtag); - req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, recvCount_xz,rank_xz,recvtag); - req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, sendCount_Xz,rank_Xz,sendtag); - req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, recvCount_xZ,rank_xZ,recvtag); - req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, sendCount_xZ,rank_xZ,sendtag); - req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, recvCount_Xz,rank_Xz,recvtag); - req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, sendCount_yz,rank_yz,sendtag); - req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, recvCount_YZ,rank_YZ,recvtag); - req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, sendCount_YZ,rank_YZ,sendtag); - req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, recvCount_yz,rank_yz,recvtag); - req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, sendCount_Yz,rank_Yz,sendtag); - req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, recvCount_yZ,rank_yZ,recvtag); - req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, sendCount_yZ,rank_yZ,sendtag); - req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, recvCount_Yz,rank_Yz,recvtag); + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); + MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); + MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]); + MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]); + MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]); + MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]); + MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]); + MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]); + MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]); + MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]); + MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]); + MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]); + MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]); + MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]); + MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]); + MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]); + MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]); + MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]); + MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]); + MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]); + MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]); + MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]); + MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]); + MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); + MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); + MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); //................................................................................... } void ScaLBL_Communicator::RecvHalo(double *data){ //................................................................................... - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); + MPI_Waitall(18,req1,stat1); + MPI_Waitall(18,req2,stat2); ScaLBL_DeviceBarrier(); //................................................................................... //................................................................................... @@ -1562,7 +1564,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl LocInletArea = double(sendCount_z); else LocInletArea = 0.f; - InletArea = MPI_COMM_SCALBL.sumReduce( LocInletArea ); + MPI_Allreduce(&LocInletArea,&InletArea,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL); //printf("Inlet area = %f \n", InletArea); // Set the flux BC @@ -1571,7 +1573,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl if (kproc == 0) locsum = ScaLBL_D3Q19_AAeven_Flux_BC_z(dvcSendList_z, fq, flux, InletArea, sendCount_z, N); - sum = MPI_COMM_SCALBL.sumReduce( locsum ); + MPI_Allreduce(&locsum,&sum,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL); din = flux/InletArea + sum; //if (rank==0) printf("computed din (even) =%f \n",din); if (kproc == 0) @@ -1581,7 +1583,7 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl if (kproc == 0) locsum = ScaLBL_D3Q19_AAodd_Flux_BC_z(neighborList, dvcSendList_z, fq, flux, InletArea, sendCount_z, N); - sum = MPI_COMM_SCALBL.sumReduce( locsum ); + MPI_Allreduce(&locsum,&sum,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_SCALBL); din = flux/InletArea + sum; //if (rank==0) printf("computed din (odd)=%f \n",din); diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 51195f5a..c737659c 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -202,8 +202,9 @@ private: // Give the object it's own MPI communicator RankInfoStruct rank_info; MPI_Group Group; // Group of processors associated with this domain - Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain + MPI_Comm MPI_COMM_SCALBL; // MPI Communicator for this domain MPI_Request req1[18],req2[18]; + MPI_Status stat1[18],stat2[18]; //...................................................................................... // MPI ranks for all 18 neighbors //...................................................................................... diff --git a/common/SpherePack.cpp b/common/SpherePack.cpp index 18057653..a7246b72 100644 --- a/common/SpherePack.cpp +++ b/common/SpherePack.cpp @@ -9,6 +9,7 @@ #include "common/Array.h" #include "common/Utilities.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "common/Database.h" #include "common/SpherePack.h" diff --git a/common/SpherePack.h b/common/SpherePack.h index 56284a40..5075b289 100644 --- a/common/SpherePack.h +++ b/common/SpherePack.h @@ -12,6 +12,7 @@ #include "common/Array.h" #include "common/Utilities.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "common/Database.h" diff --git a/common/UnitTest.cpp b/common/UnitTest.cpp index aeb9026e..b995fa68 100755 --- a/common/UnitTest.cpp +++ b/common/UnitTest.cpp @@ -14,49 +14,44 @@ /******************************************************************** * Constructor/Destructor * ********************************************************************/ -UnitTest::UnitTest() : d_verbose( false ), d_comm( MPI_COMM_SELF ) +UnitTest::UnitTest() { - if ( Utilities::MPI::MPI_active() ) - d_comm = MPI_COMM_WORLD; +#ifdef USE_MPI + comm = MPI_COMM_WORLD; +#endif } UnitTest::~UnitTest() { reset(); } void UnitTest::reset() { - d_mutex.lock(); + mutex.lock(); // Clear the data forcing a reallocation - std::vector().swap( d_pass ); - std::vector().swap( d_fail ); - std::vector().swap( d_expected ); - d_mutex.unlock(); + std::vector().swap( pass_messages ); + std::vector().swap( fail_messages ); + std::vector().swap( expected_fail_messages ); + mutex.unlock(); } /******************************************************************** * Add a pass, fail, expected failure message in a thread-safe way * ********************************************************************/ -void UnitTest::passes( std::string in ) +void UnitTest::passes( const std::string &in ) { - d_mutex.lock(); - if ( d_verbose ) - printf( "UnitTest: %i passes: %s\n", d_comm.getRank(), in.data() ); - d_pass.emplace_back( std::move( in ) ); - d_mutex.unlock(); + mutex.lock(); + pass_messages.push_back( in ); + mutex.unlock(); } -void UnitTest::failure( std::string in ) +void UnitTest::failure( const std::string &in ) { - d_mutex.lock(); - if ( d_verbose ) - printf( "UnitTest: %i failed: %s\n", d_comm.getRank(), in.data() ); - d_fail.emplace_back( std::move( in ) ); - d_mutex.unlock(); + mutex.lock(); + fail_messages.push_back( in ); + mutex.unlock(); } -void UnitTest::expected_failure( std::string in ) +void UnitTest::expected_failure( const std::string &in ) { - d_mutex.lock(); - if ( d_verbose ) - printf( "UnitTest: %i expected_failure: %s\n", d_comm.getRank(), in.data() ); - d_expected.emplace_back( std::move( in ) ); - d_mutex.unlock(); + mutex.lock(); + expected_fail_messages.push_back( in ); + mutex.unlock(); } @@ -64,6 +59,23 @@ void UnitTest::expected_failure( std::string in ) * Print a global report * * Note: only rank 0 will print, all messages will be aggregated * ********************************************************************/ +inline std::vector UnitTest::allGather( int value ) const +{ + int size = getSize(); + std::vector data( size, value ); +#ifdef USE_MPI + if ( size > 1 ) + MPI_Allgather( &value, 1, MPI_INT, data.data(), 1, MPI_INT, comm ); +#endif + return data; +} +inline void UnitTest::barrier() const +{ +#ifdef USE_MPI + if ( getSize() > 1 ) + MPI_Barrier( comm ); +#endif +} static inline void print_messages( const std::vector> &messages ) { if ( messages.size() > 1 ) { @@ -81,27 +93,28 @@ static inline void print_messages( const std::vector> & } void UnitTest::report( const int level0 ) const { - d_mutex.lock(); - int size = d_comm.getSize(); - int rank = d_comm.getRank(); - // Give all processors a chance to print any remaining messages - d_comm.barrier(); - Utilities::sleep_ms( 10 ); + mutex.lock(); + int size = getSize(); + int rank = getRank(); // Broadcast the print level from rank 0 - int level = d_comm.bcast( level0, 0 ); + int level = level0; +#ifdef USE_MPI + if ( getSize() > 1 ) + MPI_Bcast( &level, 1, MPI_INT, 0, comm ); +#endif if ( level < 0 || level > 2 ) ERROR( "Invalid print level" ); // Perform a global all gather to get the number of failures per processor - auto N_pass = d_comm.allGather( d_pass.size() ); - auto N_fail = d_comm.allGather( d_fail.size() ); - auto N_expected = d_comm.allGather( d_expected.size() ); - int N_pass_tot = 0; - int N_fail_tot = 0; - int N_expected_tot = 0; + auto N_pass = allGather( pass_messages.size() ); + auto N_fail = allGather( fail_messages.size() ); + auto N_expected_fail = allGather( expected_fail_messages.size() ); + int N_pass_tot = 0; + int N_fail_tot = 0; + int N_expected_fail_tot = 0; for ( int i = 0; i < size; i++ ) { N_pass_tot += N_pass[i]; N_fail_tot += N_fail[i]; - N_expected_tot += N_expected[i]; + N_expected_fail_tot += N_expected_fail[i]; } // Send all messages to rank 0 (if needed) std::vector> pass_messages_rank( size ); @@ -109,13 +122,13 @@ void UnitTest::report( const int level0 ) const std::vector> expected_fail_rank( size ); // Get the pass messages if ( ( level == 1 && N_pass_tot <= 20 ) || level == 2 ) - pass_messages_rank = UnitTest::gatherMessages( d_pass, 1 ); + pass_messages_rank = UnitTest::gatherMessages( pass_messages, 1 ); // Get the fail messages if ( level == 1 || level == 2 ) - fail_messages_rank = UnitTest::gatherMessages( d_fail, 2 ); + fail_messages_rank = UnitTest::gatherMessages( fail_messages, 2 ); // Get the expected_fail messages - if ( ( level == 1 && N_expected_tot <= 50 ) || level == 2 ) - expected_fail_rank = UnitTest::gatherMessages( d_expected, 2 ); + if ( ( level == 1 && N_expected_fail_tot <= 50 ) || level == 2 ) + expected_fail_rank = UnitTest::gatherMessages( expected_fail_messages, 2 ); // Print the results of all messages (only rank 0 will print) if ( rank == 0 ) { pout << std::endl; @@ -161,31 +174,31 @@ void UnitTest::report( const int level0 ) const pout << std::endl; // Print the tests that expected failed pout << "Tests expected failed" << std::endl; - if ( level == 0 || ( level == 1 && N_expected_tot > 50 ) ) { + if ( level == 0 || ( level == 1 && N_expected_fail_tot > 50 ) ) { // We want to print a summary if ( size > 8 ) { // Print 1 summary for all processors printp( " %i tests expected failed (use report level 2 for more detail)\n", - N_expected_tot ); + N_expected_fail_tot ); } else { // Print a summary for each processor for ( int i = 0; i < size; i++ ) printp( " %i tests expected failed (proc %i) (use report level 2 for more " "detail)\n", - N_expected[i], i ); + N_expected_fail[i], i ); } } else { // We want to print all messages for ( int i = 0; i < size; i++ ) - ASSERT( (int) expected_fail_rank[i].size() == N_expected[i] ); + ASSERT( (int) expected_fail_rank[i].size() == N_expected_fail[i] ); print_messages( expected_fail_rank ); } pout << std::endl; } // Add a barrier to synchronize all processors (rank 0 is much slower) - d_comm.barrier(); + barrier(); Utilities::sleep_ms( 10 ); // Need a brief pause to allow any printing to finish - d_mutex.unlock(); + mutex.unlock(); } @@ -195,8 +208,8 @@ void UnitTest::report( const int level0 ) const std::vector> UnitTest::gatherMessages( const std::vector &local_messages, int tag ) const { - const int rank = d_comm.getRank(); - const int size = d_comm.getSize(); + const int rank = getRank(); + const int size = getSize(); std::vector> messages( size ); if ( rank == 0 ) { // Rank 0 should receive all messages @@ -220,6 +233,7 @@ std::vector> UnitTest::gatherMessages( void UnitTest::pack_message_stream( const std::vector &messages, const int rank, const int tag ) const { +#ifdef USE_MPI // Get the size of the messages auto N_messages = (int) messages.size(); auto *msg_size = new int[N_messages]; @@ -240,11 +254,18 @@ void UnitTest::pack_message_stream( k += msg_size[i]; } // Send the message stream (using a non-blocking send) - auto request = d_comm.Isend( data, size_data, rank, tag ); + MPI_Request request; + MPI_Isend( data, size_data, MPI_CHAR, rank, tag, comm, &request ); // Wait for the communication to send and free the temporary memory - d_comm.wait( request ); + MPI_Status status; + MPI_Wait( &request, &status ); delete[] data; delete[] msg_size; +#else + NULL_USE( messages ); + NULL_USE( rank ); + NULL_USE( tag ); +#endif } @@ -253,15 +274,20 @@ void UnitTest::pack_message_stream( ********************************************************************/ std::vector UnitTest::unpack_message_stream( const int rank, const int tag ) const { +#ifdef USE_MPI // Probe the message to get the message size - int size_data = d_comm.probe( rank, tag ); + MPI_Status status; + MPI_Probe( rank, tag, comm, &status ); + int size_data = -1; + MPI_Get_count( &status, MPI_BYTE, &size_data ); ASSERT( size_data >= 0 ); // Allocate memory to receive the data auto *data = new char[size_data]; // receive the data (using a non-blocking receive) - auto request = d_comm.Irecv( data, size_data, rank, tag ); + MPI_Request request; + MPI_Irecv( data, size_data, MPI_CHAR, rank, tag, comm, &request ); // Wait for the communication to be received - d_comm.wait( request ); + MPI_Wait( &request, &status ); // Unpack the message stream int N_messages = 0; memcpy( &N_messages, data, sizeof( int ) ); @@ -277,16 +303,77 @@ std::vector UnitTest::unpack_message_stream( const int rank, const messages[i] = std::string( &data[k], msg_size[i] ); k += msg_size[i]; } - // Delete the temporary memory delete[] data; return messages; +#else + NULL_USE( rank ); + NULL_USE( tag ); + return std::vector(); +#endif } /******************************************************************** * Other functions * ********************************************************************/ -size_t UnitTest::NumPassGlobal() const { return d_comm.sumReduce( d_pass.size() ); } -size_t UnitTest::NumFailGlobal() const { return d_comm.sumReduce( d_fail.size() ); } -size_t UnitTest::NumExpectedFailGlobal() const { return d_comm.sumReduce( d_expected.size() ); } - +int UnitTest::getRank() const +{ + int rank = 0; +#ifdef USE_MPI + int flag = 0; + MPI_Initialized( &flag ); + if ( flag ) + MPI_Comm_rank( comm, &rank ); +#endif + return rank; +} +int UnitTest::getSize() const +{ + int size = 1; +#ifdef USE_MPI + int flag = 0; + MPI_Initialized( &flag ); + if ( flag ) + MPI_Comm_size( comm, &size ); +#endif + return size; +} +size_t UnitTest::NumPassGlobal() const +{ + size_t num = pass_messages.size(); +#ifdef USE_MPI + if ( getSize() > 1 ) { + auto send = static_cast( num ); + int sum = 0; + MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm ); + num = static_cast( sum ); + } +#endif + return num; +} +size_t UnitTest::NumFailGlobal() const +{ + size_t num = fail_messages.size(); +#ifdef USE_MPI + if ( getSize() > 1 ) { + auto send = static_cast( num ); + int sum = 0; + MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm ); + num = static_cast( sum ); + } +#endif + return num; +} +size_t UnitTest::NumExpectedFailGlobal() const +{ + size_t num = expected_fail_messages.size(); +#ifdef USE_MPI + if ( getSize() > 1 ) { + auto send = static_cast( num ); + int sum = 0; + MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm ); + num = static_cast( sum ); + } +#endif + return num; +} diff --git a/common/UnitTest.h b/common/UnitTest.h index 9d452747..80503d19 100755 --- a/common/UnitTest.h +++ b/common/UnitTest.h @@ -1,11 +1,13 @@ #ifndef included_UnitTest #define included_UnitTest -#include "common/MPI.h" - #include +#include #include #include +#ifdef USE_MPI +#include "mpi.h" +#endif /*! @@ -26,47 +28,47 @@ * \endcode */ -class UnitTest final +class UnitTest { public: //! Constructor UnitTest(); //! Destructor - ~UnitTest(); - - // Copy constructor - UnitTest( const UnitTest & ) = delete; - - // Assignment operator - UnitTest &operator=( const UnitTest & ) = delete; + virtual ~UnitTest(); //! Indicate a passed test (thread-safe) - void passes( std::string in ); + virtual void passes( const std::string &in ); //! Indicate a failed test (thread-safe) - void failure( std::string in ); + virtual void failure( const std::string &in ); //! Indicate an expected failed test (thread-safe) - void expected_failure( std::string in ); + virtual void expected_failure( const std::string &in ); //! Return the number of passed tests locally - inline size_t NumPassLocal() const { return d_pass.size(); } + virtual size_t NumPassLocal() const { return pass_messages.size(); } //! Return the number of failed tests locally - inline size_t NumFailLocal() const { return d_fail.size(); } + virtual size_t NumFailLocal() const { return fail_messages.size(); } //! Return the number of expected failed tests locally - inline size_t NumExpectedFailLocal() const { return d_expected.size(); } + virtual size_t NumExpectedFailLocal() const { return expected_fail_messages.size(); } //! Return the number of passed tests locally - size_t NumPassGlobal() const; + virtual size_t NumPassGlobal() const; //! Return the number of failed tests locally - size_t NumFailGlobal() const; + virtual size_t NumFailGlobal() const; //! Return the number of expected failed tests locally - size_t NumExpectedFailGlobal() const; + virtual size_t NumExpectedFailGlobal() const; + + //! Return the rank of the current processor + int getRank() const; + + //! Return the number of processors + int getSize() const; /*! * Print a report of the passed and failed tests. @@ -75,28 +77,29 @@ public: * to print correctly). * @param level Optional integer specifying the level of reporting (default: 1) * 0: Report the number of tests passed, failed, and expected failures. - * 1: Report the passed tests (if <=20) or number passed, - * Report all failures, - * Report the expected failed tests (if <=50) or the number passed. + * 1: Report the number of passed tests (if <=20) or the number passed + * otherwise, report all failures, report the number of expected + * failed tests (if <=50) or the number passed otherwise. * 2: Report all passed, failed, and expected failed tests. */ - void report( const int level = 1 ) const; + virtual void report( const int level = 1 ) const; //! Clear the messages void reset(); - //! Make the unit test operator verbose? - void verbose( bool verbose = true ) { d_verbose = verbose; } +protected: + std::vector pass_messages; + std::vector fail_messages; + std::vector expected_fail_messages; + mutable std::mutex mutex; +#ifdef USE_MPI + MPI_Comm comm; +#endif private: - std::vector d_pass; - std::vector d_fail; - std::vector d_expected; - bool d_verbose; - mutable std::mutex d_mutex; - Utilities::MPI d_comm; + // Make the copy constructor private + UnitTest( const UnitTest & ) {} -private: // Function to pack the messages into a single data stream and send to the given processor // Note: This function does not return until the message stream has been sent void pack_message_stream( @@ -106,7 +109,9 @@ private: // Note: This function does not return until the message stream has been received std::vector unpack_message_stream( const int rank, const int tag ) const; - // Gather the messages + // Helper functions + inline void barrier() const; + inline std::vector allGather( int value ) const; inline std::vector> gatherMessages( const std::vector &local_messages, int tag ) const; }; diff --git a/common/UtilityMacros.h b/common/UtilityMacros.h index 2c374ef1..bfac172f 100644 --- a/common/UtilityMacros.h +++ b/common/UtilityMacros.h @@ -143,43 +143,35 @@ * Be sure to follow with ENABLE_WARNINGS */ // clang-format off -#ifndef DISABLE_WARNINGS -#if defined( USING_MSVC ) +#ifdef DISABLE_WARNINGS + // Macros previously defined +#elif defined( USING_MSVC ) #define DISABLE_WARNINGS __pragma( warning( push, 0 ) ) #define ENABLE_WARNINGS __pragma( warning( pop ) ) #elif defined( USING_CLANG ) #define DISABLE_WARNINGS \ - _Pragma( "clang diagnostic push" ) \ - _Pragma( "clang diagnostic ignored \"-Wall\"" ) \ + _Pragma( "clang diagnostic push" ) _Pragma( "clang diagnostic ignored \"-Wall\"" ) \ _Pragma( "clang diagnostic ignored \"-Wextra\"" ) \ _Pragma( "clang diagnostic ignored \"-Wunused-private-field\"" ) \ - _Pragma( "clang diagnostic ignored \"-Wdeprecated-declarations\"" ) \ - _Pragma( "clang diagnostic ignored \"-Winteger-overflow\"" ) + _Pragma( "clang diagnostic ignored \"-Wmismatched-new-delete\"" ) #define ENABLE_WARNINGS _Pragma( "clang diagnostic pop" ) #elif defined( USING_GCC ) + // Note: We cannot disable the -Wliteral-suffix message with this macro because the + // pragma command cannot suppress warnings from the C++ preprocessor. See gcc bug #53431. #define DISABLE_WARNINGS \ - _Pragma( "GCC diagnostic push" ) \ - _Pragma( "GCC diagnostic ignored \"-Wpragmas\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wall\"" ) \ + _Pragma( "GCC diagnostic push" ) _Pragma( "GCC diagnostic ignored \"-Wall\"" ) \ _Pragma( "GCC diagnostic ignored \"-Wextra\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wpedantic\"" ) \ + _Pragma( "GCC diagnostic ignored \"-Wpragmas\"" ) \ _Pragma( "GCC diagnostic ignored \"-Wunused-local-typedefs\"" ) \ _Pragma( "GCC diagnostic ignored \"-Woverloaded-virtual\"" ) \ _Pragma( "GCC diagnostic ignored \"-Wunused-parameter\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wdeprecated-declarations\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wvirtual-move-assign\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wunused-function\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Woverflow\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wunused-variable\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wignored-qualifiers\"" ) \ - _Pragma( "GCC diagnostic ignored \"-Wenum-compare\"" ) \ + _Pragma( "GCC diagnostic ignored \"-Warray-bounds\"" ) \ _Pragma( "GCC diagnostic ignored \"-Wterminate\"" ) #define ENABLE_WARNINGS _Pragma( "GCC diagnostic pop" ) #else #define DISABLE_WARNINGS #define ENABLE_WARNINGS #endif -#endif // clang-format on diff --git a/cpu/BGK.cpp b/cpu/BGK.cpp index bccc5b77..436ab381 100644 --- a/cpu/BGK.cpp +++ b/cpu/BGK.cpp @@ -1,4 +1,5 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; // conserved momemnts double rho,ux,uy,uz,uu; // non-conserved moments @@ -110,12 +111,14 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int } extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; // conserved momemnts double rho,ux,uy,uz,uu; // non-conserved moments double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18; + int nread; for (int n=start; n 0 ){ // Get the density value (Streaming already performed) - double Na = Den[n]; - double Nb = Den[N+n]; + Na = Den[n]; + Nb = Den[N+n]; Phi[n] = (Na-Nb)/(Na+Nb); } } + //................................................................... } extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice){ - for (int n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){ + int n; + for (n=Slice*Nx*Ny; n<(Slice+1)*Nx*Ny; n++){ Phi[n] = value; } } @@ -1246,7 +1255,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - int ijk,nn; + int ijk,nn,n; double fq; // conserved momemnts double rho,jx,jy,jz; @@ -1829,7 +1838,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ - int nn,ijk,nread; + int n,nn,ijk,nread; int nr1,nr2,nr3,nr4,nr5,nr6; int nr7,nr8,nr9,nr10; int nr11,nr12,nr13,nr14; @@ -2483,7 +2492,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - int idx, nread; + int idx,n,nread; double fq,nA,nB; for (int n=start; n #include -ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM): +ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) @@ -167,9 +167,9 @@ void ScaLBL_ColorModel::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object - comm.barrier(); + MPI_Barrier(comm); Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); // Read domain parameters rank = Dm->rank(); nprocx = Dm->nprocx(); @@ -292,7 +292,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) for (int i=0; iid[i] = Mask->id[i]; for (size_t idx=0; idxComm.sumReduce( label_count[idx] ); + label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]); if (rank==0){ printf("Component labels: %lu \n",NLABELS); @@ -333,7 +333,7 @@ void ScaLBL_ColorModel::Create(){ Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE @@ -465,7 +465,7 @@ void ScaLBL_ColorModel::Initialize(){ ScaLBL_CopyToDevice(Phi,cPhi,N*sizeof(double)); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); } if (rank==0) printf ("Initializing phase field \n"); @@ -651,7 +651,7 @@ void ScaLBL_ColorModel::Run(){ //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -700,8 +700,7 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; @@ -736,10 +735,10 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ + MPI_Barrier(comm); PROFILE_STOP("Update"); if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ @@ -980,7 +979,7 @@ void ScaLBL_ColorModel::Run(){ //morph_delta *= (-1.0); REVERSE_FLOW_DIRECTION = false; } - comm.barrier(); + MPI_Barrier(comm); } morph_timesteps += analysis_interval; } @@ -990,7 +989,7 @@ void ScaLBL_ColorModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -1035,17 +1034,17 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ } } - Count = Dm->Comm.sumReduce( Count ); - PoreCount = Dm->Comm.sumReduce( PoreCount ); + Count=sumReduce( Dm->Comm, Count); + PoreCount=sumReduce( Dm->Comm, PoreCount); if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); - comm.barrier(); + MPI_Barrier(comm); ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - comm.barrier(); + MPI_Barrier(comm); ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); @@ -1077,7 +1076,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ BlobIDstruct new_index; double vF=0.0; double vS=0.0; ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); long long count_connected=0; long long count_porespace=0; @@ -1099,9 +1098,9 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ } } } - count_connected = Dm->Comm.sumReduce( count_connected); - count_porespace = Dm->Comm.sumReduce( count_porespace); - count_water = Dm->Comm.sumReduce( count_water); + count_connected=sumReduce( Dm->Comm, count_connected); + count_porespace=sumReduce( Dm->Comm, count_porespace); + count_water=sumReduce( Dm->Comm, count_water); for (int k=0; kComm.sumReduce( count_morphopen); + count_morphopen=sumReduce( Dm->Comm, count_morphopen); volume_change = double(count_morphopen - count_connected); if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); @@ -1279,8 +1278,8 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ mass_loss += random_value*seed_water_in_oil; } - count = Dm->Comm.sumReduce( count ); - mass_loss = Dm->Comm.sumReduce( mass_loss ); + count= sumReduce( Dm->Comm, count); + mass_loss= sumReduce( Dm->Comm, mass_loss); if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); // Need to initialize Aq, Bq, Den, Phi directly @@ -1317,7 +1316,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta } } } - double volume_initial = Dm->Comm.sumReduce( count); + double volume_initial = sumReduce( Dm->Comm, count); /* sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); FILE *INPUT = fopen(LocalRankFilename,"wb"); @@ -1327,7 +1326,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta // 2. Identify connected components of phase field -> phase_label BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - comm.barrier(); + MPI_Barrier(comm); // only operate on component "0" count = 0.0; @@ -1349,8 +1348,8 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta } } } - double volume_connected = Dm->Comm.sumReduce( count ); - second_biggest = Dm->Comm.sumReduce( second_biggest ); + double volume_connected = sumReduce( Dm->Comm, count); + second_biggest = sumReduce( Dm->Comm, second_biggest); /*int reach_x, reach_y, reach_z; for (int k=0; kComm.sumReduce( count ); + double volume_final= sumReduce( Dm->Comm, count); delta_volume = (volume_final-volume_initial); if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); diff --git a/models/ColorModel.h b/models/ColorModel.h index c52f04c3..a3b3a124 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -12,13 +12,13 @@ Implementation of color lattice boltzmann model #include "common/Communication.h" #include "analysis/TwoPhase.h" #include "analysis/runAnalysis.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" class ScaLBL_ColorModel{ public: - ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM); + ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM); ~ScaLBL_ColorModel(); // functions in they should be run @@ -68,7 +68,7 @@ public: double *Pressure; private: - Utilities::MPI comm; + MPI_Comm comm; int dist_mem_size; int neighborSize; diff --git a/models/DFHModel.cpp b/models/DFHModel.cpp index ced5853f..4eb03bea 100644 --- a/models/DFHModel.cpp +++ b/models/DFHModel.cpp @@ -3,7 +3,7 @@ color lattice boltzmann model */ #include "models/DFHModel.h" -ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM): +ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) @@ -100,16 +100,16 @@ void ScaLBL_DFHModel::ReadParams(string filename){ } void ScaLBL_DFHModel::SetDomain(){ - Dm = std::make_shared(domain_db,comm); // full domain for analysis - Mask = std::make_shared(domain_db,comm); // mask domain removes immobile phases + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; id = new char [N]; - for (int i=0; iid[i] = 1; // initialize this way - Averages = std::make_shared( Dm ); // TwoPhase analysis object - comm.barrier(); + for (int i=0; iid[i] = 1; // initialize this way + Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); rank = Dm->rank(); } @@ -131,7 +131,7 @@ void ScaLBL_DFHModel::ReadInput(){ sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; } @@ -206,7 +206,7 @@ void ScaLBL_DFHModel::Create(){ Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE @@ -424,7 +424,7 @@ void ScaLBL_DFHModel::Initialize(){ } } } - count_wet_global = comm.sumReduce( count_wet ); + MPI_Allreduce(&count_wet,&count_wet_global,1,MPI_DOUBLE,MPI_SUM,comm); if (rank==0) printf("Wetting phase volume fraction =%f \n",count_wet_global/double(Nx*Ny*Nz*nprocs)); // initialize phi based on PhaseLabel (include solid component labels) ScaLBL_CopyToDevice(Phi, PhaseLabel, Np*sizeof(double)); @@ -446,7 +446,7 @@ void ScaLBL_DFHModel::Initialize(){ timestep=0; } } - comm.bcast(×tep,1,0); + MPI_Bcast(×tep,1,MPI_INT,0,comm); // Read in the restart file to CPU buffers double *cPhi = new double[Np]; double *cDist = new double[19*Np]; @@ -468,7 +468,7 @@ void ScaLBL_DFHModel::Initialize(){ ScaLBL_DeviceBarrier(); delete [] cPhi; delete [] cDist; - comm.barrier(); + MPI_Barrier(comm); } if (rank==0) printf ("Initializing phase field \n"); @@ -486,7 +486,7 @@ void ScaLBL_DFHModel::Run(){ //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -532,8 +532,7 @@ void ScaLBL_DFHModel::Run(){ } ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; @@ -569,9 +568,9 @@ void ScaLBL_DFHModel::Run(){ } ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ + MPI_Barrier(comm); PROFILE_STOP("Update"); // Run the analysis @@ -582,7 +581,7 @@ void ScaLBL_DFHModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep diff --git a/models/DFHModel.h b/models/DFHModel.h index 00e6e6b3..883ec6f8 100644 --- a/models/DFHModel.h +++ b/models/DFHModel.h @@ -12,13 +12,13 @@ Implementation of color lattice boltzmann model #include "common/Communication.h" #include "analysis/TwoPhase.h" #include "analysis/runAnalysis.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" class ScaLBL_DFHModel{ public: - ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM); + ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM); ~ScaLBL_DFHModel(); // functions in they should be run @@ -66,7 +66,7 @@ public: double *Pressure; private: - Utilities::MPI comm; + MPI_Comm comm; int dist_mem_size; int neighborSize; diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 23925930..c1db7c1c 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -5,7 +5,7 @@ #include "analysis/distance.h" #include "common/ReadMicroCT.h" -ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM): +ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) @@ -83,9 +83,9 @@ void ScaLBL_MRTModel::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object - comm.barrier(); + MPI_Barrier(comm); Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); rank = Dm->rank(); nprocx = Dm->nprocx(); @@ -171,7 +171,7 @@ void ScaLBL_MRTModel::Create(){ Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE //........................................................................... @@ -190,7 +190,7 @@ void ScaLBL_MRTModel::Create(){ if (rank==0) printf ("Setting up device map and neighbor list \n"); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - comm.barrier(); + MPI_Barrier(comm); } @@ -225,8 +225,7 @@ void ScaLBL_MRTModel::Run(){ //.......create and start timer............ double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); if (rank==0) printf("********************************************************\n"); @@ -240,21 +239,18 @@ void ScaLBL_MRTModel::Run(){ ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ if (timestep%1000==0){ ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); @@ -276,10 +272,10 @@ void ScaLBL_MRTModel::Run(){ } } } - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); vax /= count; vay /= count; @@ -309,10 +305,10 @@ void ScaLBL_MRTModel::Run(){ double As = Morphology.A(); double Hs = Morphology.H(); double Xs = Morphology.X(); - Vs = Dm->Comm.sumReduce( Vs); - As = Dm->Comm.sumReduce( As); - Hs = Dm->Comm.sumReduce( Hs); - Xs = Dm->Comm.sumReduce( Xs); + Vs=sumReduce( Dm->Comm, Vs); + As=sumReduce( Dm->Comm, As); + Hs=sumReduce( Dm->Comm, Hs); + Xs=sumReduce( Dm->Comm, Xs); double h = Dm->voxel_length; double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; if (rank==0) { @@ -346,8 +342,7 @@ void ScaLBL_MRTModel::VelocityField(){ /* Minkowski Morphology(Mask); int SIZE=Np*sizeof(double); ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier();. - comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); @@ -374,10 +369,10 @@ void ScaLBL_MRTModel::VelocityField(){ vaz_loc += VELOCITY[2*Np+n]; count_loc+=1.0; } - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); vax /= count; vay /= count; diff --git a/models/MRTModel.h b/models/MRTModel.h index 7e23cc44..aa4ee1f0 100644 --- a/models/MRTModel.h +++ b/models/MRTModel.h @@ -11,13 +11,13 @@ #include "common/ScaLBL.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "analysis/Minkowski.h" #include "ProfilerApp.h" class ScaLBL_MRTModel{ public: - ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM); + ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM); ~ScaLBL_MRTModel(); // functions in they should be run @@ -63,7 +63,7 @@ public: DoubleArray Velocity_y; DoubleArray Velocity_z; private: - Utilities::MPI comm; + MPI_Comm comm; // filenames char LocalRankString[8]; diff --git a/tests/BlobAnalyzeParallel.cpp b/tests/BlobAnalyzeParallel.cpp index 48e9e230..c9e3f8fc 100644 --- a/tests/BlobAnalyzeParallel.cpp +++ b/tests/BlobAnalyzeParallel.cpp @@ -100,10 +100,11 @@ inline void WriteBlobStates(TwoPhase TCAT, double D, double porosity){ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); Utilities::setAbortBehavior( true, true, true ); Utilities::setErrorHandlers(); PROFILE_ENABLE(0); @@ -136,7 +137,7 @@ int main(int argc, char **argv) domain >> Ly; domain >> Lz; } - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&nx,1,MPI_INT,0,comm); MPI_Bcast(&ny,1,MPI_INT,0,comm); @@ -149,7 +150,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // Check that the number of processors >= the number of ranks if ( rank==0 ) { @@ -208,7 +209,7 @@ int main(int argc, char **argv) // WriteLocalSolidID(LocalRankFilename, id, N); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); ReadBinaryFile(LocalRankFilename, Averages.SDs.get(), N); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //....................................................................... //copies of data needed to perform checkpointing from cpu @@ -220,7 +221,7 @@ int main(int argc, char **argv) if (rank==0) printf("Reading restart file! \n"); // Read in the restart file to CPU buffers ReadCheckpoint(LocalRestartFile, Den, DistEven, DistOdd, N); - comm.barrier(); + MPI_Barrier(comm); //......................................................................... // Populate the arrays needed to perform averaging if (rank==0) printf("Populate arrays \n"); @@ -328,14 +329,14 @@ int main(int argc, char **argv) // BlobContainer Blobs; DoubleArray RecvBuffer(dimx); // MPI_Allreduce(&Averages.BlobAverages.get(),&Blobs.get(),1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf("Number of components is %i \n",dimy); for (int b=0; b 0.0){ double Vn,pn,awn,ans,Jwn,Kwn,lwns,cwns,trawn,trJwn; @@ -481,7 +482,7 @@ int main(int argc, char **argv) fclose(BLOBS);*/ PROFILE_STOP("main"); PROFILE_SAVE("BlobIdentifyParallel",false); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/BlobIdentifyParallel.cpp b/tests/BlobIdentifyParallel.cpp index b8929a11..f93371cb 100644 --- a/tests/BlobIdentifyParallel.cpp +++ b/tests/BlobIdentifyParallel.cpp @@ -47,10 +47,11 @@ void readRankData( int proc, int nx, int ny, int nz, DoubleArray& Phase, DoubleA int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); #ifdef PROFILE PROFILE_ENABLE(0); PROFILE_DISABLE_TRACE(); @@ -128,7 +129,7 @@ int main(int argc, char **argv) PROFILE_STOP("main"); PROFILE_SAVE("BlobIdentifyParallel",false); #endif - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/ColorToBinary.cpp b/tests/ColorToBinary.cpp index fae156d1..7ac740bc 100644 --- a/tests/ColorToBinary.cpp +++ b/tests/ColorToBinary.cpp @@ -114,10 +114,11 @@ inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, int nx, int ny, int int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); printf("----------------------------------------------------------\n"); printf("Creating single Binary file from restart (8-bit integer)\n"); @@ -275,7 +276,7 @@ int main(int argc, char **argv) */ // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/ComponentLabel.cpp b/tests/ComponentLabel.cpp index 624ce8f4..07ef6555 100644 --- a/tests/ComponentLabel.cpp +++ b/tests/ComponentLabel.cpp @@ -119,10 +119,11 @@ inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, DoubleArray &Pressu int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); printf("----------------------------------------------------------\n"); printf("COMPUTING TCAT ANALYSIS FOR NON-WETTING PHASE FEATURES \n"); @@ -432,7 +433,7 @@ int main(int argc, char **argv) fclose(DISTANCE); */ // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 43434092..d1917619 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -9,7 +9,7 @@ //#include "common/pmmc.h" #include "common/Domain.h" #include "common/SpherePack.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" /* @@ -70,8 +70,8 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny } } // total Global is the number of nodes in the pore-space - totalGlobal = Dm.Comm.sumReduce( count ); - maxdistGlobal = Dm.Comm.sumReduce( maxdist ); + MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2); double porosity=totalGlobal/volume; if (rank==0) printf("Media Porosity: %f \n",porosity); @@ -148,6 +148,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny double Rcrit_old=0.0; double Rcrit_new=0.0; + double GlobalNumber = 1.f; int imin,jmin,kmin,imax,jmax,kmax; Rcrit_new = maxdistGlobal; @@ -214,41 +215,41 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, - recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, - recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, - recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, - recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, - recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, - recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, - recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, - recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, - recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, - recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, - recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, - recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, - recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, - recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, - recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, - recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, - recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, - recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm.getCommunicator(),MPI_STATUS_IGNORE); + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE); //...................................................................................... UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); @@ -270,7 +271,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny UnpackID(Dm.recvList_YZ, Dm.recvCount_YZ ,recvID_YZ, id); //...................................................................................... - //double GlobalNumber = Dm.Comm.sumReduce( LocalNumber ); + MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); count = 0.f; for (int k=1; k= the number of ranks if ( rank==0 ) { @@ -253,14 +254,14 @@ int main(int argc, char **argv) cz[0]=0.25*Lz; cx[1]=0.75*Lz; cx[2]=0.25*Lz; cx[3]=0.25*Lz; rad[0]=rad[1]=rad[2]=rad[3]=0.1*Lx; - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); //....................................................................... SignedDistance(Averages.Phase.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz, Dm->iproc(),Dm->jproc(),Dm->kproc(),Dm->nprocx(),Dm->nprocy(),Dm->nprocz()); @@ -316,7 +317,7 @@ int main(int argc, char **argv) delete [] rad; } // Limit scope so variables that contain communicators will free before MPI_Finialize - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/TestBlobIdentify.cpp b/tests/TestBlobIdentify.cpp index 7eb5c270..ccfc6afc 100644 --- a/tests/TestBlobIdentify.cpp +++ b/tests/TestBlobIdentify.cpp @@ -23,19 +23,21 @@ inline double rand2() // Test if all ranks agree on a value -bool allAgree( int x, const Utilities::MPI& comm ) { +bool allAgree( int x, MPI_Comm comm ) { int x2 = x; - comm.bcast(&x2,1,0); + MPI_Bcast(&x2,1,MPI_INT,0,comm); int diff = x==x2 ? 0:1; - int diff2 = comm.sumReduce( diff ); + int diff2 = 0; + MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm); return diff2==0; } template -bool allAgree( const std::vector& x, const Utilities::MPI& comm ) { +bool allAgree( const std::vector& x, MPI_Comm comm ) { std::vector x2 = x; - comm.bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),0); + MPI_Bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),MPI_INT,0,comm); int diff = x==x2 ? 0:1; - int diff2 = comm.sumReduce( diff ); + int diff2 = 0; + MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm); return diff2==0; } @@ -72,9 +74,9 @@ struct bubble_struct { // Create a random set of bubles -std::vector create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, const Utilities::MPI& comm ) +std::vector create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, MPI_Comm comm ) { - int rank = comm.getRank(); + int rank = comm_rank(comm); std::vector bubbles(N_bubbles); if ( rank == 0 ) { double R0 = 0.2*Lx*Ly*Lz/pow((double)N_bubbles,0.333); @@ -89,7 +91,7 @@ std::vector create_bubbles( int N_bubbles, double Lx, double Ly, } } size_t N_bytes = N_bubbles*sizeof(bubble_struct); - comm.bcast((char*)&bubbles[0],N_bytes,0); + MPI_Bcast((char*)&bubbles[0],N_bytes,MPI_CHAR,0,comm); return bubbles; } @@ -122,7 +124,7 @@ void fillBubbleData( const std::vector& bubbles, DoubleArray& Pha // Shift all of the data by the given number of cells -void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, const Utilities::MPI& comm ) +void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, MPI_Comm comm ) { int nx = data.size(0)-2; int ny = data.size(1)-2; @@ -152,10 +154,11 @@ void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); PROFILE_ENABLE(1); PROFILE_DISABLE_TRACE(); PROFILE_SYNCHRONIZE(); @@ -294,7 +297,7 @@ int main(int argc, char **argv) velocity[i].z = bubbles[i].radius*(2*rand2()-1); } } - comm.bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),0); + MPI_Bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),MPI_CHAR,0,comm); fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info ); fillData.fill(Phase); fillData.fill(SignDist); @@ -388,8 +391,8 @@ int main(int argc, char **argv) printf("\n"); } } - comm.bcast(&N1,1,0); - comm.bcast(&N2,1,0); + MPI_Bcast(&N1,1,MPI_INT,0,comm); + MPI_Bcast(&N2,1,MPI_INT,0,comm); if ( N1!=nblobs || N2!=nblobs2 ) { if ( rank==0 ) printf("Error, blob ids do not map in moving bubble test (%i,%i,%i,%i)\n", @@ -409,7 +412,7 @@ int main(int argc, char **argv) // Finished PROFILE_STOP("main"); PROFILE_SAVE("TestBlobIdentify",false); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return N_errors; } diff --git a/tests/TestBlobIdentifyCorners.cpp b/tests/TestBlobIdentifyCorners.cpp index 904e52e0..4795f610 100644 --- a/tests/TestBlobIdentifyCorners.cpp +++ b/tests/TestBlobIdentifyCorners.cpp @@ -18,9 +18,10 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + MPI_Comm_size(MPI_COMM_WORLD,&nprocs); /*if ( nprocs != 8 ) { printf("This tests requires 8 processors\n"); return -1; diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index e7e0ced8..c03e5dea 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -7,7 +7,7 @@ #include "analysis/pmmc.h" #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -32,15 +32,14 @@ int main(int argc, char **argv) // Initialize MPI int provided_thread_support = -1; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_support(domain_db,comm); - comm.barrier(); + MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; int N = Nx*Ny*Nz; @@ -249,7 +250,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE @@ -386,7 +387,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -436,7 +437,7 @@ int main(int argc, char **argv) } ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; @@ -472,9 +473,9 @@ int main(int argc, char **argv) } ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("Update"); // Run the analysis @@ -486,7 +487,7 @@ int main(int argc, char **argv) PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -546,8 +547,9 @@ int main(int argc, char **argv) PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); MPI_Finalize(); return check; } diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 1f42a71e..0e6ea25a 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/ColorModel.h" using namespace std; @@ -64,11 +64,15 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { if (rank == 0){ @@ -93,7 +97,7 @@ int main(int argc, char **argv) ColorModel.WriteDebug(); } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestColorGrad.cpp b/tests/TestColorGrad.cpp index df1c1daf..5cd6d924 100644 --- a/tests/TestColorGrad.cpp +++ b/tests/TestColorGrad.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -15,11 +15,15 @@ using namespace std; //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check; { // parallel domain size (# of sub-domains) @@ -112,7 +116,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -125,7 +129,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -142,7 +146,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; int BoundaryCondition=0; @@ -171,7 +175,7 @@ int main(int argc, char **argv) } } Dm.CommInit(); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -188,7 +192,7 @@ int main(int argc, char **argv) neighborList= new int[18*Np]; ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); @@ -256,7 +260,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestColorGradDFH.cpp b/tests/TestColorGradDFH.cpp index b04aebce..d6376d82 100644 --- a/tests/TestColorGradDFH.cpp +++ b/tests/TestColorGradDFH.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -25,11 +25,15 @@ std::shared_ptr loadInputs( int nprocs ) //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { // parallel domain size (# of sub-domains) @@ -78,7 +82,7 @@ int main(int argc, char **argv) } } Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -101,7 +105,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int neighborSize=18*Np*sizeof(int); @@ -207,7 +211,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index 78508f9b..c05c245e 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -15,11 +15,15 @@ using namespace std; //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { // parallel domain size (# of sub-domains) @@ -38,7 +42,7 @@ int main(int argc, char **argv) // Initialize compute device // int device=ScaLBL_SetDevice(rank); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); Utilities::setErrorHandlers(); // Variables that specify the computational domain @@ -73,7 +77,7 @@ int main(int argc, char **argv) // Get the rank info const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - comm.barrier(); + MPI_Barrier(comm); if (nprocs != nprocx*nprocy*nprocz){ printf("nprocx = %i \n",nprocx); @@ -117,7 +121,7 @@ int main(int argc, char **argv) std::shared_ptr Dm(new Domain(domain_db,comm)); for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; int N = Nx*Ny*Nz; @@ -149,7 +153,7 @@ int main(int argc, char **argv) } } Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -166,7 +170,7 @@ int main(int argc, char **argv) Npad=Np+32; neighborList= new int[18*Npad]; Np=ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); @@ -268,7 +272,7 @@ int main(int argc, char **argv) ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -328,7 +332,7 @@ int main(int argc, char **argv) ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ printf("Check after even time \n"); @@ -411,7 +415,7 @@ int main(int argc, char **argv) ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -472,7 +476,7 @@ int main(int argc, char **argv) ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ printf("Check after even time \n"); @@ -519,7 +523,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** return check; diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index cf8a9566..9807f0e8 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/ColorModel.h" std::shared_ptr loadInputs( int nprocs ) @@ -84,11 +84,15 @@ void InitializeSquareTube(ScaLBL_ColorModel &ColorModel){ //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { if (rank == 0){ @@ -109,7 +113,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index d2799355..e1fa821f 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -6,7 +6,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -164,10 +164,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check; { @@ -262,14 +263,14 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); double iVol_global=1.f/double((Nx-2)*(Ny-2)*(Nz-2)*nprocx*nprocy*nprocz); porosity = 1.0-sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); //....................................................................... //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //........................................................................... @@ -284,7 +285,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); Map.fill(-2); Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); int neighborSize=18*Np*sizeof(int); //......................device distributions................................. dist_mem_size = Np*sizeof(double); @@ -354,7 +355,7 @@ int main(int argc, char **argv) GlobalFlipScaLBL_D3Q19_Init(fq_host, Map, Np, Nx-2, Ny-2, Nz-2, iproc,jproc,kproc,nprocx,nprocy,nprocz); ScaLBL_CopyToDevice(fq, fq_host, 19*dist_mem_size); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); //************************************************************************* // First timestep ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL @@ -377,7 +378,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -397,7 +398,7 @@ int main(int argc, char **argv) //********************************************* ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); // Iteration completed! timestep++; //................................................................... @@ -426,7 +427,7 @@ int main(int argc, char **argv) if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9); } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestDatabase.cpp b/tests/TestDatabase.cpp index ced704e2..00bf87e2 100644 --- a/tests/TestDatabase.cpp +++ b/tests/TestDatabase.cpp @@ -9,7 +9,7 @@ #include "common/UnitTest.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Database.h" #include "ProfilerApp.h" @@ -17,8 +17,11 @@ // Main int main(int argc, char **argv) { + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); Utilities::setAbortBehavior(true,2); Utilities::setErrorHandlers(); UnitTest ut; @@ -66,7 +69,7 @@ int main(int argc, char **argv) // Finished PROFILE_SAVE("TestDatabase",true); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return err; } diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 3e999715..020bbd89 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -1,5 +1,5 @@ #include -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include "common/ScaLBL.h" @@ -18,9 +18,9 @@ std::shared_ptr loadInputs( int nprocs ) int main (int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + int rank = MPI_WORLD_RANK(); + int nprocs = MPI_WORLD_SIZE(); // set the error code // Note: the error code should be consistent across all processors @@ -89,7 +89,7 @@ int main (int argc, char **argv) neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); @@ -149,7 +149,7 @@ int main (int argc, char **argv) double *VEL; VEL= new double [3*Np]; int SIZE=3*Np*sizeof(double); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_CopyToHost(&VEL[0],&dvc_vel[0],SIZE); double Q = 0.f; @@ -192,7 +192,7 @@ int main (int argc, char **argv) din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL @@ -201,7 +201,7 @@ int main (int argc, char **argv) din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -265,7 +265,7 @@ int main (int argc, char **argv) } // Finished - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return error; } diff --git a/tests/TestForceD3Q19.cpp b/tests/TestForceD3Q19.cpp index f8569624..b8f88aae 100644 --- a/tests/TestForceD3Q19.cpp +++ b/tests/TestForceD3Q19.cpp @@ -1,5 +1,5 @@ #include -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include @@ -443,9 +443,8 @@ inline void MRT_Transform(double *dist, int Np, double Fx, double Fy, double Fz) int main (int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + int rank = MPI_WORLD_RANK(); + int nprocs = MPI_WORLD_SIZE(); for (int i=0; i #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -46,11 +46,15 @@ std::shared_ptr loadInputs( int nprocs ) //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { // parallel domain size (# of sub-domains) @@ -94,7 +98,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -102,7 +106,7 @@ int main(int argc, char **argv) if (rank == 0) { printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); } - comm.barrier(); + MPI_Barrier(comm); if (rank == 1){ printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); printf("\n\n"); @@ -139,7 +143,7 @@ int main(int argc, char **argv) } } Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; int Np=0; // number of local pore nodes @@ -184,7 +188,7 @@ int main(int argc, char **argv) if (rank == 0) PrintNeighborList(neighborList,Np, rank); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); @@ -209,13 +213,13 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); /************ MAIN ITERATION LOOP (timing communications)***************************************/ //ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]); //ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); if (rank==0) printf("Beginning AA timesteps...\n"); if (rank==0) printf("********************************************************\n"); @@ -227,14 +231,14 @@ int main(int argc, char **argv) ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -327,7 +331,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index d2c901df..40d53b47 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -2,7 +2,7 @@ #include #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -18,9 +18,13 @@ int main (int argc, char *argv[]) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + + int i,j,k; // Load inputs string FILENAME = argv[1]; @@ -36,7 +40,7 @@ int main (int argc, char *argv[]) Nx+=2; Ny+=2; Nz+=2; - for (int i=0; iid[i] = 1; + for (i=0; iid[i] = 1; Dm->CommInit(); @@ -47,9 +51,9 @@ int main (int argc, char *argv[]) double dist1,dist2; Cx = Cy = Cz = N*0.5; - for (int k=0; k #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -488,11 +488,15 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check; { // parallel domain size (# of sub-domains) @@ -578,7 +582,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -591,7 +595,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** // ************************************************************** @@ -609,7 +613,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); kproc = rank/(nprocx*nprocy); jproc = (rank-nprocx*nprocy*kproc)/nprocx; iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -617,7 +621,7 @@ int main(int argc, char **argv) if (rank == 0) { printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); } - comm.barrier(); + MPI_Barrier(comm); if (rank == 1){ printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); printf("\n\n"); @@ -646,7 +650,7 @@ int main(int argc, char **argv) fread(Dm.id,1,N,IDFILE); fclose(IDFILE); - comm.barrier(); + MPI_Barrier(comm); Dm.CommInit(); //....................................................................... @@ -667,12 +671,12 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -702,7 +706,7 @@ int main(int argc, char **argv) neighborList= new int[18*Np]; ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); @@ -730,7 +734,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); while (timestep < timesteps) { @@ -739,14 +743,14 @@ int main(int argc, char **argv) ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.next, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.next, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -779,7 +783,7 @@ int main(int argc, char **argv) VEL= new double [3*Np]; int SIZE=3*Np*sizeof(double); ScaLBL_D3Q19_Momentum(dist,Velocity, Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_CopyToHost(&VEL[0],&Velocity[0],SIZE); sum_local=0.f; @@ -801,7 +805,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index f3010081..a47c0d9e 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" using namespace std; @@ -26,9 +26,15 @@ std::shared_ptr loadInputs( int nprocs ) //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { @@ -39,7 +45,6 @@ int main(int argc, char **argv) {1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}}; - int rank = comm.getRank(); if (rank == 0){ printf("********************************************************\n"); printf("Running unit test: TestMap \n"); @@ -47,7 +52,7 @@ int main(int argc, char **argv) } // Load inputs - auto db = loadInputs( comm.getSize() ); + auto db = loadInputs( nprocs ); int Nx = db->getVector( "n" )[0]; int Ny = db->getVector( "n" )[1]; int Nz = db->getVector( "n" )[2]; @@ -89,7 +94,7 @@ int main(int argc, char **argv) neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); // Check the neighborlist printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior()); @@ -192,7 +197,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index 68183cd2..bbfe8cae 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -8,7 +8,7 @@ #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/ColorModel.h" inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){ @@ -67,10 +67,11 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) if (rank == 0){ @@ -265,7 +266,7 @@ int main(int argc, char **argv) } } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/TestMicroCTReader.cpp b/tests/TestMicroCTReader.cpp index 9a54610c..4a4c6aac 100644 --- a/tests/TestMicroCTReader.cpp +++ b/tests/TestMicroCTReader.cpp @@ -1,6 +1,6 @@ // Test reading high-resolution files from the microct database -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/UnitTest.h" #include "common/Database.h" #include "common/Domain.h" @@ -13,14 +13,12 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) { - Utilities::MPI comm( MPI_COMM_WORLD ); - // Get the domain info auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); // Test reading microCT files - auto data = readMicroCT( *domain_db, comm ); + auto data = readMicroCT( *domain_db, MPI_COMM_WORLD ); // Check if we loaded the data correctly if ( data.size() == domain_db->getVector( "n" ) ) @@ -32,7 +30,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) auto n = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); int N[3] = { n[0]*nproc[0], n[1]*nproc[1], n[2]*nproc[2] }; - int rank = comm.getRank(); + int rank = comm_rank(MPI_COMM_WORLD); RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] ); std::vector meshData( 1 ); auto Var = std::make_shared(); @@ -43,7 +41,7 @@ void testReadMicroCT( const std::string& filename, UnitTest& ut ) meshData[0].meshName = "grid"; meshData[0].mesh = std::make_shared(rankInfo,n[0],n[1],n[2],N[0],N[1],N[2]); meshData[0].vars.push_back(Var); - IO::writeData( 0, meshData, comm ); + IO::writeData( 0, meshData, MPI_COMM_WORLD ); } diff --git a/tests/TestMomentsD3Q19.cpp b/tests/TestMomentsD3Q19.cpp index 6bd3e8ff..b26d7bed 100644 --- a/tests/TestMomentsD3Q19.cpp +++ b/tests/TestMomentsD3Q19.cpp @@ -1,5 +1,5 @@ #include -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" #include @@ -463,14 +463,13 @@ inline void MRT_Transform(double *dist, int Np) { int main (int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + int rank = MPI_WORLD_RANK(); + int nprocs = MPI_WORLD_SIZE(); for (int i=0; i tmp = netcdf::getVar( fid, "tmp" ); @@ -96,8 +95,7 @@ int main(int argc, char **argv) { // Initialize MPI MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - const int rank = comm.getRank(); + int rank = comm_rank(MPI_COMM_WORLD); UnitTest ut; PROFILE_START("Main"); diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 744d292d..e69507e1 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -7,7 +7,7 @@ #include #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/MRTModel.h" void ParallelPlates(ScaLBL_MRTModel &MRT){ @@ -47,11 +47,15 @@ void ParallelPlates(ScaLBL_MRTModel &MRT){ //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { if (rank == 0){ @@ -73,7 +77,7 @@ int main(int argc, char **argv) int SIZE=MRT.Np*sizeof(double); ScaLBL_D3Q19_Momentum(MRT.fq,MRT.Velocity, MRT.Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); ScaLBL_CopyToHost(&Vz[0],&MRT.Velocity[0],3*SIZE); if (rank == 0) printf("Force: %f,%f,%f \n",MRT.Fx,MRT.Fy,MRT.Fz); @@ -87,7 +91,7 @@ int main(int argc, char **argv) j=Ny/2; k=Nz/2; if (rank == 0) printf("Channel width=%f \n",W); if (rank == 0) printf("ID flag vz analytical\n"); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) { for (i=0;i #include #include "common/ScaLBL.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int check=0; { if (rank == 0){ @@ -45,7 +50,7 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - comm.barrier(); + MPI_Barrier(comm); int kproc = rank/(nprocx*nprocy); int jproc = (rank-nprocx*nprocy*kproc)/nprocx; int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; @@ -53,7 +58,7 @@ int main(int argc, char **argv) if (rank == 0) { printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); } - comm.barrier(); + MPI_Barrier(comm); if (rank == 1){ printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); printf("\n\n"); @@ -97,11 +102,11 @@ int main(int argc, char **argv) } } } - sum = comm.sumReduce( sum_local ); + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum*iVol_global; if (rank==0) printf("Media porosity = %f \n",porosity); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -128,7 +133,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - comm.barrier(); + MPI_Barrier(comm); //......................device distributions................................. if (rank==0) printf ("Allocating distributions \n"); @@ -189,7 +194,7 @@ int main(int argc, char **argv) } } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** return check; diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index b5e23ec8..ece3222d 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -39,10 +39,11 @@ std::shared_ptr loadInputs( int nprocs ) int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { @@ -97,7 +98,7 @@ int main(int argc, char **argv) } } - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf("Initialized! Converting to Signed Distance function \n"); double t1 = MPI_Wtime(); @@ -115,7 +116,7 @@ int main(int argc, char **argv) } } } - err = Dm.Comm.sumReduce( err ); + err = sumReduce( Dm.Comm, err ); err = sqrt( err / (nx*ny*nz*nprocs) ); if (rank==0) printf("Mean error %0.4f \n", err); @@ -141,7 +142,7 @@ int main(int argc, char **argv) IO::writeData( "testSegDist", data, MPI_COMM_WORLD ); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; diff --git a/tests/TestSubphase.cpp b/tests/TestSubphase.cpp index 9738812f..fd6383be 100644 --- a/tests/TestSubphase.cpp +++ b/tests/TestSubphase.cpp @@ -26,10 +26,11 @@ std::shared_ptr loadInputs( int nprocs ) int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // Limit scope so variables that contain communicators will free before MPI_Finialize if ( rank==0 ) { @@ -136,7 +137,7 @@ int main(int argc, char **argv) // Averages->Reduce(); } // Limit scope so variables that contain communicators will free before MPI_Finialize - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/TestTopo3D.cpp b/tests/TestTopo3D.cpp index 948bb1d6..8d00ef5a 100644 --- a/tests/TestTopo3D.cpp +++ b/tests/TestTopo3D.cpp @@ -26,10 +26,11 @@ std::shared_ptr loadInputs( int nprocs ) int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // Limit scope so variables that contain communicators will free before MPI_Finialize if ( rank==0 ) { @@ -225,7 +226,7 @@ int main(int argc, char **argv) IO::writeData( timestep, visData, comm ); } // Limit scope so variables that contain communicators will free before MPI_Finialize - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index 5125ce92..2d486774 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -26,10 +26,11 @@ std::shared_ptr loadInputs( int nprocs ) int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // Limit scope so variables that contain communicators will free before MPI_Finialize if ( rank==0 ) { @@ -164,7 +165,7 @@ int main(int argc, char **argv) // Averages->Reduce(); } // Limit scope so variables that contain communicators will free before MPI_Finialize - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/TestTorusEvolve.cpp b/tests/TestTorusEvolve.cpp index 32cf7fd8..1a65d268 100644 --- a/tests/TestTorusEvolve.cpp +++ b/tests/TestTorusEvolve.cpp @@ -26,10 +26,11 @@ std::shared_ptr loadInputs( int nprocs ) int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // Limit scope so variables that contain communicators will free before MPI_Finialize if ( rank==0 ) { @@ -156,7 +157,7 @@ int main(int argc, char **argv) } } // Limit scope so variables that contain communicators will free before MPI_Finialize - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/TestTwoPhase.cpp b/tests/TestTwoPhase.cpp index fa54d98d..a979314a 100644 --- a/tests/TestTwoPhase.cpp +++ b/tests/TestTwoPhase.cpp @@ -8,7 +8,7 @@ #include #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -17,10 +17,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // Limit scope so Domain can free it's communicator printf("Running two-phase averaging test on %i processors \n",nprocs); @@ -109,7 +110,7 @@ int main(int argc, char **argv) fclose(PHASE); } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); } // Limit scope so Domain will free it's communicator MPI_Finalize(); return 0; diff --git a/tests/TestWriter.cpp b/tests/TestWriter.cpp index 37858202..8936aaff 100644 --- a/tests/TestWriter.cpp +++ b/tests/TestWriter.cpp @@ -8,7 +8,7 @@ #include "common/UnitTest.h" #include "common/Utilities.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" @@ -34,9 +34,11 @@ inline double distance( const Point& p ) // Test writing and reading the given format void testWriter( const std::string& format, std::vector& meshData, UnitTest& ut ) { - Utilities::MPI comm( MPI_COMM_WORLD ); - int nprocs = comm.getSize(); - comm.barrier(); + int rank, nprocs; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + MPI_Barrier(comm); // Get the format std::string format2 = format; @@ -61,7 +63,7 @@ void testWriter( const std::string& format, std::vector& mes IO::initialize( "test_"+format, format2, false ); IO::writeData( 0, meshData, comm ); IO::writeData( 3, meshData, comm ); - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP(format+"-write"); // Get the summary name for reading @@ -226,10 +228,11 @@ void testWriter( const std::string& format, std::vector& mes // Main int main(int argc, char **argv) { + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); Utilities::setAbortBehavior(true,2); Utilities::setErrorHandlers(); UnitTest ut; @@ -386,7 +389,7 @@ int main(int argc, char **argv) ut.report(); PROFILE_SAVE("TestWriter",true); int N_errors = ut.NumFailGlobal(); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return N_errors; } diff --git a/tests/convertIO.cpp b/tests/convertIO.cpp index 27605237..0937729f 100644 --- a/tests/convertIO.cpp +++ b/tests/convertIO.cpp @@ -5,7 +5,7 @@ #include #include -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" #include "common/Utilities.h" #include "IO/Mesh.h" @@ -17,10 +17,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); Utilities::setErrorHandlers(); PROFILE_ENABLE(2); PROFILE_ENABLE_TRACE(); @@ -69,20 +70,20 @@ int main(int argc, char **argv) i++; } - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("Read"); // Save the mesh data to a new file PROFILE_START("Write"); IO::writeData( timestep, meshData, MPI_COMM_WORLD ); - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("Write"); } } // Limit scope PROFILE_STOP("Main"); PROFILE_SAVE("convertData",true); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/hello_world.cpp b/tests/hello_world.cpp index 810d3a9c..d236bf0e 100644 --- a/tests/hello_world.cpp +++ b/tests/hello_world.cpp @@ -1,19 +1,18 @@ #include -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Utilities.h" int main (int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + int rank = MPI_WORLD_RANK(); + int nprocs = MPI_WORLD_SIZE(); for (int i=0; i loadInputs( ) @@ -24,11 +24,15 @@ std::shared_ptr loadInputs( ) //*************************************************************************************** int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //***************************************** // MPI ranks for all 18 neighbors @@ -92,7 +96,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nz += 2; Nx = Ny = Nz; // Cubic domain @@ -181,7 +185,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index 97df6812..1c619c5a 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -9,7 +9,7 @@ #include "common/Communication.h" #include "analysis/TwoPhase.h" #include "analysis/runAnalysis.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" @@ -30,9 +30,10 @@ int main(int argc, char **argv) // Initialize MPI int provided_thread_support = -1; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); { // Limit scope so variables that contain communicators will free before MPI_Finialize // parallel domain size (# of sub-domains) @@ -51,7 +52,7 @@ int main(int argc, char **argv) // int device=ScaLBL_SetDevice(rank); //printf("Using GPU ID %i for rank %i \n",device,rank); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); @@ -170,7 +171,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm); MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm); @@ -206,7 +207,7 @@ int main(int argc, char **argv) // Get the rank info const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - comm.barrier(); + MPI_Barrier(comm); if (nprocs != nprocx*nprocy*nprocz){ printf("nprocx = %i \n",nprocx); @@ -261,7 +262,7 @@ int main(int argc, char **argv) // Mask that excludes the solid phase Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - comm.barrier(); + MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; int N = Nx*Ny*Nz; @@ -296,7 +297,7 @@ int main(int argc, char **argv) sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); @@ -340,7 +341,7 @@ int main(int argc, char **argv) delete [] cDen; delete [] cfq; */ - comm.barrier(); + MPI_Barrier(comm); } fflush(stdout); @@ -415,7 +416,7 @@ int main(int argc, char **argv) neighborList= new int[18*Npad]; Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask.id,Np); if (rank==0) printf ("Set up memory efficient layout Npad=%i, Np=%i \n",Npad,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE //........................................................................... @@ -536,7 +537,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -588,7 +589,7 @@ int main(int argc, char **argv) } ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; @@ -621,10 +622,10 @@ int main(int argc, char **argv) } ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); comm.barrier(); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("Update"); // Run the analysis @@ -636,7 +637,7 @@ int main(int argc, char **argv) PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -656,8 +657,9 @@ int main(int argc, char **argv) PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); MPI_Finalize(); } diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index cef13189..1f63c653 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -28,9 +28,10 @@ int main(int argc, char **argv) { // Limit scope so variables that contain communicators will free before MPI_Finialize - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); if (rank == 0){ printf("********************************************************\n"); @@ -40,7 +41,7 @@ int main(int argc, char **argv) // Initialize compute device ScaLBL_SetDevice(rank); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); @@ -50,7 +51,7 @@ int main(int argc, char **argv) Utilities::setErrorHandlers(); auto filename = argv[1]; - ScaLBL_ColorModel ColorModel(rank,nprocs,comm.dup()); + ScaLBL_ColorModel ColorModel(rank,nprocs,comm); ColorModel.ReadParams(filename); ColorModel.SetDomain(); ColorModel.ReadInput(); @@ -63,7 +64,8 @@ int main(int argc, char **argv) PROFILE_SAVE("lbpm_color_simulator",1); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); + MPI_Comm_free(&comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index 0d5902df..1e8dc0f9 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -26,9 +26,10 @@ int main(int argc, char **argv) // Initialize MPI int provided_thread_support = -1; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); if ( rank==0 && provided_thread_support 1) depth=atoi(argv[1]); @@ -218,7 +222,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nx += 2; Ny += 2; @@ -273,13 +277,13 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Reading the disc packing \n"); if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0){ cout << "Domain set." << endl; printf("************ \n"); @@ -384,7 +388,7 @@ int main(int argc, char **argv) //...................................................................... // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_inkbottle_pp.cpp b/tests/lbpm_inkbottle_pp.cpp index 669ab8c0..3c39219d 100644 --- a/tests/lbpm_inkbottle_pp.cpp +++ b/tests/lbpm_inkbottle_pp.cpp @@ -9,15 +9,19 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; @@ -79,7 +83,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -92,7 +96,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** if (nprocs != nprocx*nprocy*nprocz){ @@ -119,7 +123,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nz += 2; Nx = Ny = Nz; // Cubic domain @@ -217,7 +221,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_juanes_bench_disc_pp.cpp b/tests/lbpm_juanes_bench_disc_pp.cpp index 47d8cb84..6f04cffa 100644 --- a/tests/lbpm_juanes_bench_disc_pp.cpp +++ b/tests/lbpm_juanes_bench_disc_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" // This includes mpi.h +#include "common/MPI_Helpers.h" // This includes mpi.h #include "common/SpherePack.h" /* @@ -130,11 +130,15 @@ inline void SignedDistanceDiscPack(double *Distance, int ndiscs, double *List_cx int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -190,7 +194,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. // Computational domain MPI_Bcast(&Nx,1,MPI_INT,0,comm); @@ -204,7 +208,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** double Rin,Rout; @@ -236,7 +240,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nx += 2; Ny += 2; Nz += 2; int N = Nx*Ny*Nz; @@ -290,13 +294,13 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Reading the disc packing \n"); if (rank == 0) ReadDiscPacking(ndiscs,cx,cy,rad); - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes MPI_Bcast(cx,ndiscs,MPI_DOUBLE,0,comm); MPI_Bcast(cy,ndiscs,MPI_DOUBLE,0,comm); MPI_Bcast(rad,ndiscs,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); /* if (rank == 0){ cout << "Domain set." << endl; printf("************ \n"); @@ -308,7 +312,7 @@ int main(int argc, char **argv) } */ - comm.barrier(); + MPI_Barrier(comm); if (nprocz > 1 && rank==0) printf("Disc packs are 2D -- are you sure you want nprocz > 1? \n"); if (rank ==0) printf("Compute the signed distance part I \n"); //....................................................................... @@ -486,7 +490,7 @@ int main(int argc, char **argv) //...................................................................... // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_minkowski_scalar.cpp b/tests/lbpm_minkowski_scalar.cpp index 721207a1..3e3ede6d 100644 --- a/tests/lbpm_minkowski_scalar.cpp +++ b/tests/lbpm_minkowski_scalar.cpp @@ -14,7 +14,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -28,11 +28,13 @@ int main(int argc, char **argv) { + // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { Utilities::setErrorHandlers(); PROFILE_START("Main"); @@ -85,7 +87,7 @@ int main(int argc, char **argv) fclose(SEGDAT); printf("Read segmented data from %s \n",Filename.c_str()); } - comm.barrier(); + MPI_Barrier(comm); // Get the rank info int N = (nx+2)*(ny+2)*(nz+2); @@ -150,7 +152,7 @@ int main(int argc, char **argv) } else{ printf("Sending data to process %i \n", rnk); - comm.send(tmp,N,rnk,15); + MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm); } } } @@ -159,12 +161,13 @@ int main(int argc, char **argv) else{ // Recieve the subdomain from rank = 0 printf("Ready to recieve data %i at process %i \n", N,rank); - comm.recv(Dm->id,N,0,15); + MPI_Recv(Dm->id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE); } - comm.barrier(); + MPI_Barrier(comm); // Compute the Minkowski functionals - auto Averages = std::make_shared(Dm); + MPI_Barrier(comm); + std::shared_ptr Averages(new Minkowski(Dm)); // Calculate the distance // Initialize the domain and communication @@ -209,7 +212,7 @@ int main(int argc, char **argv) } PROFILE_STOP("Main"); PROFILE_SAVE("Minkowski",true); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/lbpm_morph_pp.cpp b/tests/lbpm_morph_pp.cpp index 939fdc32..8fe8b228 100644 --- a/tests/lbpm_morph_pp.cpp +++ b/tests/lbpm_morph_pp.cpp @@ -23,9 +23,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //....................................................................... // Reading the domain information file @@ -125,13 +127,13 @@ int main(int argc, char **argv) if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); CalcDist(SignDist,id_solid,*Dm); - comm.barrier(); + MPI_Barrier(comm); // Extract only the connected part of NWP BlobIDstruct new_index; double vF=0.0; double vS=0.0; ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); int count_connected=0; int count_porespace=0; @@ -153,9 +155,9 @@ int main(int argc, char **argv) } } } - count_connected = Dm->Comm.sumReduce( count_connected ); - count_porespace = Dm->Comm.sumReduce( count_porespace ); - count_water = Dm->Comm.sumReduce( count_water ); + count_connected=sumReduce( Dm->Comm, count_connected); + count_porespace=sumReduce( Dm->Comm, count_porespace); + count_water=sumReduce( Dm->Comm, count_water); for (int k=0; kComm.sumReduce( count_water ); + count_water=sumReduce( Dm->Comm, count_water); SW = double(count_water) / count_porespace; if(rank==0) printf("Final saturation: %f \n", SW); @@ -234,13 +236,13 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); auto filename2 = READFILE + ".morph.raw"; if (rank==0) printf("Writing file to: %s \n", filename2.c_str()); Mask->AggregateLabels(filename2); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); } diff --git a/tests/lbpm_morphdrain_pp.cpp b/tests/lbpm_morphdrain_pp.cpp index d3c5a428..8d73b1e4 100644 --- a/tests/lbpm_morphdrain_pp.cpp +++ b/tests/lbpm_morphdrain_pp.cpp @@ -23,9 +23,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //....................................................................... // Reading the domain information file @@ -119,7 +121,7 @@ int main(int argc, char **argv) if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); CalcDist(SignDist,id_solid,*Dm); - comm.barrier(); + MPI_Barrier(comm); // Run the morphological opening MorphDrain(SignDist, id, Dm, SW); @@ -194,13 +196,13 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); auto filename2 = READFILE + ".morphdrain.raw"; if (rank==0) printf("Writing file to: %s \n", filename2.data() ); Mask->AggregateLabels( filename2 ); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); } diff --git a/tests/lbpm_morphopen_pp.cpp b/tests/lbpm_morphopen_pp.cpp index a6209240..f8819348 100644 --- a/tests/lbpm_morphopen_pp.cpp +++ b/tests/lbpm_morphopen_pp.cpp @@ -23,9 +23,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //....................................................................... // Reading the domain information file @@ -121,7 +123,7 @@ int main(int argc, char **argv) if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); CalcDist(SignDist,id_solid,*Dm); - comm.barrier(); + MPI_Barrier(comm); // Run the morphological opening MorphOpen(SignDist, id, Dm, SW, ErodeLabel, OpenLabel); @@ -196,13 +198,13 @@ int main(int argc, char **argv) } } } - comm.barrier(); + MPI_Barrier(comm); auto filename2 = READFILE + ".morphopen.raw"; if (rank==0) printf("Writing file to: %s \n", filename2.data()); Mask->AggregateLabels(filename2); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); } diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index 096dc790..40672375 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -9,7 +9,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" //#define WRITE_SURFACES @@ -77,11 +77,15 @@ int main(int argc, char **argv) } else { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; @@ -156,7 +160,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); //................................................. MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); //MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); @@ -181,7 +185,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); RESTART_INTERVAL=interval; // ************************************************************** @@ -218,7 +222,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nx += 2; Ny += 2; Nz += 2; @@ -258,7 +262,7 @@ int main(int argc, char **argv) // WriteLocalSolidID(LocalRankFilename, id, N); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); ReadBinaryFile(LocalRankFilename, Averages.SDs.data(), N); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; //....................................................................... @@ -432,7 +436,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... @@ -481,7 +485,7 @@ int main(int argc, char **argv) } //................................................................................... ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); // Timestep completed! @@ -553,7 +557,7 @@ int main(int argc, char **argv) //************************************************************************/ fclose(NONDARCY); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -571,7 +575,7 @@ int main(int argc, char **argv) NULL_USE(RESTART_INTERVAL); } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_nonnewtonian_simulator.cpp b/tests/lbpm_nonnewtonian_simulator.cpp index ff8792e7..5c33841f 100644 --- a/tests/lbpm_nonnewtonian_simulator.cpp +++ b/tests/lbpm_nonnewtonian_simulator.cpp @@ -9,7 +9,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "common/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" @@ -99,12 +99,21 @@ inline void ZeroHalo(double *Data, int Nx, int Ny, int Nz) int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + //MPI_Init(&argc,&argv); + + /* + * Definitely seems to be an issue - let's hope James gets back to me... + */ int provided_thread_support = -1; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); if ( rank==0 && provided_thread_supportSDs.data(), N); - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; /* 3 */ //....................................................................... @@ -589,14 +598,14 @@ int main(int argc, char **argv) delete [] cDen; delete [] cDistEven; delete [] cDistOdd; - comm.barrier(); + MPI_Barrier(comm); } /* 14 */ // //...................................................................... // ScaLBL_D3Q7_Init(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); // ScaLBL_D3Q7_Init(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); // ScaLBL_DeviceBarrier(); -// comm.barrier(); /* 15 */ +// MPI_Barrier(comm); /* 15 */ //....................................................................... // Once phase has been initialized, map solid to account for 'smeared' interface @@ -622,7 +631,7 @@ int main(int argc, char **argv) // ScaLBL_Comm.SendHalo(Phi); // ScaLBL_Comm.RecvHalo(Phi); // ScaLBL_DeviceBarrier(); -// comm.barrier(); +// MPI_Barrier(comm); // //************************************************************************* /* 18 */ @@ -661,7 +670,7 @@ int main(int argc, char **argv) //.......create and start timer............ double starttime,stoptime,cputime; - comm.barrier(); + MPI_Barrier(comm); starttime = MPI_Wtime(); /* @@ -795,7 +804,7 @@ int main(int argc, char **argv) } //................................................................................... ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); // Timestep completed! timestep++; @@ -809,7 +818,7 @@ int main(int argc, char **argv) } //************************************************************************/ ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -826,7 +835,7 @@ int main(int argc, char **argv) NULL_USE(RESTART_INTERVAL); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); //**************************************************** } diff --git a/tests/lbpm_nonnewtonian_simulator.h b/tests/lbpm_nonnewtonian_simulator.h index 4df5e628..20da1ac3 100644 --- a/tests/lbpm_nonnewtonian_simulator.h +++ b/tests/lbpm_nonnewtonian_simulator.h @@ -1,7 +1,7 @@ // Run the analysis, blob identification, and write restart files #include "common/Array.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" //#define ANALYSIS_INTERVAL 6 @@ -9,9 +9,20 @@ #define BLOBID_INTERVAL 1000 + + + + enum AnalysisType{ AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02, CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20 }; + + + + + + + template void DeleteArray( const TYPE *p ) { @@ -19,6 +30,12 @@ void DeleteArray( const TYPE *p ) } + + + + + + // Structure used to store ids struct AnalysisWaitIdStruct { ThreadPool::thread_id_t blobID; @@ -28,6 +45,7 @@ struct AnalysisWaitIdStruct { }; + // Helper class to write the restart file from a seperate thread class WriteRestartWorkItem: public ThreadPool::WorkItem { @@ -66,9 +84,9 @@ typedef std::shared_ptr > BlobIDList; // timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_), // phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_) // { -// newcomm = Utilities::MPI(MPI_COMM_WORLD).dup(); +// MPI_Comm_dup(MPI_COMM_WORLD,&newcomm); // } -// ~BlobIdentificationWorkItem1() {} +// ~BlobIdentificationWorkItem1() { MPI_Comm_free(&newcomm); } // virtual void run() { // // Compute the global blob id and compare to the previous version // PROFILE_START("Identify blobs",1); @@ -88,7 +106,7 @@ typedef std::shared_ptr > BlobIDList; // const DoubleArray& dist; // BlobIDstruct last_id, new_index, new_id; // BlobIDList new_list; -// Utilities::MPI newcomm; +// MPI_Comm newcomm; //}; // @@ -104,9 +122,9 @@ typedef std::shared_ptr > BlobIDList; // timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_), // phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_) // { -// newcomm = Utilities::MPI(MPI_COMM_WORLD).dup(); +// MPI_Comm_dup(MPI_COMM_WORLD,&newcomm); // } -// ~BlobIdentificationWorkItem2() { } +// ~BlobIdentificationWorkItem2() { MPI_Comm_free(&newcomm); } // virtual void run() { // // Compute the global blob id and compare to the previous version // PROFILE_START("Identify blobs maps",1); @@ -140,7 +158,7 @@ typedef std::shared_ptr > BlobIDList; // const DoubleArray& dist; // BlobIDstruct last_id, new_index, new_id; // BlobIDList new_list; -// Utilities::MPI newcomm; +// MPI_Comm newcomm; //}; // @@ -153,9 +171,9 @@ public: TwoPhase& Avgerages_, fillHalo& fillData_ ): timestep(timestep_), visData(visData_), Averages(Avgerages_), fillData(fillData_) { - newcomm = Utilities::MPI(MPI_COMM_WORLD).dup(); + MPI_Comm_dup(MPI_COMM_WORLD,&newcomm); } - ~WriteVisWorkItem() {} + ~WriteVisWorkItem() { MPI_Comm_free(&newcomm); } virtual void run() { PROFILE_START("Save Vis",1); ASSERT(visData[0].vars[0]->name=="phase"); @@ -180,7 +198,7 @@ private: std::vector& visData; TwoPhase& Averages; fillHalo& fillData; - Utilities::MPI newcomm; + MPI_Comm newcomm; }; @@ -400,7 +418,7 @@ void run_analysis( int timestep, int restart_interval, // Spawn a thread to write the restart file if ( (type&CreateRestart) != 0 ) { - int rank = comm.getRank(); + int rank = MPI_WORLD_RANK(); // Wait for previous restart files to finish writing (not necessary, but helps to ensure memory usage is limited) tpool.wait(wait.restart); diff --git a/tests/lbpm_permeability_simulator.cpp b/tests/lbpm_permeability_simulator.cpp index eb5e6d4b..dbcfb96b 100644 --- a/tests/lbpm_permeability_simulator.cpp +++ b/tests/lbpm_permeability_simulator.cpp @@ -9,7 +9,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/MRTModel.h" //#define WRITE_SURFACES @@ -24,10 +24,11 @@ using namespace std; int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { if (rank == 0){ printf("********************************************************\n"); @@ -38,7 +39,7 @@ int main(int argc, char **argv) int device=ScaLBL_SetDevice(rank); NULL_USE( device ); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); ScaLBL_MRTModel MRT(rank,nprocs,comm); auto filename = argv[1]; @@ -51,7 +52,7 @@ int main(int argc, char **argv) MRT.VelocityField(); } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_plates_pp.cpp b/tests/lbpm_plates_pp.cpp index acd64f52..8344df47 100644 --- a/tests/lbpm_plates_pp.cpp +++ b/tests/lbpm_plates_pp.cpp @@ -9,15 +9,19 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; @@ -75,7 +79,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -88,7 +92,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** if (nprocs != nprocx*nprocy*nprocz){ @@ -112,7 +116,7 @@ int main(int argc, char **argv) std::shared_ptr Averages( new TwoPhase(Dm) ); - comm.barrier(); + MPI_Barrier(comm); Nz += 2; Nx = Ny = Nz; // Cubic domain @@ -196,7 +200,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_porenetwork_pp.cpp b/tests/lbpm_porenetwork_pp.cpp index 4a6ccda7..496f9d86 100644 --- a/tests/lbpm_porenetwork_pp.cpp +++ b/tests/lbpm_porenetwork_pp.cpp @@ -9,15 +9,19 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; @@ -65,7 +69,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -78,7 +82,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** if (nprocs != nprocx*nprocy*nprocz){ @@ -104,7 +108,7 @@ int main(int argc, char **argv) Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); - comm.barrier(); + MPI_Barrier(comm); Nx += 2; Ny += 2; Nz += 2; @@ -289,7 +293,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_random_pp.cpp b/tests/lbpm_random_pp.cpp index ad4b83cc..07c56e6f 100644 --- a/tests/lbpm_random_pp.cpp +++ b/tests/lbpm_random_pp.cpp @@ -52,10 +52,11 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int InitialWetting; double Saturation; @@ -96,7 +97,7 @@ int main(int argc, char **argv) domain >> Lz; } - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&nx,1,MPI_INT,0,comm); MPI_Bcast(&ny,1,MPI_INT,0,comm); @@ -109,7 +110,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // Check that the number of processors >= the number of ranks if ( rank==0 ) { @@ -421,7 +422,7 @@ int main(int argc, char **argv) fwrite(id,1,N,ID); fclose(ID); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 149ae673..d90dbb04 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -16,10 +16,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //....................................................................... @@ -421,7 +422,7 @@ int main(int argc, char **argv) } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index 1bc89adb..3384e454 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -18,10 +18,12 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { @@ -82,7 +84,7 @@ int main(int argc, char **argv) image >> zStart; } - comm.barrier(); + MPI_Barrier(comm); // Computational domain //................................................. MPI_Bcast(&nx,1,MPI_INT,0,comm); @@ -103,7 +105,7 @@ int main(int argc, char **argv) MPI_Bcast(&yStart,1,MPI_INT,0,comm); MPI_Bcast(&zStart,1,MPI_INT,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // Check that the number of processors >= the number of ranks if ( rank==0 ) { @@ -127,7 +129,7 @@ int main(int argc, char **argv) fclose(SEGDAT); printf("Read segmented data from %s \n",Filename); } - comm.barrier(); + MPI_Barrier(comm); // Get the rank info int N = (nx+2)*(ny+2)*(nz+2); @@ -202,7 +204,7 @@ int main(int argc, char **argv) printf("Ready to recieve data %i at process %i \n", N,rank); MPI_Recv(Dm.id,N,MPI_CHAR,0,15,comm,MPI_STATUS_IGNORE); } - comm.barrier(); + MPI_Barrier(comm); nx+=2; ny+=2; nz+=2; N=nx*ny*nz; @@ -338,7 +340,7 @@ int main(int argc, char **argv) if (!MULTINPUT){ if (rank==0) printf("Writing symmetric domain reflection\n"); - comm.barrier(); + MPI_Barrier(comm); int symrank,sympz; sympz = 2*nprocz - Dm.kproc() -1; symrank = sympz*nprocx*nprocy + Dm.jproc()*nprocx + Dm.iproc(); @@ -364,6 +366,6 @@ int main(int argc, char **argv) fclose(SYMID); } } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); } diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 39cf0bd1..007ff9d1 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -115,10 +115,11 @@ double ReadFromBlock( char *ID, int iproc, int jproc, int kproc, int Nx, int Ny, int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { //....................................................................... // Reading the domain information file @@ -230,7 +231,7 @@ int main(int argc, char **argv) fclose(DIST); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; diff --git a/tests/lbpm_sphere_pp.cpp b/tests/lbpm_sphere_pp.cpp index 2e053eed..98778b8d 100644 --- a/tests/lbpm_sphere_pp.cpp +++ b/tests/lbpm_sphere_pp.cpp @@ -9,7 +9,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" #include "common/SpherePack.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Communication.h" /* @@ -22,11 +22,15 @@ using namespace std; int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) int iproc,jproc,kproc; int sendtag,recvtag; @@ -123,14 +127,14 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Reading the sphere packing \n"); if (rank == 0) ReadSpherePacking(nspheres,cx,cy,cz,rad); - comm.barrier(); + MPI_Barrier(comm); // Broadcast the sphere packing to all processes MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm); MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm); //........................................................................... - comm.barrier(); + MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank == 0){ // Compute the Sauter mean diameter @@ -213,7 +217,7 @@ int main(int argc, char **argv) fclose(ID); // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_squaretube_pp.cpp b/tests/lbpm_squaretube_pp.cpp index c1f05aee..42715773 100644 --- a/tests/lbpm_squaretube_pp.cpp +++ b/tests/lbpm_squaretube_pp.cpp @@ -9,15 +9,19 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/TwoPhase.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" int main(int argc, char **argv) { + //***************************************** + // ***** MPI STUFF **************** + //***************************************** // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; @@ -81,7 +85,7 @@ int main(int argc, char **argv) } // ************************************************************** // Broadcast simulation parameters from rank 0 to all other procs - comm.barrier(); + MPI_Barrier(comm); // Computational domain MPI_Bcast(&Nx,1,MPI_INT,0,comm); MPI_Bcast(&Ny,1,MPI_INT,0,comm); @@ -94,7 +98,7 @@ int main(int argc, char **argv) MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); //................................................. - comm.barrier(); + MPI_Barrier(comm); // ************************************************************** if (nprocs != nprocx*nprocy*nprocz){ @@ -121,7 +125,7 @@ int main(int argc, char **argv) rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); Nz += 2; Nx = Ny = Nz; // Cubic domain @@ -255,7 +259,7 @@ int main(int argc, char **argv) } // **************************************************** - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); // **************************************************** } diff --git a/tests/lbpm_uCT_maskfilter.cpp b/tests/lbpm_uCT_maskfilter.cpp index 857bc4e0..cff41ad7 100644 --- a/tests/lbpm_uCT_maskfilter.cpp +++ b/tests/lbpm_uCT_maskfilter.cpp @@ -14,7 +14,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -30,11 +30,13 @@ int main(int argc, char **argv) { + // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); Utilities::setErrorHandlers(); PROFILE_START("Main"); @@ -149,7 +151,7 @@ int main(int argc, char **argv) } netcdf::close( distid ); - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("ReadDistance"); if (rank==0) printf("Finished reading distance =\n"); @@ -182,7 +184,7 @@ int main(int argc, char **argv) fillFloat[0]->fill( LOCVOL[0] ); } netcdf::close( fid ); - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("ReadVolume"); if (rank==0) printf("Read complete\n"); @@ -445,7 +447,7 @@ int main(int argc, char **argv) PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_uCT_maskfilter",true); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/lbpm_uCT_pp.cpp b/tests/lbpm_uCT_pp.cpp index 6e8d1bde..0285b864 100644 --- a/tests/lbpm_uCT_pp.cpp +++ b/tests/lbpm_uCT_pp.cpp @@ -14,7 +14,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "IO/MeshDatabase.h" #include "IO/Mesh.h" #include "IO/Writer.h" @@ -31,10 +31,11 @@ int main(int argc, char **argv) { // Initialize MPI + int rank, nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); { Utilities::setErrorHandlers(); PROFILE_START("Main"); @@ -187,7 +188,7 @@ int main(int argc, char **argv) fillFloat[0]->fill( LOCVOL[0] ); } netcdf::close( fid ); - comm.barrier(); + MPI_Barrier(comm); PROFILE_STOP("ReadVolume"); if (rank==0) printf("Read complete\n"); @@ -250,15 +251,15 @@ int main(int argc, char **argv) } } } - count_plus = Dm[0]->Comm.sumReduce( count_plus); - count_minus = Dm[0]->Comm.sumReduce( count_minus); + count_plus=sumReduce( Dm[0]->Comm, count_plus); + count_minus=sumReduce( Dm[0]->Comm, count_minus); if (rank==0) printf("minimum value=%f, max value=%f \n",min_value,max_value); if (rank==0) printf("plus=%i, minus=%i \n",count_plus,count_minus); ASSERT( count_plus > 0 && count_minus > 0 ); - comm.barrier(); - mean_plus = Dm[0]->Comm.sumReduce( mean_plus ) / count_plus; - mean_minus = Dm[0]->Comm.sumReduce( mean_minus ) / count_minus; - comm.barrier(); + MPI_Barrier(comm); + mean_plus = sumReduce( Dm[0]->Comm, mean_plus ) / count_plus; + mean_minus = sumReduce( Dm[0]->Comm, mean_minus ) / count_minus; + MPI_Barrier(comm); if (rank==0) printf(" Region 1 mean (+): %f, Region 2 mean (-): %f \n",mean_plus, mean_minus); //if (rank==0) printf("Scale the input data (size = %i) \n",LOCVOL[0].length()); @@ -279,7 +280,7 @@ int main(int argc, char **argv) // Fill the source data for the coarse meshes if (rank==0) printf("Coarsen the mesh for N_levels=%i \n",N_levels); - comm.barrier(); + MPI_Barrier(comm); PROFILE_START("CoarsenMesh"); for (int i=1; i filter(ratio[0],ratio[1],ratio[2]); @@ -295,7 +296,7 @@ int main(int argc, char **argv) printf(" filter_x=%i, filter_y=%i, filter_z=%i \n",int(filter.size(0)),int(filter.size(1)),int(filter.size(2)) ); printf(" ratio= %i,%i,%i \n",int(ratio[0]),int(ratio[1]),int(ratio[2]) ); } - comm.barrier(); + MPI_Barrier(comm); } PROFILE_STOP("CoarsenMesh"); @@ -307,7 +308,7 @@ int main(int argc, char **argv) NonLocalMean.back(), *fillFloat.back(), *Dm.back(), nprocx, rough_cutoff, lamda, nlm_sigsq, nlm_depth); PROFILE_STOP("Solve coarse mesh"); - comm.barrier(); + MPI_Barrier(comm); // Refine the solution PROFILE_START("Refine distance"); @@ -321,7 +322,7 @@ int main(int argc, char **argv) rough_cutoff, lamda, nlm_sigsq, nlm_depth); } PROFILE_STOP("Refine distance"); - comm.barrier(); + MPI_Barrier(comm); // Perform a final filter PROFILE_START("Filtering final domains"); @@ -417,14 +418,14 @@ int main(int argc, char **argv) meshData[0].vars.push_back(filter_Dist2_var); fillDouble[0]->copy( filter_Dist2, filter_Dist2_var->data ); #endif - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf("Writing output \n"); // Write visulization data IO::writeData( 0, meshData, comm ); if (rank==0) printf("Finished. \n"); // Compute the Minkowski functionals - comm.barrier(); + MPI_Barrier(comm); auto Averages = std::make_shared(Dm[0]); Array phase_label(Nx[0]+2,Ny[0]+2,Nz[0]+2); @@ -456,7 +457,7 @@ int main(int argc, char **argv) } PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_uCT_pp",true); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return 0; } diff --git a/tests/testCommunication.cpp b/tests/testCommunication.cpp index 911ef1c5..57ce0959 100644 --- a/tests/testCommunication.cpp +++ b/tests/testCommunication.cpp @@ -6,7 +6,7 @@ #include #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Array.h" using namespace std; @@ -15,9 +15,11 @@ using namespace std; //*************************************************************************************** -int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int nprocz ) +int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz ) { - int rank = comm.getRank(); + int rank,nprocs; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); int iproc,jproc,kproc; int sendtag,recvtag; if (rank==0) printf("\nRunning test %i %i %i\n",nprocx,nprocy,nprocz); @@ -36,7 +38,7 @@ int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); //********************************** @@ -83,7 +85,7 @@ int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0; sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0; - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("SendLists are ready on host\n"); //...................................................................................... // Use MPI to fill in the recvCounts form the associated processes @@ -156,7 +158,7 @@ int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz, rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz ); - comm.barrier(); + MPI_Barrier(comm); if (rank==0) printf ("RecvLists finished\n"); // Free memory @@ -179,9 +181,11 @@ int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int template -int testHalo( const Utilities::MPI& comm, int nprocx, int nprocy, int nprocz, int depth ) +int testHalo( MPI_Comm comm, int nprocx, int nprocy, int nprocz, int depth ) { - int rank = comm.getRank(); + int rank,nprocs; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); if ( rank==0 ) printf("\nRunning Halo test %i %i %i %i\n",nprocx,nprocy,nprocz,depth); @@ -251,10 +255,11 @@ int testHalo( const Utilities::MPI& comm, int nprocx, int nprocy, int nprocz, in int main(int argc, char **argv) { // Initialize MPI + int rank,nprocs; MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - int rank = comm.getRank(); - int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); // Run the test with different domains int N_errors = 0; @@ -284,9 +289,10 @@ int main(int argc, char **argv) } // Finished - comm.barrier(); - int N_errors_global = comm.sumReduce( N_errors ); - comm.barrier(); + MPI_Barrier(comm); + int N_errors_global=0; + MPI_Allreduce( &N_errors, &N_errors_global, 1, MPI_INT, MPI_SUM, comm ); + MPI_Barrier(comm); MPI_Finalize(); if ( rank==0 ) { if ( N_errors_global==0 ) diff --git a/tests/test_dcel_minkowski.cpp b/tests/test_dcel_minkowski.cpp index 2669b522..0d6cbca9 100644 --- a/tests/test_dcel_minkowski.cpp +++ b/tests/test_dcel_minkowski.cpp @@ -26,9 +26,9 @@ std::shared_ptr loadInputs( ) int main(int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); - //int rank = comm.getRank(); - //int nprocs = comm.getSize(); + MPI_Comm comm = MPI_COMM_WORLD; + //int rank = MPI_WORLD_RANK(); + //int nprocs = MPI_WORLD_SIZE(); int toReturn = 0; { int i,j,k; @@ -99,7 +99,7 @@ int main(int argc, char **argv) } PROFILE_SAVE("test_dcel_minkowski"); - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return toReturn; } diff --git a/tests/test_dcel_tri_normal.cpp b/tests/test_dcel_tri_normal.cpp index b6497140..1e85b1f3 100644 --- a/tests/test_dcel_tri_normal.cpp +++ b/tests/test_dcel_tri_normal.cpp @@ -26,7 +26,7 @@ std::shared_ptr loadInputs( ) int main(int argc, char **argv) { MPI_Init(&argc,&argv); - Utilities::MPI comm( MPI_COMM_WORLD ); + MPI_Comm comm = MPI_COMM_WORLD; int toReturn = 0; { int i,j,k; @@ -136,7 +136,7 @@ int main(int argc, char **argv) if (count_check > 0) toReturn=2; else printf("Succeeded. \n"); } - comm.barrier(); + MPI_Barrier(comm); MPI_Finalize(); return toReturn; } From 679c53a4690876755a64a44c79d4db856c58dd01 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 19 Mar 2020 13:35:10 -0400 Subject: [PATCH 048/270] Add wall factor to morphgrow to change solid penalty term --- analysis/morphology.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/analysis/morphology.cpp b/analysis/morphology.cpp index 72a17892..8f658328 100644 --- a/analysis/morphology.cpp +++ b/analysis/morphology.cpp @@ -692,6 +692,8 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, int Nz = Dm->Nz; int rank = Dm->rank(); + double WALL_FACTOR = 0.0; // 1.0 if you want to penalize movements close to solid + double count=0.0; for (int k=1; k &id, for (int j=1; j MAX_DISPLACEMENT) MAX_DISPLACEMENT= fabs(wallweight*morph_delta); if (Dist(i,j,k) - wallweight*morph_delta < 0.0){ @@ -769,7 +770,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, for (int j=1; j Date: Thu, 19 Mar 2020 13:41:31 -0400 Subject: [PATCH 049/270] Add wall factor to morphgrow to change solid penalty term --- analysis/morphology.cpp | 10 ++++------ analysis/morphology.h | 2 +- models/ColorModel.cpp | 3 ++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/analysis/morphology.cpp b/analysis/morphology.cpp index 8f658328..f6bb3469 100644 --- a/analysis/morphology.cpp +++ b/analysis/morphology.cpp @@ -685,15 +685,13 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr &id, std::shared_ptr Dm, double TargetGrowth) +double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, std::shared_ptr Dm, double TargetGrowth, double WallFactor) { int Nx = Dm->Nx; int Ny = Dm->Ny; int Nz = Dm->Nz; int rank = Dm->rank(); - - double WALL_FACTOR = 0.0; // 1.0 if you want to penalize movements close to solid - + double count=0.0; for (int k=1; k &id, for (int j=1; j MAX_DISPLACEMENT) MAX_DISPLACEMENT= fabs(wallweight*morph_delta); if (Dist(i,j,k) - wallweight*morph_delta < 0.0){ @@ -770,7 +768,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, for (int j=1; j Dm, double VoidFraction, signed char ErodeLabel, signed char ReplaceLabel); double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr Dm, double VoidFraction); -double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, std::shared_ptr Dm, double TargetVol); +double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array &id, std::shared_ptr Dm, double TargetVol, double WallFactor); diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 4ef7573f..05004110 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1296,6 +1296,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta double vF = 0.f; double vS = 0.f; double delta_volume; + double WallFactor = 0.0; DoubleArray phase(Nx,Ny,Nz); IntArray phase_label(Nx,Ny,Nz);; @@ -1395,7 +1396,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta double target_delta_volume_incremental = target_delta_volume; if (fabs(target_delta_volume) > 0.01*volume_initial) target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); for (int k=0; k Date: Sat, 21 Mar 2020 09:45:43 -0400 Subject: [PATCH 050/270] make sure input database is updated across all ranks --- analysis/runAnalysis.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 6c76f58b..384d4d69 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -907,9 +907,8 @@ void runAnalysis::run(int timestep, std::shared_ptr input_db, TwoPhase // Spawn a thread to write the restart file // if ( matches(type,AnalysisType::CreateRestart) ) { if (timestep%d_restart_interval==0){ - + input_db->putScalar( "Restart", true ); if (d_rank==0) { - input_db->putScalar( "Restart", true ); std::ofstream OutStream("Restart.db"); input_db->print(OutStream, ""); OutStream.close(); @@ -1010,10 +1009,11 @@ void runAnalysis::basic(int timestep, std::shared_ptr input_db, SubPha ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double)); ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double)); + color_db->putScalar("timestep",timestep); + color_db->putScalar( "Restart", true ); + input_db->putDatabase("Color", color_db); + if (d_rank==0) { - color_db->putScalar("timestep",timestep); - color_db->putScalar( "Restart", true ); - input_db->putDatabase("Color", color_db); std::ofstream OutStream("Restart.db"); input_db->print(OutStream, ""); OutStream.close(); From 8d9f35d1d384e26ba84fb2e3bcdc7318a43eac4f Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 21 Mar 2020 09:45:57 -0400 Subject: [PATCH 051/270] updating R helper functions --- example/Workflow/HelperFunctions.R | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/example/Workflow/HelperFunctions.R b/example/Workflow/HelperFunctions.R index 6c8bd903..669b28fe 100644 --- a/example/Workflow/HelperFunctions.R +++ b/example/Workflow/HelperFunctions.R @@ -7,19 +7,20 @@ ReadDatabase<-function(FILE){ INPUT<-gsub(';','',readLines(FILE)) S<-gsub('tauA = ','',gsub("\\s+"," ",(grep("tauA",INPUT,value=TRUE)))) - TAU_A = as.numeric(S) + TAU_A = as.numeric(gsub("/.*","",S)) S<-gsub('tauB = ','',gsub("\\s+"," ",(grep("tauB",INPUT,value=TRUE)))) - TAU_B = as.numeric(S) + TAU_B = as.numeric(gsub("/.*","",S)) S<-gsub('rhoA = ','',gsub("\\s+"," ",(grep("rhoA",INPUT,value=TRUE)))) - RHO_A = as.numeric(S) + RHO_A = as.numeric(gsub("/.*","",S)) S<-gsub('rhoB = ','',gsub("\\s+"," ",(grep("rhoB",INPUT,value=TRUE)))) - RHO_B = as.numeric(S) + RHO_B = as.numeric(gsub("/.*","",S)) S<-gsub('alpha = ','',gsub("\\s+"," ",(grep("alpha",INPUT,value=TRUE)))) - ALPHA = as.numeric(S) + ALPHA = as.numeric(gsub("/.*","",S)) # Read the affinity S<-gsub('ComponentAffinity = ','',gsub("\\s+"," ",(grep("ComponentAffinity",INPUT,value=TRUE)))) + S<-gsub("/.*","",S) AFFINITY<-as.numeric(unlist(strsplit(S,", "))) PARAMETERS<-c(TAU_A,TAU_B,RHO_A,RHO_B,ALPHA,AFFINITY) From ad20322f31c01d3604b321553a113e55bc972e49 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 21 Mar 2020 09:53:47 -0400 Subject: [PATCH 052/270] refactor refine pp tool --- tests/lbpm_refine_pp.cpp | 41 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index d90dbb04..ad729aa2 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -40,7 +40,6 @@ int main(int argc, char **argv) auto domain_db = db->getDatabase( "Domain" ); // Read domain parameters - auto L = domain_db->getVector( "L" ); auto size = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); auto ReadValues = domain_db->getVector( "ReadValues" ); @@ -92,8 +91,42 @@ int main(int argc, char **argv) } } Dm.CommInit(); - + + Domain Mask(rnx,rny,rnz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Mask->ReadIDs(); + Mask.CommInit(); + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array Labels(nx,ny,nz); DoubleArray SignDist(nx,ny,nz); + + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) Labels(i,j,k) = 1; + else Labels(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs(i,j,k) = 2.0*double(Labels(i,j,k))-1.0; + } + } + } + // MeanFilter(Averages->SDs); + if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + CalcDist(SignDist,Labels,*Mask); + // Read the signed distance from file sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"rb"); @@ -102,7 +135,7 @@ int main(int argc, char **argv) if (ReadSignDist != size_t(N)) printf("lbpm_refine_pp: Error reading signed distance function (rank=%i)\n",rank); fclose(DIST); - char *Labels; + /* char *Labels; Labels = new char[N]; sprintf(LocalRankFilename,"ID.%05i",rank); FILE *LABELS = fopen(LocalRankFilename,"rb"); @@ -110,7 +143,7 @@ int main(int argc, char **argv) ReadLabels=fread(Labels,1,N,LABELS); if (ReadLabels != size_t(N)) printf("lbpm_refine_pp: Error reading ID (rank=%i)\n",rank); fclose(LABELS); - +*/ if ( rank==0 ) printf("Set up Domain, read input distance \n"); DoubleArray RefinedSignDist(rnx,rny,rnz); From afbef5075208d4fcf8308a0bc1f8b5034480a322 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 21 Mar 2020 10:03:20 -0400 Subject: [PATCH 053/270] update lbpm_refine_pp --- tests/lbpm_refine_pp.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index ad729aa2..5f1c5875 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -12,6 +12,7 @@ #include "common/Communication.h" #include "common/Domain.h" #include "analysis/pmmc.h" +#include "analysis/distance.h" int main(int argc, char **argv) { @@ -93,10 +94,9 @@ int main(int argc, char **argv) Dm.CommInit(); Domain Mask(rnx,rny,rnz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - Mask->ReadIDs(); + Mask.ReadIDs(); Mask.CommInit(); - for (int i=0; iid[i]; // save what was read - + // Generate the signed distance map // Initialize the domain and communication Array Labels(nx,ny,nz); @@ -108,7 +108,7 @@ int main(int argc, char **argv) for (int i=0;iid[n]; + signed char label = Mask.id[n]; if (label > 0) Labels(i,j,k) = 1; else Labels(i,j,k) = 0; } @@ -119,13 +119,13 @@ int main(int argc, char **argv) for (int j=0;jSDs(i,j,k) = 2.0*double(Labels(i,j,k))-1.0; + SignDist(i,j,k) = 2.0*double(Labels(i,j,k))-1.0; } } } // MeanFilter(Averages->SDs); if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); - CalcDist(SignDist,Labels,*Mask); + CalcDist(SignDist,Labels,Mask); // Read the signed distance from file sprintf(LocalRankFilename,"SignDist.%05i",rank); @@ -178,7 +178,7 @@ int main(int argc, char **argv) pt.y=0.5*(rj-1)+1.f; pt.z=0.5*(rk-1)+1.f; RefinedSignDist(ri,rj,rk) = LocalApprox.eval(pt); - RefineLabel(ri,rj,rk) = Labels[k*nx*ny+j*nx+i]; + RefineLabel(ri,rj,rk) = Labels(i,j,k); } } } From b206ad80a22d5d82112bd3137f0af7cf9d12ca1c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 21 Mar 2020 10:06:36 -0400 Subject: [PATCH 054/270] use Filename in refine pp --- tests/lbpm_refine_pp.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 5f1c5875..be2ba346 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -94,7 +94,13 @@ int main(int argc, char **argv) Dm.CommInit(); Domain Mask(rnx,rny,rnz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - Mask.ReadIDs(); + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask.Decomp(Filename); + } + else{ + Mask.ReadIDs(); + } Mask.CommInit(); // Generate the signed distance map From dbbd8e30b7e28951231f9a4c7445894d6e779750 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 21 Mar 2020 10:32:16 -0400 Subject: [PATCH 055/270] fix refine pp --- tests/lbpm_refine_pp.cpp | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index be2ba346..0f0ffdda 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -52,6 +52,7 @@ int main(int argc, char **argv) int nprocx = nproc[0]; int nprocy = nproc[1]; int nprocz = nproc[2]; + int BoundaryCondition=0; // Check that the number of processors >= the number of ranks if ( rank==0 ) { @@ -63,15 +64,26 @@ int main(int argc, char **argv) ERROR("Insufficient number of processors"); } - char LocalRankFilename[40]; + //Domain Mask(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Domain Mask(domain_db,MPI_COMM_WORLD); + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + if (rank==0) printf("Reading domain from %s \n",Filename.c_str()); + Mask.Decomp(Filename); + if (rank==0) printf("Complete. \n"); + } + else{ + Mask.ReadIDs(); + } + Mask.CommInit(); + char LocalRankFilename[40]; int rnx=2*nx; int rny=2*ny; int rnz=2*nz; if (rank==0) printf("Refining mesh to %i x %i x %i \n",rnx,rny,rnz); - int BoundaryCondition=0; Domain Dm(rnx,rny,rnz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); // Communication the halos @@ -83,6 +95,7 @@ int main(int argc, char **argv) int N = nx*ny*nz; // Define communication sub-domain -- everywhere + if (rank==0) printf("Initialize refined domain \n"); for (int k=0; kkeyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask.Decomp(Filename); - } - else{ - Mask.ReadIDs(); - } - Mask.CommInit(); - // Generate the signed distance map // Initialize the domain and communication Array Labels(nx,ny,nz); @@ -133,7 +136,7 @@ int main(int argc, char **argv) if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); CalcDist(SignDist,Labels,Mask); - // Read the signed distance from file + /* // Read the signed distance from file sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"rb"); size_t ReadSignDist; @@ -141,7 +144,7 @@ int main(int argc, char **argv) if (ReadSignDist != size_t(N)) printf("lbpm_refine_pp: Error reading signed distance function (rank=%i)\n",rank); fclose(DIST); - /* char *Labels; + char *Labels; Labels = new char[N]; sprintf(LocalRankFilename,"ID.%05i",rank); FILE *LABELS = fopen(LocalRankFilename,"rb"); From 05ed256b30ab969000fc76750cedad22fe4ded05 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 21 Mar 2020 10:37:32 -0400 Subject: [PATCH 056/270] adding refine options --- tests/lbpm_refine_pp.cpp | 433 ++++++++++++++++----------------------- 1 file changed, 173 insertions(+), 260 deletions(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 0f0ffdda..1a7ff05b 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -192,277 +192,190 @@ int main(int argc, char **argv) } } fillData.fill(RefinedSignDist); - // sprintf(LocalRankFilename,"ID.%05i",rank); - //FILE *ID = fopen(LocalRankFilename,"wb"); - //fwrite(id,1,N,ID); - //fclose(ID); -/* - sprintf(LocalRankFilename,"RefineDist.%05i",rank); - FILE *REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(RefinedSignDist.data(),8,rnx*rny*rnz,REFINEDIST); - fclose(REFINEDIST); -*/ - if ( rank==0 ) printf("Write output \n"); - DoubleArray BlockDist(nx,ny,nz); - FILE *WRITEID, *REFINEDIST; - char * id; - id = new char [N]; - int writerank; - // Write output blocks with the same sub-domain size as origina - // refinement increases the size of the process grid - writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc(); - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i,j,k); + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + if ( rank==0 ) printf("Write output \n"); + sprintf(LocalRankFilename,Filename.c_str(),".refine"); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(RefineLabel.data(),1,rnx*rny*rnz,WRITEID); + fclose(WRITEID); + } + else{ + DoubleArray BlockDist(nx,ny,nz); + FILE *WRITEID, *REFINEDIST; + char * id; + id = new char [N]; + int writerank; + + // Write output blocks with the same sub-domain size as origina + // refinement increases the size of the process grid + writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc(); + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i,j,k); + } } } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); - -/* for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]= 0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc()+1; - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j,k); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); - -/* for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); - writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1; - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j+ny-2,k); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); -/* for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; + writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc()+1; + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j,k); + } } } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc(); - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i,j+ny-2,k); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); -/* - for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc(); - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i,j,k+nz-2); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); -/* - for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc()+1; - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j,k+nz-2); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); - -/* for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc(); - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i,j+ny-2,k+nz-2); - } - } - } - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); -/* - for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); - - writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1; - for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; - else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j+ny-2,k+nz-2); - } - } - } - - sprintf(LocalRankFilename,"RefineDist.%05i",writerank); - REFINEDIST = fopen(LocalRankFilename,"wb"); - fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); - fclose(REFINEDIST); - -/* for (int k=0; k 0.f) - id[k*nx*ny + j*nx + i]=2; - else - id[k*nx*ny + j*nx + i]=0; - } - } - } - */ - sprintf(LocalRankFilename,"RefineID.%05i",writerank); - WRITEID = fopen(LocalRankFilename,"wb"); - fwrite(id,1,nx*ny*nz,WRITEID); - fclose(WRITEID); + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + + writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1; + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j+ny-2,k); + } + } + } + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc(); + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i,j+ny-2,k); + } + } + } + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc(); + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i,j,k+nz-2); + } + } + } + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc()+1; + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j,k+nz-2); + } + } + } + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc(); + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i,j+ny-2,k+nz-2); + } + } + } + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + + writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1; + for (int k=0; k 0) id[k*nx*ny + j*nx + i]=2; + else id[k*nx*ny + j*nx + i] = RefineLabel(i+nx-2,j+ny-2,k+nz-2); + } + } + } + + sprintf(LocalRankFilename,"RefineDist.%05i",writerank); + REFINEDIST = fopen(LocalRankFilename,"wb"); + fwrite(BlockDist.data(),8,nx*ny*nz,REFINEDIST); + fclose(REFINEDIST); + + sprintf(LocalRankFilename,"RefineID.%05i",writerank); + WRITEID = fopen(LocalRankFilename,"wb"); + fwrite(id,1,nx*ny*nz,WRITEID); + fclose(WRITEID); + } } MPI_Barrier(comm); MPI_Finalize(); From 8645d0b2a778a47e4c8d9c73d092375b1f19034f Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 21 Mar 2020 10:42:45 -0400 Subject: [PATCH 057/270] write full refined ID --- tests/lbpm_refine_pp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 1a7ff05b..df6edbf1 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -198,7 +198,7 @@ int main(int argc, char **argv) auto Filename = domain_db->getScalar( "Filename" ); if ( rank==0 ) printf("Write output \n"); sprintf(LocalRankFilename,Filename.c_str(),".refine"); - WRITEID = fopen(LocalRankFilename,"wb"); + FILE *WRITEID = fopen("refine.raw","wb"); fwrite(RefineLabel.data(),1,rnx*rny*rnz,WRITEID); fclose(WRITEID); } From 28f3f9dcf8e7d09c2949d1d8cc727dce27fc5165 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 21 Mar 2020 13:02:17 -0400 Subject: [PATCH 058/270] using aggregator to write 1x 2x data --- tests/lbpm_refine_pp.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index df6edbf1..4734c19b 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -187,7 +187,8 @@ int main(int argc, char **argv) pt.y=0.5*(rj-1)+1.f; pt.z=0.5*(rk-1)+1.f; RefinedSignDist(ri,rj,rk) = LocalApprox.eval(pt); - RefineLabel(ri,rj,rk) = Labels(i,j,k); + RefineLabel(ri,rj,rk) = Labels(i,j,k); + Dm.id[n] = Labels(i,j,k); } } } @@ -197,10 +198,11 @@ int main(int argc, char **argv) if (domain_db->keyExists( "Filename" )){ auto Filename = domain_db->getScalar( "Filename" ); if ( rank==0 ) printf("Write output \n"); - sprintf(LocalRankFilename,Filename.c_str(),".refine"); - FILE *WRITEID = fopen("refine.raw","wb"); - fwrite(RefineLabel.data(),1,rnx*rny*rnz,WRITEID); - fclose(WRITEID); + Dm.AggregateLabels("id_2x.raw"); + Mask.AggregateLabels("id.raw"); + //FILE *WRITEID = fopen("refine.raw","wb"); + //fwrite(RefineLabel.data(),1,rnx*rny*rnz,WRITEID); + //fclose(WRITEID); } else{ DoubleArray BlockDist(nx,ny,nz); From 7258867983a421957435ae2934c81bc61607c8d6 Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 31 Mar 2020 15:43:36 -0400 Subject: [PATCH 059/270] update shell aggregation protocol --- models/ColorModel.cpp | 64 +++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 05004110..a7d06409 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1297,6 +1297,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta double vS = 0.f; double delta_volume; double WallFactor = 0.0; + bool USE_CONNECTED_NWP = false; DoubleArray phase(Nx,Ny,Nz); IntArray phase_label(Nx,Ny,Nz);; @@ -1325,32 +1326,55 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta fclose(INPUT); */ // 2. Identify connected components of phase field -> phase_label - BlobIDstruct new_index; - ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(comm); - // only operate on component "0" - count = 0.0; - double second_biggest = 0.0; + if (USE_CONNECTED_NWP){ + BlobIDstruct new_index; + ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); + MPI_Barrier(comm); - for (int k=0; kComm, count); + second_biggest = sumReduce( Dm->Comm, second_biggest); + } + else { + // use the whole NWP + for (int k=0; kSDs(i,j,k) > 0.f){ + if (phase(i,j,k) > 0.f ){ + phase_id(i,j,k) = 0; + } + else { + phase_id(i,j,k) = 1; + } + } + else { + phase_id(i,j,k) = 1; + } } } } - } - double volume_connected = sumReduce( Dm->Comm, count); - second_biggest = sumReduce( Dm->Comm, second_biggest); + } /*int reach_x, reach_y, reach_z; for (int k=0; k Date: Tue, 31 Mar 2020 15:55:05 -0400 Subject: [PATCH 060/270] cloning databse for restart --- analysis/runAnalysis.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 384d4d69..51b0214c 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -907,10 +907,11 @@ void runAnalysis::run(int timestep, std::shared_ptr input_db, TwoPhase // Spawn a thread to write the restart file // if ( matches(type,AnalysisType::CreateRestart) ) { if (timestep%d_restart_interval==0){ + auto Restart_db = input_db->clone(); input_db->putScalar( "Restart", true ); if (d_rank==0) { std::ofstream OutStream("Restart.db"); - input_db->print(OutStream, ""); + Restart_db->print(OutStream, ""); OutStream.close(); } // Write the restart file (using a seperate thread) @@ -1008,22 +1009,21 @@ void runAnalysis::basic(int timestep, std::shared_ptr input_db, SubPha cfq = std::shared_ptr(new double[19*d_Np],DeleteArray); ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double)); ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double)); - - color_db->putScalar("timestep",timestep); - color_db->putScalar( "Restart", true ); - input_db->putDatabase("Color", color_db); - + // clone the input database to avoid modifying shared data + auto Restart_db = input_db->clone(); + auto tmp_color_db = Restart_db.getDatabase( "Color" ); + tmp_color_db.putScalar("timestep",timestep); + tmp_color_db.putScalar( "Restart", true ); + Restart_db.putDatabase("Color", tmp_color_db); if (d_rank==0) { std::ofstream OutStream("Restart.db"); - input_db->print(OutStream, ""); + Restart_db.print(OutStream, ""); OutStream.close(); - } // Write the restart file (using a seperate thread) auto work1 = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np); work1->add_dependency(d_wait_restart); d_wait_restart = d_tpool.add_work(work1); - } if (timestep%d_visualization_interval==0){ From c4f15d8727516207d48ca63031c55ddcfc06945e Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 31 Mar 2020 16:12:24 -0400 Subject: [PATCH 061/270] fixed issue cloning db --- analysis/runAnalysis.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 51b0214c..7150ab31 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -907,12 +907,12 @@ void runAnalysis::run(int timestep, std::shared_ptr input_db, TwoPhase // Spawn a thread to write the restart file // if ( matches(type,AnalysisType::CreateRestart) ) { if (timestep%d_restart_interval==0){ - auto Restart_db = input_db->clone(); - input_db->putScalar( "Restart", true ); + auto Restart_db = input_db->cloneDatabase(); + // Restart_db->putScalar( "Restart", true ); if (d_rank==0) { - std::ofstream OutStream("Restart.db"); - Restart_db->print(OutStream, ""); - OutStream.close(); + // std::ofstream OutStream("Restart.db"); + // Restart_db->print(OutStream, ""); + // OutStream.close(); } // Write the restart file (using a seperate thread) auto work = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np); @@ -1010,14 +1010,14 @@ void runAnalysis::basic(int timestep, std::shared_ptr input_db, SubPha ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double)); ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double)); // clone the input database to avoid modifying shared data - auto Restart_db = input_db->clone(); - auto tmp_color_db = Restart_db.getDatabase( "Color" ); - tmp_color_db.putScalar("timestep",timestep); - tmp_color_db.putScalar( "Restart", true ); - Restart_db.putDatabase("Color", tmp_color_db); + auto Restart_db = input_db->cloneDatabase(); + auto tmp_color_db = Restart_db->getDatabase( "Color" ); + tmp_color_db->putScalar("timestep",timestep); + tmp_color_db->putScalar( "Restart", true ); + Restart_db->putDatabase("Color", tmp_color_db); if (d_rank==0) { std::ofstream OutStream("Restart.db"); - Restart_db.print(OutStream, ""); + Restart_db->print(OutStream, ""); OutStream.close(); } // Write the restart file (using a seperate thread) From 7b67f2acfc7c0dfcd570d5b74fca52d81726eea1 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Tue, 31 Mar 2020 18:05:32 -0400 Subject: [PATCH 062/270] refactor shell aggregation --- models/ColorModel.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a7d06409..25716a1e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1327,6 +1327,8 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta */ // 2. Identify connected components of phase field -> phase_label + double volume_connected = 0.0; + double second_biggest = 0.0; if (USE_CONNECTED_NWP){ BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); @@ -1334,7 +1336,6 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta // only operate on component "0" count = 0.0; - double second_biggest = 0.0; for (int k=0; kComm, count); + volume_connected = sumReduce( Dm->Comm, count); second_biggest = sumReduce( Dm->Comm, second_biggest); } else { @@ -1409,13 +1410,16 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta } } + if (USE_CONNECTED_NWP){ if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){ // if connected volume is less than 2% just delete the whole thing if (rank==0) printf("Connected region has shrunk! \n"); REVERSE_FLOW_DIRECTION = true; } + /* else{*/ if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest ); + } if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); double target_delta_volume_incremental = target_delta_volume; if (fabs(target_delta_volume) > 0.01*volume_initial) From 0d493275b4eb54253c9cc300f012861224e97a8e Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 08:19:59 -0400 Subject: [PATCH 063/270] use kr as target for morph change --- models/ColorModel.cpp | 47 +++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 25716a1e..c6cb563c 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -782,6 +782,20 @@ void ScaLBL_ColorModel::Run(){ double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); + if (SET_CAPILLARY_NUMBER && CURRENT_STEADY_TIMESTEPS%MIN_STEADY_TIMESTEPS < analysis_interval ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + if ( morph_timesteps > morph_interval ){ bool isSteady = false; @@ -789,28 +803,21 @@ void ScaLBL_ColorModel::Run(){ isSteady = true; if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; - - if (SET_CAPILLARY_NUMBER && RESCALE_FORCE_COUNT < RESCALE_FORCE_MAX){ - RESCALE_FORCE_COUNT++; - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } if ( isSteady ){ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + /** morphological target based on relative permeability for A **/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + log_krA = log(krA_TMP); + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); + log_krA_prev = log_krA; + volA_prev = volA; + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); @@ -884,7 +891,6 @@ void ScaLBL_ColorModel::Run(){ Fx *= capillary_number / Ca; Fy *= capillary_number / Ca; Fz *= capillary_number / Ca; - RESCALE_FORCE_COUNT = 1; if (force_mag > 1e-3){ Fx *= 1e-3/force_mag; // impose ceiling for stability Fy *= 1e-3/force_mag; @@ -904,6 +910,7 @@ void ScaLBL_ColorModel::Run(){ Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); color_db->putVector("F",{Fx,Fy,Fz}); } + CURRENT_STEADY_TIMESTEPS = 0; } else{ @@ -979,7 +986,7 @@ void ScaLBL_ColorModel::Run(){ //morph_delta *= (-1.0); REVERSE_FLOW_DIRECTION = false; } - MPI_Barrier(comm); + comm.barrier(); } morph_timesteps += analysis_interval; } From abfe86152f23b2334800a170415a76e6cf3a8ed0 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 1 Apr 2020 12:13:04 -0400 Subject: [PATCH 064/270] fix bug --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 09c8b946..f6d15b43 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1002,7 +1002,7 @@ void ScaLBL_ColorModel::Run(){ //morph_delta *= (-1.0); REVERSE_FLOW_DIRECTION = false; } - comm.barrier(); + MPI_Barrier(comm); } morph_timesteps += analysis_interval; } From e50a099c13eab93a6a302c82c0824c7f4fb89bcc Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:20:21 -0400 Subject: [PATCH 065/270] cleaning up barriers in color model --- models/ColorModel.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index f6d15b43..a62ec927 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -716,7 +716,8 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); // *************EVEN TIMESTEP************* timestep++; @@ -751,10 +752,9 @@ void ScaLBL_ColorModel::Run(){ } ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ - - MPI_Barrier(comm); PROFILE_STOP("Update"); if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ @@ -763,7 +763,6 @@ void ScaLBL_ColorModel::Run(){ // Run the analysis analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - // allow initial ramp-up to get closer to steady state if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ analysis.finish(); @@ -1002,17 +1001,17 @@ void ScaLBL_ColorModel::Run(){ //morph_delta *= (-1.0); REVERSE_FLOW_DIRECTION = false; } - MPI_Barrier(comm); } morph_timesteps += analysis_interval; } + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep @@ -1062,12 +1061,12 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - MPI_Barrier(comm); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); @@ -1442,7 +1441,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta if (USE_CONNECTED_NWP){ BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(comm); + MPI_Barrier(Dm->comm); // only operate on component "0" count = 0.0; From 7ef292e2fcc2a1864a626c1ef5475b6c708eda4e Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:23:00 -0400 Subject: [PATCH 066/270] cleaning up barriers in color model --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a62ec927..b86c0918 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1441,7 +1441,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta if (USE_CONNECTED_NWP){ BlobIDstruct new_index; ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(Dm->comm); + MPI_Barrier(Dm->Comm); // only operate on component "0" count = 0.0; From 64a19a718bc5a6725ef11bb249c5352745549646 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 12:26:55 -0400 Subject: [PATCH 067/270] make ScaLBL communicator public for --- common/ScaLBL.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a50ab7ed..0d2ee0cf 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -134,6 +134,7 @@ public: //ScaLBL_Communicator(Domain &Dm, IntArray &Map); ~ScaLBL_Communicator(); //...................................................................................... + MPI_Comm MPI_COMM_SCALBL; // MPI Communicator unsigned long int CommunicationCount,SendCount,RecvCount; int Nx,Ny,Nz,N; int BoundaryCondition; @@ -207,7 +208,6 @@ private: // Give the object it's own MPI communicator RankInfoStruct rank_info; MPI_Group Group; // Group of processors associated with this domain - MPI_Comm MPI_COMM_SCALBL; // MPI Communicator for this domain MPI_Request req1[18],req2[18]; MPI_Status stat1[18],stat2[18]; //...................................................................................... From 50b84071456fb6806c734735730c6982384420e9 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 13:13:57 -0400 Subject: [PATCH 068/270] clean up ScaLBL barriers --- cpu/Extras.cpp | 2 +- gpu/Extras.cu | 1 + models/ColorModel.cpp | 4 +--- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index 71f5c04a..efe820d3 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -49,5 +49,5 @@ extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t siz } extern "C" void ScaLBL_DeviceBarrier(){ -// cudaDeviceSynchronize(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/gpu/Extras.cu b/gpu/Extras.cu index 8aeedc87..cd9c265c 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -59,4 +59,5 @@ extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ extern "C" void ScaLBL_DeviceBarrier(){ cudaDeviceSynchronize(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b86c0918..57e50411 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -717,7 +717,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + ; // *************EVEN TIMESTEP************* timestep++; @@ -753,7 +753,6 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ PROFILE_STOP("Update"); @@ -1011,7 +1010,6 @@ void ScaLBL_ColorModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep From f12f8154b12b5aa4fc2a124a1a615ec3735bb176 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 1 Apr 2020 13:15:24 -0400 Subject: [PATCH 069/270] Revert "clean up ScaLBL barriers" This reverts commit 50b84071456fb6806c734735730c6982384420e9. --- cpu/Extras.cpp | 2 +- gpu/Extras.cu | 1 - models/ColorModel.cpp | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpu/Extras.cpp b/cpu/Extras.cpp index efe820d3..71f5c04a 100644 --- a/cpu/Extras.cpp +++ b/cpu/Extras.cpp @@ -49,5 +49,5 @@ extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t siz } extern "C" void ScaLBL_DeviceBarrier(){ - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) +// cudaDeviceSynchronize(); } diff --git a/gpu/Extras.cu b/gpu/Extras.cu index cd9c265c..8aeedc87 100644 --- a/gpu/Extras.cu +++ b/gpu/Extras.cu @@ -59,5 +59,4 @@ extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){ extern "C" void ScaLBL_DeviceBarrier(){ cudaDeviceSynchronize(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL) } diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 57e50411..b86c0918 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -717,7 +717,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); - ; + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); // *************EVEN TIMESTEP************* timestep++; @@ -753,6 +753,7 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ PROFILE_STOP("Update"); @@ -1010,6 +1011,7 @@ void ScaLBL_ColorModel::Run(){ PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep From d1d626ac414eff6306669c1d70ba904a73238b45 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 2 Apr 2020 10:38:14 -0400 Subject: [PATCH 070/270] fix header in greyscale --- models/GreyscaleModel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index a99925b1..ea807048 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -10,7 +10,8 @@ Implementation of color lattice boltzmann model #include #include "common/Communication.h" -#include "common/MPI.h" +//#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "common/Database.h" #include "common/ScaLBL.h" #include "ProfilerApp.h" From ce7d348a206aa2fc70089202e522a54a27334863 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 2 Apr 2020 10:43:10 -0400 Subject: [PATCH 071/270] fix sumReduce --- models/GreyscaleModel.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 11d92c80..c28c88c5 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -261,8 +261,7 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm // Set Dm to match Mask for (int i=0; iid[i] = Mask->id[i]; - for (int idx=0; idxComm.sumReduce(label_count[idx]); - + for (int idx=0; idxComm, label_count[idx]); //Initialize a weighted porosity after considering grey voxels GreyPorosity=0.0; for (unsigned int idx=0; idxComm); //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); + vax = sumReduce( Mask->Comm, vax_loc); + vay = sumReduce( Mask->Comm, vay_loc); + vaz = sumReduce( Mask->Comm, vaz_loc); + count = sumReduce( Mask->Comm, count_loc); vax /= count; vay /= count; @@ -634,10 +633,10 @@ void ScaLBL_GreyscaleModel::Run(){ double As = Morphology.A(); double Hs = Morphology.H(); double Xs = Morphology.X(); - Vs = Dm->Comm.sumReduce( Vs); - As = Dm->Comm.sumReduce( As); - Hs = Dm->Comm.sumReduce( Hs); - Xs = Dm->Comm.sumReduce( Xs); + Vs = sumReduce( Dm->Comm, Vs); + As = sumReduce( Dm->Comm, As); + Hs = sumReduce( Dm->Comm, Hs); + Xs = sumReduce( Dm->Comm, Xs); double h = Dm->voxel_length; //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; From b4a51e266b43f838ae96af26f702cea6486b810f Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 2 Apr 2020 10:55:04 -0400 Subject: [PATCH 072/270] remove warnings for greyscale --- tests/lbpm_greyscale_simulator.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index a54b6fc4..b17778ce 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -8,7 +8,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "models/GreyscaleModel.h" //#define WRITE_SURFACES @@ -33,8 +33,6 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; if (rank == 0){ printf("********************************************************\n"); @@ -43,6 +41,7 @@ int main(int argc, char **argv) } // Initialize compute device int device=ScaLBL_SetDevice(rank); + NULL_USE(device); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); From 4398b09cc08ae3191cc429e3cea331a8ee5919e2 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 2 Apr 2020 12:53:54 -0400 Subject: [PATCH 073/270] enabling endpoint adaptation for color model --- models/ColorModel.cpp | 50 ++++++++++++++++++++--------------------- models/GreyscaleModel.h | 1 - 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b86c0918..7af1cafe 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -500,6 +500,7 @@ void ScaLBL_ColorModel::Run(){ bool USE_SEED = false; bool USE_DIRECT = false; bool USE_MORPHOPEN_OIL = false; + bool USE_TARGET_VOLUME_CHANGE = false; int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine int MIN_STEADY_TIMESTEPS = 100000; int MAX_STEADY_TIMESTEPS = 200000; @@ -523,11 +524,11 @@ void ScaLBL_ColorModel::Run(){ bool USE_BUMP_RATE = false; /* history for morphological algoirthm */ - double KRA_MORPH_FACTOR=0.8; + double KRA_MORPH_FACTOR=0.5; double volA_prev = 0.0; double log_krA_prev = 1.0; double log_krA_target = 1.0; - double log_krA = 0.0; + double log_krA = 1.0; double slope_krA_volume = 0.0; if (color_db->keyExists( "vol_A_previous" )){ volA_prev = color_db->getScalar( "vol_A_previous" ); @@ -555,17 +556,19 @@ void ScaLBL_ColorModel::Run(){ seed_water = 0.01; USE_SEED = true; USE_MORPH = true; + USE_TARGET_VOLUME_CHANGE = true; } else if (protocol == "open connected oil"){ morph_delta = 0.05; USE_MORPH = true; USE_MORPHOPEN_OIL = true; + USE_TARGET_VOLUME_CHANGE = true; } else if (protocol == "shell aggregation"){ morph_delta = 0.05; USE_MORPH = true; + USE_TARGET_VOLUME_CHANGE = true; } - if (color_db->keyExists( "residual_endpoint_threshold" )){ RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar( "residual_endpoint_threshold" ); } @@ -822,16 +825,28 @@ void ScaLBL_ColorModel::Run(){ if ( isSteady ){ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; - //delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - /** morphological target based on relative permeability for A **/ + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); log_krA = log(krA_TMP); - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + } + } log_krA_prev = log_krA; volA_prev = volA; - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + //******************************** **/ /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); @@ -977,14 +992,6 @@ void ScaLBL_ColorModel::Run(){ CURRENT_STEADY_TIMESTEPS=0; initial_volume = volA*Dm->Volume; delta_volume = 0.0; - if (USE_DIRECT){ - //BoundaryCondition = 0; - //ScaLBL_Comm->BoundaryCondition = 0; - //ScaLBL_Comm_Regular->BoundaryCondition = 0; - //Fx = capillary_number*dir_x*force_mag / Ca; - //Fy = capillary_number*dir_y*force_mag / Ca; - //Fz = capillary_number*dir_z*force_mag / Ca; - } } else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { MORPH_ADAPT = false; @@ -992,15 +999,6 @@ void ScaLBL_ColorModel::Run(){ initial_volume = volA*Dm->Volume; delta_volume = 0.0; } - if ( REVERSE_FLOW_DIRECTION ){ - //if (rank==0) printf("*****REVERSE FLOW DIRECTION***** \n"); - delta_volume = 0.0; - // flow direction will reverse after next steady point - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - //morph_delta *= (-1.0); - REVERSE_FLOW_DIRECTION = false; - } } morph_timesteps += analysis_interval; } diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index ea807048..c670239f 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -10,7 +10,6 @@ Implementation of color lattice boltzmann model #include #include "common/Communication.h" -//#include "common/MPI.h" #include "common/MPI_Helpers.h" #include "common/Database.h" #include "common/ScaLBL.h" From 7f83f55e1bc3e476ad2d8c85a0f1fa166083caf2 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 07:16:44 -0400 Subject: [PATCH 074/270] clean up target Ca --- models/ColorModel.cpp | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 7af1cafe..ddf669bb 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -800,20 +800,6 @@ void ScaLBL_ColorModel::Run(){ double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); - if (SET_CAPILLARY_NUMBER && CURRENT_STEADY_TIMESTEPS%MIN_STEADY_TIMESTEPS < analysis_interval ){ - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - if ( morph_timesteps > morph_interval ){ bool isSteady = false; @@ -926,16 +912,6 @@ void ScaLBL_ColorModel::Run(){ Fy *= 1e-3/force_mag; Fz *= 1e-3/force_mag; } - if (flow_rate_A < NOISE_THRESHOLD && USE_BUMP_RATE){ - if (rank==0) printf("Hit noise threshold (%f): bumping capillary number by %f X \n",NOISE_THRESHOLD,BUMP_RATE); - Fx *= BUMP_RATE; // impose bump condition - Fy *= BUMP_RATE; - Fz *= BUMP_RATE; - capillary_number *= BUMP_RATE; - color_db->putScalar("capillary_number",capillary_number); - current_db->putDatabase("Color", color_db); - MORPH_ADAPT = false; // re-run current point if below noise threshold - } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); color_db->putVector("F",{Fx,Fy,Fz}); @@ -1293,7 +1269,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ count= sumReduce( Dm->Comm, count); mass_loss= sumReduce( Dm->Comm, mass_loss); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_lojavascript:void(0)ss,count); // Need to initialize Aq, Bq, Den, Phi directly //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); From e7e14a9b642a1ce6b1812fdc04660cfbf0b93e6f Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 07:29:14 -0400 Subject: [PATCH 075/270] clean up flow adapt --- models/ColorModel.cpp | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index ddf669bb..a35c3c44 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -500,7 +500,6 @@ void ScaLBL_ColorModel::Run(){ bool USE_SEED = false; bool USE_DIRECT = false; bool USE_MORPHOPEN_OIL = false; - bool USE_TARGET_VOLUME_CHANGE = false; int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine int MIN_STEADY_TIMESTEPS = 100000; int MAX_STEADY_TIMESTEPS = 200000; @@ -518,10 +517,6 @@ void ScaLBL_ColorModel::Run(){ double initial_volume = 0.0; double delta_volume = 0.0; double delta_volume_target = 0.0; - double RESIDUAL_ENDPOINT_THRESHOLD = 0.04; - double NOISE_THRESHOLD = 0.0; - double BUMP_RATE = 2.0; - bool USE_BUMP_RATE = false; /* history for morphological algoirthm */ double KRA_MORPH_FACTOR=0.5; @@ -569,18 +564,6 @@ void ScaLBL_ColorModel::Run(){ USE_MORPH = true; USE_TARGET_VOLUME_CHANGE = true; } - if (color_db->keyExists( "residual_endpoint_threshold" )){ - RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar( "residual_endpoint_threshold" ); - } - NULL_USE( RESIDUAL_ENDPOINT_THRESHOLD ); - if (color_db->keyExists( "noise_threshold" )){ - NOISE_THRESHOLD = color_db->getScalar( "noise_threshold" ); - USE_BUMP_RATE = true; - } - if (color_db->keyExists( "bump_rate" )){ - BUMP_RATE = color_db->getScalar( "bump_rate" ); - USE_BUMP_RATE = true; - } if (color_db->keyExists( "capillary_number" )){ capillary_number = color_db->getScalar( "capillary_number" ); SET_CAPILLARY_NUMBER=true; @@ -1269,7 +1252,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ count= sumReduce( Dm->Comm, count); mass_loss= sumReduce( Dm->Comm, mass_loss); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_lojavascript:void(0)ss,count); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); // Need to initialize Aq, Bq, Den, Phi directly //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); From 377d259884a61cc815f1c4b230328f3c8b834a71 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 07:30:17 -0400 Subject: [PATCH 076/270] clean up flow adapt --- models/ColorModel.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a35c3c44..02ff6844 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -551,18 +551,15 @@ void ScaLBL_ColorModel::Run(){ seed_water = 0.01; USE_SEED = true; USE_MORPH = true; - USE_TARGET_VOLUME_CHANGE = true; } else if (protocol == "open connected oil"){ morph_delta = 0.05; USE_MORPH = true; USE_MORPHOPEN_OIL = true; - USE_TARGET_VOLUME_CHANGE = true; } else if (protocol == "shell aggregation"){ morph_delta = 0.05; USE_MORPH = true; - USE_TARGET_VOLUME_CHANGE = true; } if (color_db->keyExists( "capillary_number" )){ capillary_number = color_db->getScalar( "capillary_number" ); From 3d31d2672256af61c25c05a25a9743091f970494 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 07:37:29 -0400 Subject: [PATCH 077/270] clean up flow adapt --- models/ColorModel.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 02ff6844..954bca7e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -566,9 +566,6 @@ void ScaLBL_ColorModel::Run(){ SET_CAPILLARY_NUMBER=true; //RESCALE_FORCE_MAX = 1; } -// if (analysis_db->keyExists( "rescale_force_count" )){ -// RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); -// } if (color_db->keyExists( "timestep" )){ timestep = color_db->getScalar( "timestep" ); } From 32f1bae784af2fe8c7e3e380c26cce9e6059ae58 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 08:24:28 -0400 Subject: [PATCH 078/270] don't unpack distributions when external BC are applied (D3Q7/D3Q19) --- common/ScaLBL.cpp | 148 +++++++++++++++++++++++++++++----------------- 1 file changed, 94 insertions(+), 54 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 21656757..3e2d0f07 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1011,19 +1011,6 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N); ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N); //................................................................................... - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N); - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); - //.................................................................................. //...Pack the xy edge (8)................................ ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,dist,N); //...Pack the Xy edge (9)................................ @@ -1032,22 +1019,75 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,dist,N); //...Pack the XY edge (7)................................ ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,dist,N); - //...Pack the xz edge (12)................................ - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N); - //...Pack the xZ edge (14)................................ - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N); - //...Pack the Xz edge (13)................................ - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N); - //...Pack the XZ edge (11)................................ - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N); - //...Pack the yz edge (16)................................ - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N); - //...Pack the yZ edge (18)................................ - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N); - //...Pack the Yz edge (17)................................ - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N); - //...Pack the YZ edge (15)................................ - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N); + + if (BoundaryCondition > 0){ + if (kproc != 0){ + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N); + //...Pack the xz edge (12)................................ + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N); + //...Pack the Xz edge (13)................................ + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N); + //...Pack the yz edge (16)................................ + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N); + //...Pack the Yz edge (17)................................ + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N); + //.................................................................................. + } + if (kproc != nprocz-1){ + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + //...Pack the xZ edge (14)................................ + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N); + //...Pack the XZ edge (11)................................ + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N); + //...Pack the yZ edge (18)................................ + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N); + //...Pack the YZ edge (15)................................ + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N); + //.................................................................................. + } + } + else { + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N); + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N); + //.................................................................................. + //...Pack the xz edge (12)................................ + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N); + //...Pack the xZ edge (14)................................ + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N); + //...Pack the Xz edge (13)................................ + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N); + //...Pack the XZ edge (11)................................ + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N); + //...Pack the yz edge (16)................................ + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N); + //...Pack the yZ edge (18)................................ + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N); + //...Pack the Yz edge (17)................................ + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N); + //...Pack the YZ edge (15)................................ + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N); + } + //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... @@ -1225,18 +1265,18 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N); ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,recvCount_Y,recvCount_Y,recvbuf_Y,Bq,N); //................................................................................... - - if (BoundaryCondition > 0 && kproc == 0){ - // don't unpack little z - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); - } - else if (BoundaryCondition > 0 && kproc == nprocz-1){ - // don't unpack big z - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N); + + if (BoundaryCondition > 0){ + if (kproc != 0){ + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N); + } + if (kproc != nprocz-1){ + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); + } } else { //...Packing for z face(6,12,13,16,17)................................ @@ -1343,19 +1383,19 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){ ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Cq,N); //................................................................................... - if (BoundaryCondition > 0 && kproc == 0){ - // don't unpack little z - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N); - } - else if (BoundaryCondition > 0 && kproc == nprocz-1){ - // don't unpack big z - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N); - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N); + if (BoundaryCondition > 0){ + if (kproc != 0){ + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N); + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N); + } + if (kproc != nprocz-1){ + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N); + } } else { //...Packing for z face(6,12,13,16,17)................................ From e641e2e3ed01bdc9588a541efbd448804724e335 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 08:26:48 -0400 Subject: [PATCH 079/270] remove old comments --- common/ScaLBL.h | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 901e0e3b..3bf50f6f 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -119,11 +119,6 @@ extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double // BOUNDARY CONDITION ROUTINES -//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din, -// int Nx, int Ny, int Nz); -//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout, -// int Nx, int Ny, int Nz, int outlet); - extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np); extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np); @@ -178,18 +173,8 @@ public: int LastInterior(); int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np); -// void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np); -// void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np); -// void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np); -// void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np); -// void SendD3Q19(double *f_even, double *f_odd); -// void RecvD3Q19(double *f_even, double *f_odd); -// void SendD3Q19AA(double *f_even, double *f_odd); -// void RecvD3Q19AA(double *f_even, double *f_odd); void SendD3Q19AA(double *dist); void RecvD3Q19AA(double *dist); -// void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); -// void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); void BiSendD3Q7AA(double *Aq, double *Bq); void BiRecvD3Q7AA(double *Aq, double *Bq); void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq); @@ -206,9 +191,6 @@ public: void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time); double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time); -// void TestSendD3Q19(double *f_even, double *f_odd); -// void TestRecvD3Q19(double *f_even, double *f_odd); - // Debugging and unit testing functions void PrintD3Q19(); From e64d44e43835ee268d367b82522c385e80fb1e72 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 09:30:55 -0400 Subject: [PATCH 080/270] added D3Q19 reflection BVC --- common/ScaLBL.cpp | 11 +++++++++ common/ScaLBL.h | 6 +++++ cpu/D3Q19.cpp | 36 +++++++++++++++++++++++++++++ gpu/D3Q19.cu | 59 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 3e2d0f07..8f2aacee 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1633,6 +1633,17 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl return din; } +void ScaLBL_Communicator::D3Q19_Reflection_BC_z(int *neighborList, double *fq){ + if (kproc == 0) + ScaLBL_D3Q19_AAeven_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N); + +} + +void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(int *neighborList, double *fq){ + if (kproc == nprocz-1) + ScaLBL_D3Q19_AAeven_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N); +} + void ScaLBL_Communicator::PrintD3Q19(){ printf("Printing D3Q19 communication buffer contents \n"); diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 3bf50f6f..51ee66f4 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -137,6 +137,10 @@ extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np); +extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_z(int *list, double *dist, int count, int Np); + +extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_Z(int *list, double *dist, int count, int Np); + extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); class ScaLBL_Communicator{ @@ -189,6 +193,8 @@ public: void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB); void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time); void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time); + void D3Q19_Reflection_BC_z(int *neighborList, double *fq); + void D3Q19_Reflection_BC_Z(int *neighborList, double *fq); double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time); // Debugging and unit testing functions diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 2c0e686d..2c67501c 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -448,6 +448,42 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub return dout; } +extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_z(int *list, double *dist, int count, int Np){ + for (int idx=0; idx>>(disteven, distodd, dout, Nx, Ny, Nz, outlet); -//} +extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Reflection_BC_z<<>>(neighborList, list, dist, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Reflection_BC_Z<<>>(neighborList, list, dist, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz){ From 81f25486330fbe833bbaea5bb23d932a815f8e30 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 09:34:35 -0400 Subject: [PATCH 081/270] fix a few warnings --- common/ScaLBL.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 8f2aacee..71beb152 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1520,7 +1520,7 @@ void ScaLBL_Communicator::RecvHalo(double *data){ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray ®data){ // Gets data from the device and stores in regular layout - int i,j,k,n,idx; + int i,j,k,idx; int Nx = map.size(0); int Ny = map.size(1); int Nz = map.size(2); @@ -1551,7 +1551,6 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){ - double Value=(vA-vB)/(vA+vB); if (kproc == 0) { // Set the phase indicator field and density on the z inlet ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N); @@ -1560,7 +1559,6 @@ void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double } void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){ - double Value=(vA-vB)/(vA+vB); if (kproc == nprocz-1){ // Set the phase indicator field and density on the Z outlet ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N); From e62208caaabec617c84e8b514e15b4b7f7da450e Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 09:52:23 -0400 Subject: [PATCH 082/270] add reflection BC to MRT / Color --- common/ScaLBL.cpp | 4 ++-- common/ScaLBL.h | 4 ++-- models/ColorModel.cpp | 12 ++++++++++-- models/MRTModel.cpp | 26 ++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 71beb152..a612bc73 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1631,13 +1631,13 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl return din; } -void ScaLBL_Communicator::D3Q19_Reflection_BC_z(int *neighborList, double *fq){ +void ScaLBL_Communicator::D3Q19_Reflection_BC_z(double *fq){ if (kproc == 0) ScaLBL_D3Q19_AAeven_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N); } -void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(int *neighborList, double *fq){ +void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(double *fq){ if (kproc == nprocz-1) ScaLBL_D3Q19_AAeven_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N); } diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 51ee66f4..bac60b0d 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -193,8 +193,8 @@ public: void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB); void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time); void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time); - void D3Q19_Reflection_BC_z(int *neighborList, double *fq); - void D3Q19_Reflection_BC_Z(int *neighborList, double *fq); + void D3Q19_Reflection_BC_z(double *fq); + void D3Q19_Reflection_BC_Z(double *fq); double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time); // Debugging and unit testing functions diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 954bca7e..4e9720ed 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -673,7 +673,7 @@ void ScaLBL_ColorModel::Run(){ // Perform the collision operation ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - if (BoundaryCondition > 0){ + if (BoundaryCondition > 0 && BoundaryCondition < 5){ ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } @@ -694,6 +694,10 @@ void ScaLBL_ColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); @@ -711,7 +715,7 @@ void ScaLBL_ColorModel::Run(){ // Perform the collision operation ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL // Halo exchange for phase field - if (BoundaryCondition > 0){ + if (BoundaryCondition > 0 && BoundaryCondition < 5){ ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } @@ -730,6 +734,10 @@ void ScaLBL_ColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index c1db7c1c..acfb8821 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -238,12 +238,38 @@ void ScaLBL_MRTModel::Run(){ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ From f72d401be6c88ed60ed3c71609a9c163f2a687bb Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 09:56:56 -0400 Subject: [PATCH 083/270] fix bugs in cu --- gpu/D3Q19.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gpu/D3Q19.cu b/gpu/D3Q19.cu index d43c5b29..f6f396be 100644 --- a/gpu/D3Q19.cu +++ b/gpu/D3Q19.cu @@ -1758,7 +1758,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, //................................................... } } -__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *d_neighborList, int *list, double *dist, int count, int Np){ +__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ int idx, n; idx = blockIdx.x*blockDim.x + threadIdx.x; if (idx < count){ @@ -1777,7 +1777,7 @@ __global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *d_neighborList, int *list } } -__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *d_neighborList, int *list, double *dist, int count, int Np){ +__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){ int idx, n; idx = blockIdx.x*blockDim.x + threadIdx.x; if (idx < count){ @@ -2691,7 +2691,7 @@ extern "C" double deviceReduce(double *in, double* out, int N) { extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Reflection_BC_z<<>>(neighborList, list, dist, count, N); + dvc_ScaLBL_D3Q19_Reflection_BC_z<<>>(list, dist, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err)); @@ -2700,7 +2700,7 @@ extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Reflection_BC_Z<<>>(neighborList, list, dist, count, N); + dvc_ScaLBL_D3Q19_Reflection_BC_Z<<>>(list, dist, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err)); From 10b630662a8f5016458ca71b1fc0150765aa9373 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 10:00:17 -0400 Subject: [PATCH 084/270] fix reflection name --- common/ScaLBL.cpp | 4 ++-- common/ScaLBL.h | 4 ++-- cpu/D3Q19.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index a612bc73..0fc3d6d2 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1633,13 +1633,13 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl void ScaLBL_Communicator::D3Q19_Reflection_BC_z(double *fq){ if (kproc == 0) - ScaLBL_D3Q19_AAeven_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N); + ScaLBL_D3Q19_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N); } void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(double *fq){ if (kproc == nprocz-1) - ScaLBL_D3Q19_AAeven_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N); + ScaLBL_D3Q19_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N); } void ScaLBL_Communicator::PrintD3Q19(){ diff --git a/common/ScaLBL.h b/common/ScaLBL.h index bac60b0d..11445d2a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -137,9 +137,9 @@ extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_z(int *list, double *dist, int count, int Np); +extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_Z(int *list, double *dist, int count, int Np); +extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np); extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 2c67501c..b4f7c005 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -448,7 +448,7 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub return dout; } -extern "C" void ScaLBL_D3Q19_AAeven_Reflection_BC_z(int *list, double *dist, int count, int Np){ +extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ for (int idx=0; idx Date: Fri, 3 Apr 2020 16:30:54 -0400 Subject: [PATCH 085/270] fix cudamemcpy bug --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 4e9720ed..9c46be83 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1190,7 +1190,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Vel_tmp, Velocity, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Vel_tmp, Velocity, 3*Np*sizeof(double)); //Extract averged velocity double vx_glb = (Averages->gnb.Px+Averages->gwb.Px)/(Averages->gnb.M+Averages->gwb.M); From e4d836e7fcf7b8e268d2ed2e26601d5cbbed23d8 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 20:24:29 -0400 Subject: [PATCH 086/270] add reflection condition for color grad --- common/ScaLBL.cpp | 19 ++++++++++++++----- common/ScaLBL.h | 2 ++ cpu/Color.cpp | 10 +++++++++- gpu/Color.cu | 13 +++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 0fc3d6d2..fe1ce24b 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1549,21 +1549,30 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double delete [] TmpDat; } - void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == 0) { - // Set the phase indicator field and density on the z inlet - ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N); + if (BoundaryCondition == 5){ + ScaLBL_CopySlice_z(Phi,Value,Nx,Ny,Nz,1,0); + } + else { + // Set the phase indicator field and density on the z inlet + ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N); + } //ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,0); } } void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == nprocz-1){ + if (BoundaryCondition == 5){ + ScaLBL_CopySlice_z(Phi,Value,Nx,Ny,Nz,Nz-2,Nz-1); + } + else { // Set the phase indicator field and density on the Z outlet - ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N); - //ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,Nz-1); + ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N); + } } + } void ScaLBL_Communicator::D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time){ diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 11445d2a..90209679 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -143,6 +143,8 @@ extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); +extern "C" void ScaLBL_CopySlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Source, int Destination); + class ScaLBL_Communicator{ public: //...................................................................................... diff --git a/cpu/Color.cpp b/cpu/Color.cpp index 7ae84341..1b1ce0c2 100644 --- a/cpu/Color.cpp +++ b/cpu/Color.cpp @@ -1869,7 +1869,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di const double mrt_V12=0.04166666666666666; for (int n=start; n>>(Phi,value,Nx,Ny,Nz,Slice,Dest); +} From e1e603b25f1be9598bc1d7ca710433d23a6f19d4 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 20:26:32 -0400 Subject: [PATCH 087/270] add reflection condition for color grad --- common/ScaLBL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index fe1ce24b..007a7290 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1552,7 +1552,7 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == 0) { if (BoundaryCondition == 5){ - ScaLBL_CopySlice_z(Phi,Value,Nx,Ny,Nz,1,0); + ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,1,0); } else { // Set the phase indicator field and density on the z inlet @@ -1565,7 +1565,7 @@ void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == nprocz-1){ if (BoundaryCondition == 5){ - ScaLBL_CopySlice_z(Phi,Value,Nx,Ny,Nz,Nz-2,Nz-1); + ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,Nz-2,Nz-1); } else { // Set the phase indicator field and density on the Z outlet From 735b3f5d3ea838871aa8bc69b3dcdcc53046a52c Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 20:29:37 -0400 Subject: [PATCH 088/270] fix argfs --- common/ScaLBL.h | 2 +- cpu/Color.cpp | 2 +- gpu/Color.cu | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 90209679..92956f1f 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -143,7 +143,7 @@ extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); -extern "C" void ScaLBL_CopySlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Source, int Destination); +extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination); class ScaLBL_Communicator{ public: diff --git a/cpu/Color.cpp b/cpu/Color.cpp index 1b1ce0c2..35cbd5fd 100644 --- a/cpu/Color.cpp +++ b/cpu/Color.cpp @@ -2810,7 +2810,7 @@ extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, doubl } } -extern "C" void ScaLBL_CopySlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Source, int Dest){ +extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){ int n; double value; for (n=0; n>>(Phi,value,Nx,Ny,Nz,Slice,Dest); + dvc_ScaLBL_CopySlice_z<<>>(Phi,Nx,Ny,Nz,Slice,Dest); } From 354067e2da4d8b21e7bec7902086810bbd3958d0 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 20:31:13 -0400 Subject: [PATCH 089/270] fix argfs --- gpu/Color.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpu/Color.cu b/gpu/Color.cu index 389a8dc7..7fd87e30 100644 --- a/gpu/Color.cu +++ b/gpu/Color.cu @@ -4145,7 +4145,7 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){ int GRID = Nx*Ny / 512 + 1; - dvc_ScaLBL_CopySlice_z<<>>(Phi,Nx,Ny,Nz,Slice,Dest); + dvc_ScaLBL_CopySlice_z<<>>(Phi,Nx,Ny,Nz,Source,Dest); } From bad52221a8758291dce46a9ef8f0a5d31d1905cb Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 3 Apr 2020 20:33:03 -0400 Subject: [PATCH 090/270] fix argfs --- gpu/Color.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpu/Color.cu b/gpu/Color.cu index 7fd87e30..aeeb3998 100644 --- a/gpu/Color.cu +++ b/gpu/Color.cu @@ -1241,7 +1241,7 @@ __global__ void dvc_ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny __global__ void dvc_ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){ - int n; double value; + double value; int n = blockIdx.x*blockDim.x + threadIdx.x; if (n < Nx*Ny){ value = Phi[Source*Nx*Ny+n]; From b81d4199a180c776120d5852f1f3cb63e8e594bd Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 4 Apr 2020 09:00:46 -0400 Subject: [PATCH 091/270] fix volume bug --- models/ColorModel.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 9c46be83..d8af7355 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -760,7 +760,7 @@ void ScaLBL_ColorModel::Run(){ double volA = Averages->gnb.V; volA /= Dm->Volume; volB /= Dm->Volume;; - initial_volume = volA*Dm->Volume; + //initial_volume = volA*Dm->Volume; double vA_x = Averages->gnb.Px/Averages->gnb.M; double vA_y = Averages->gnb.Py/Averages->gnb.M; double vA_z = Averages->gnb.Pz/Averages->gnb.M; @@ -1221,6 +1221,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + count += 1.0; } mass_loss += random_value*seed_water_in_oil; } @@ -1248,6 +1249,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + count += 1.0; } mass_loss += random_value*seed_water_in_oil; } From 6e065f3fe32308d4376578ac50cfe4919dc81229 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 4 Apr 2020 10:30:56 -0400 Subject: [PATCH 092/270] Revert "CPU standard MRT absperm simulator: move the body force execution from normal space to moment space" This reverts commit 7405344dac2a59513e4434497dd17dd578502f5a. --- cpu/D3Q19.cpp | 231 ++++++++++---------------------------------------- 1 file changed, 46 insertions(+), 185 deletions(-) diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 5bfe305f..2c0e686d 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -1197,35 +1197,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int //..............incorporate external force................................................ //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); -// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); -// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); -// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); -// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) - + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1236,7 +1212,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... //.................inverse transformation...................................................... @@ -1245,149 +1220,105 @@ extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; dist[1*Np+n] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; dist[2*Np+n] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; dist[3*Np+n] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; dist[4*Np+n] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; dist[5*Np+n] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; dist[6*Np+n] = fq; // q = 7 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m16-m17); + +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); dist[7*Np+n] = fq; // q = 8 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); dist[8*Np+n] = fq; // q = 9 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13+0.125*(m16+m17); + +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); dist[9*Np+n] = fq; // q = 10 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13-0.125*(m16+m17); + +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); dist[10*Np+n] = fq; // q = 11 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); dist[11*Np+n] = fq; // q = 12 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m16-m18); + -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); dist[12*Np+n] = fq; // q = 13 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); dist[13*Np+n] = fq; // q= 14 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); dist[14*Np+n] = fq; // q = 15 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); dist[15*Np+n] = fq; // q = 16 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); dist[16*Np+n] = fq; // q = 17 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); dist[17*Np+n] = fq; // q = 18 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); dist[18*Np+n] = fq; //........................................................................ @@ -1726,36 +1657,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star //..............incorporate external force................................................ //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); -// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); -// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); -// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); -// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - - //Body force is executed in the moment space - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) - + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1766,7 +1672,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... //.................inverse transformation...................................................... @@ -1775,164 +1680,120 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int star dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; nread = neighborList[n+Np]; dist[nread] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; nread = neighborList[n]; dist[nread] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; nread = neighborList[n+3*Np]; dist[nread] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; nread = neighborList[n+2*Np]; dist[nread] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; nread = neighborList[n+5*Np]; dist[nread] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; nread = neighborList[n+4*Np]; dist[nread] = fq; // q = 7 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m16-m17); + +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13+0.125*(m16+m17); + +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) -// +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12-0.25*m13-0.125*(m16+m17); + +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m16-m18); + -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); nread = neighborList[n+16*Np]; dist[nread] = fq; From fcbacd76de942f68766a2277816c8fc13bb504e5 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 4 Apr 2020 10:31:07 -0400 Subject: [PATCH 093/270] Revert "GPU standard MRT absperm simulator: move the body force execution from normal space to moment space" This reverts commit fa4a20ddf0c8e156d40dae2e3818a54ee54186de. --- gpu/D3Q19.cu | 231 +++++++++++---------------------------------------- 1 file changed, 50 insertions(+), 181 deletions(-) diff --git a/gpu/D3Q19.cu b/gpu/D3Q19.cu index 62212a17..ccd125b2 100644 --- a/gpu/D3Q19.cu +++ b/gpu/D3Q19.cu @@ -786,36 +786,11 @@ dvc_ScaLBL_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int //..............incorporate external force................................................ //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); -// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); -// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); -// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); -// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - - //Body force is executed in the moment space - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) - + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -834,157 +809,117 @@ dvc_ScaLBL_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; nread = neighborList[n+Np]; dist[nread] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; nread = neighborList[n]; dist[nread] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; nread = neighborList[n+3*Np]; dist[nread] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; nread = neighborList[n+2*Np]; dist[nread] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; nread = neighborList[n+5*Np]; dist[nread] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; nread = neighborList[n+4*Np]; dist[nread] = fq; // q = 7 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+ -// mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+ -// mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+ -// mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+ - mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ -// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ - mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); nread = neighborList[n+16*Np]; dist[nread] = fq; @@ -1287,36 +1222,11 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se //..............incorporate external force................................................ //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); -// m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); -// m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); -// m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); -// m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); -// m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); -// m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); -// m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); -// m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); -// m13 = m13 + rlx_setA*((jx*jy/rho) - m13); -// m14 = m14 + rlx_setA*((jy*jz/rho) - m14); -// m15 = m15 + rlx_setA*((jx*jz/rho) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); - - //Body force is executed in the moment space - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1) - + (1-0.5*rlx_setA)*38*(Fx*jx+Fy*jy+Fz*jz)/rho; - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2) - + (1-0.5*rlx_setA)*11*(-Fx*jx-Fy*jy-Fz*jz)/rho; - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); @@ -1327,7 +1237,6 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... //.................inverse transformation...................................................... @@ -1336,145 +1245,105 @@ dvc_ScaLBL_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_se dist[n] = fq; // q = 1 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx; dist[1*Np+n] = fq; // q=2 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; dist[2*Np+n] = fq; // q = 3 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; dist[3*Np+n] = fq; // q = 4 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; dist[4*Np+n] = fq; // q = 5 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; dist[5*Np+n] = fq; // q = 6 - //fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; dist[6*Np+n] = fq; // q = 7 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ -// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + -// 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + + 0.08333333333*(Fx+Fy); dist[7*Np+n] = fq; // q = 8 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 -// +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 - +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); dist[8*Np+n] = fq; // q = 9 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ -// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17)+ -// 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17)+ + 0.08333333333*(Fx-Fy); dist[9*Np+n] = fq; // q = 10 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ -// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- -// 0.08333333333*(Fx-Fy); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ - mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- + 0.08333333333*(Fx-Fy); dist[10*Np+n] = fq; // q = 11 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); dist[11*Np+n] = fq; // q = 12 -// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ -// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18)- -// 0.08333333333*(Fx+Fz); fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ - mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18)- + 0.08333333333*(Fx+Fz); dist[12*Np+n] = fq; // q = 13 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); dist[13*Np+n] = fq; // q= 14 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) -// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 -// -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 - -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); dist[14*Np+n] = fq; // q = 15 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); dist[15*Np+n] = fq; // q = 16 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) -// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) - -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); dist[16*Np+n] = fq; // q = 17 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) - -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); dist[17*Np+n] = fq; // q = 18 -// fq = mrt_V1*rho+mrt_V9*m1 -// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) -// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); fq = mrt_V1*rho+mrt_V9*m1 +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) - -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); dist[18*Np+n] = fq; //........................................................................ } From 5b45c3216c9f8653fc99616770bd1dde063dea4e Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 4 Apr 2020 10:34:50 -0400 Subject: [PATCH 094/270] initialize multicomponent greyscale model --- models/GreyscaleColorModel.cpp | 898 +++++++++++++++++++++++++++++++++ models/GreyscaleColorModel.h | 92 ++++ 2 files changed, 990 insertions(+) create mode 100644 models/GreyscaleColorModel.cpp create mode 100644 models/GreyscaleColorModel.h diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp new file mode 100644 index 00000000..11d92c80 --- /dev/null +++ b/models/GreyscaleColorModel.cpp @@ -0,0 +1,898 @@ +/* +Greyscale lattice boltzmann model + */ +#include "models/GreyscaleModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include +#include + +template +void DeleteArray( const TYPE *p ) +{ + delete [] p; +} + +ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + SignDist.resize(Nx,Ny,Nz); + SignDist.fill(0); + +} +ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ + +} + +void ScaLBL_GreyscaleModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + greyscale_db = db->getDatabase( "Greyscale" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tau = 1.0; + tau_eff = tau; + Den = 1.0;//constant density + tolerance = 0.01; + Fx = Fy = Fz = 0.0; + Restart=false; + din=dout=1.0; + flux=0.0; + dp = 10.0; //unit of 'dp': voxel + CollisionType = 1; //1: IMRT; 2: BGK + + // ---------------------- Greyscale Model parameters -----------------------// + if (greyscale_db->keyExists( "timestepMax" )){ + timestepMax = greyscale_db->getScalar( "timestepMax" ); + } + if (greyscale_db->keyExists( "tau" )){ + tau = greyscale_db->getScalar( "tau" ); + } + tau_eff = greyscale_db->getWithDefault( "tau_eff", tau ); + if (greyscale_db->keyExists( "Den" )){ + Den = greyscale_db->getScalar( "Den" ); + } + if (greyscale_db->keyExists( "dp" )){ + dp = greyscale_db->getScalar( "dp" ); + } + if (greyscale_db->keyExists( "F" )){ + Fx = greyscale_db->getVector( "F" )[0]; + Fy = greyscale_db->getVector( "F" )[1]; + Fz = greyscale_db->getVector( "F" )[2]; + } + if (greyscale_db->keyExists( "Restart" )){ + Restart = greyscale_db->getScalar( "Restart" ); + } + if (greyscale_db->keyExists( "din" )){ + din = greyscale_db->getScalar( "din" ); + } + if (greyscale_db->keyExists( "dout" )){ + dout = greyscale_db->getScalar( "dout" ); + } + if (greyscale_db->keyExists( "flux" )){ + flux = greyscale_db->getScalar( "flux" ); + } + if (greyscale_db->keyExists( "tolerance" )){ + tolerance = greyscale_db->getScalar( "tolerance" ); + } + auto collision = greyscale_db->getWithDefault( "collision", "IMRT" ); + if (collision == "BGK"){ + CollisionType=2; + } + // ------------------------------------------------------------------------// + + //------------------------ Other Domain parameters ------------------------// + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + // ------------------------------------------------------------------------// +} + +void ScaLBL_GreyscaleModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + + SignDist.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + PorosityMap.resize(Nx,Ny,Nz); + Pressure.resize(Nx,Ny,Nz); + + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_GreyscaleModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + int count = 0; + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kgetVector( "ComponentLabels" ); + auto PorosityList = greyscale_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + + NLABELS=LabelList.size(); + if (NLABELS != PorosityList.size()){ + ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Porosity[idx] = POROSITY; + } + } + } + } + } + + if (NLABELS != PermeabilityList.size()){ + ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); + } + for (int k=1;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + } + } + } + } + } + + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm.sumReduce(label_count[idx]); + + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); + } +} + + +void ScaLBL_GreyscaleModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + //......................................................... + // don't perform computations at the eight corners + //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + int n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n"); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + int n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n"); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; + + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + // initialize phi based on PhaseLabel (include solid component labels) + double *Poros, *Perm; + Poros = new double[Np]; + Perm = new double[Np]; + AssignComponentLabels(Poros,Perm); + ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); +} + + +void ScaLBL_GreyscaleModel::Initialize(){ + if (rank==0) printf ("Initializing distributions \n"); + //TODO: for BGK, you need to consider voxel porosity + // for IMRT, the whole set of feq is different + // if in the future you have different collison mode, need to write two set of initialization functions + if (CollisionType==1){ + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + if (rank==0) printf("Collision model: Incompressible MRT.\n"); + } + else if (CollisionType==2){ + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf("Collision model: BGK.\n"); + } + else{ + if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + CollisionType=1; + greyscale_db->putScalar( "collision", "IMRT" ); + } + + if (Restart == true){ + if (rank==0){ + printf("Initializing distributions from Restart! \n"); + } + // Read in the restart file to CPU buffers + std::shared_ptr cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + FILE *File; + File=fopen(LocalRestartFile,"rb"); + fread(cfq.get(),sizeof(double),19*Np,File); + fclose(File); + + // Copy the restart data to the GPU + ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + + MPI_Barrier(comm); + } +} + +void ScaLBL_GreyscaleModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int visualization_interval = 1000; + int restart_interval = 10000; // number of timesteps in between in saving distributions for restart + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "visualization_interval" )){ + visualization_interval = analysis_db->getScalar( "visualization_interval" ); + } + if (analysis_db->keyExists( "restart_interval" )){ + restart_interval = analysis_db->getScalar( "restart_interval" ); + } + if (greyscale_db->keyExists( "timestep" )){ + timestep = greyscale_db->getScalar( "timestep" ); + } + + if (rank==0){ + printf("********************************************************\n"); + printf("No. of timesteps: %i \n", timestepMax); + fflush(stdout); + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); + //......................................... + + Minkowski Morphology(Mask); + + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + auto current_db = db->cloneDatabase(); + double rlx = 1.0/tau; + double rlx_eff = 1.0/tau_eff; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + // *************ODD TIMESTEP*************// + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + + if (timestep%analysis_interval==0){ + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); + //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + //double px_loc,py_loc,pz_loc; + //double px,py,pz; + //double mass_loc,mass_glb; + + //parameters for domain average + int64_t i,j,k,n,imin,jmin,kmin,kmax; + // If external boundary conditions are set, do not average over the inlet and outlet + kmin=1; kmax=Nz-1; + //In case user forgets to specify the inlet/outlet buffer layers for BC>0 + if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; + if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; + + imin=jmin=1; + // If inlet/outlet layers exist use these as default + //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; + //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; + if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer + if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; + +// px_loc = py_loc = pz_loc = 0.f; +// mass_loc = 0.f; +// for (int k=kmin; k 0){ +// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +// mass_loc += Den*PorosityMap(i,j,k); +// } +// } +// } +// } +// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// +// vax = px/mass_glb; +// vay = py/mass_glb; +// vaz = pz/mass_glb; + + vax_loc = vay_loc = vaz_loc = 0.f; + for (int k=kmin; k 0){ + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; + } + } + } + } + //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax = Mask->Comm.sumReduce( vax_loc ); + vay = Mask->Comm.sumReduce( vay_loc ); + vaz = Mask->Comm.sumReduce( vaz_loc ); + count = Mask->Comm.sumReduce( count_loc ); + + vax /= count; + vay /= count; + vaz /= count; + + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + + error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); + flow_rate_previous = flow_rate; + + //if (rank==0) printf("Computing Minkowski functionals \n"); + Morphology.ComputeScalar(SignDist,0.f); + //Morphology.PrintAll(); + double mu = (tau-0.5)/3.f; + double Vs = Morphology.V(); + double As = Morphology.A(); + double Hs = Morphology.H(); + double Xs = Morphology.X(); + Vs = Dm->Comm.sumReduce( Vs); + As = Dm->Comm.sumReduce( As); + Hs = Dm->Comm.sumReduce( Hs); + Xs = Dm->Comm.sumReduce( Xs); + + double h = Dm->voxel_length; + //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; + + if (rank==0){ + printf(" AbsPerm = %.5g [micron^2]\n",absperm); + bool WriteHeader=false; + FILE * log_file = fopen("Permeability.csv","r"); + if (log_file != NULL) + fclose(log_file); + else + WriteHeader=true; + log_file = fopen("Permeability.csv","a"); + if (WriteHeader) + fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", + timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); + + fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, + h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); + fclose(log_file); + } + } + + if (timestep%visualization_interval==0){ + VelocityField(); + } + + if (timestep%restart_interval==0){ + //Use rank=0 write out Restart.db + if (rank==0) { + greyscale_db->putScalar("timestep",timestep); + greyscale_db->putScalar( "Restart", true ); + current_db->putDatabase("Greyscale", greyscale_db); + std::ofstream OutStream("Restart.db"); + current_db->print(OutStream, ""); + OutStream.close(); + + } + //Write out Restart data. + std::shared_ptr cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU + + FILE *RESTARTFILE; + RESTARTFILE=fopen(LocalRestartFile,"wb"); + fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); + fclose(RESTARTFILE); + MPI_Barrier(comm); + } + } + + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + +void ScaLBL_GreyscaleModel::VelocityField(){ + +/* Minkowski Morphology(Mask); + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); + + memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); + Morphology.Initialize(); + Morphology.UpdateMeshValues(); + Morphology.ComputeLocal(); + Morphology.Reduce(); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int n=0; nLastExterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + + for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double mu = (tau-0.5)/3.f; + if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); + if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, + Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); + */ + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + auto PressureVar = std::make_shared(); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + PressureVar->name = "Pressure"; + PressureVar->type = IO::VariableType::VolumeVariable; + PressureVar->dim = 1; + PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PressureVar); + + Array& SignData = visData[0].vars[0]->data; + Array& VelxData = visData[0].vars[1]->data; + Array& VelyData = visData[0].vars[2]->data; + Array& VelzData = visData[0].vars[3]->data; + Array& PressureData = visData[0].vars[4]->data; + + ASSERT(visData[0].vars[0]->name=="SignDist"); + ASSERT(visData[0].vars[1]->name=="Velocity_x"); + ASSERT(visData[0].vars[2]->name=="Velocity_y"); + ASSERT(visData[0].vars[3]->name=="Velocity_z"); + ASSERT(visData[0].vars[4]->name=="Pressure"); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + + fillData.copy(SignDist,SignData); + fillData.copy(Velocity_x,VelxData); + fillData.copy(Velocity_y,VelyData); + fillData.copy(Velocity_z,VelzData); + fillData.copy(Pressure,PressureData); + + IO::writeData( timestep, visData, Dm->Comm ); + +} + +void ScaLBL_GreyscaleModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + + //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); + +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); +// +// ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); +// FILE *PFILE; +// sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); +// PFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PFILE); +// fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); + FILE *POROS_FILE; + sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); + POROS_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,POROS_FILE); + fclose(POROS_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); + FILE *PERM_FILE; + sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); + PERM_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PERM_FILE); + fclose(PERM_FILE); +} diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h new file mode 100644 index 00000000..a99925b1 --- /dev/null +++ b/models/GreyscaleColorModel.h @@ -0,0 +1,92 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "common/MPI.h" +#include "common/Database.h" +#include "common/ScaLBL.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_GreyscaleModel{ +public: + ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + void VelocityField(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + int CollisionType; + double tau; + double tau_eff; + double Den;//constant density + double tolerance; + double Fx,Fy,Fz,flux; + double din,dout; + double dp;//solid particle diameter, unit in voxel + double GreyPorosity; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr greyscale_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + signed char *id; + int *NeighborList; + int *dvcMap; + double *fq; + double *Permeability;//grey voxel permeability + double *Porosity; + double *Velocity; + double *Pressure_dvc; + IntArray Map; + DoubleArray SignDist; + DoubleArray Velocity_x; + DoubleArray Velocity_y; + DoubleArray Velocity_z; + DoubleArray PorosityMap; + DoubleArray Pressure; + +private: + MPI_Comm comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + void AssignComponentLabels(double *Porosity, double *Permeablity); + +}; + From 6d4eaebf4799ce94385dfb4a2186de0534403f4a Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 6 Apr 2020 06:07:46 -0400 Subject: [PATCH 095/270] drop velocity seeding alg --- models/ColorModel.cpp | 245 ++++++++++++------------------------------ 1 file changed, 67 insertions(+), 178 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index d8af7355..21a1f597 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -56,8 +56,6 @@ void ScaLBL_ColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq File.close(); } */ - - void ScaLBL_ColorModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); @@ -1176,190 +1174,81 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ } return(volume_change); } - double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ - srand(time(NULL)); - double mass_loss =0.f; - double count =0.f; - double *Aq_tmp, *Bq_tmp; - double *Vel_tmp; - - Aq_tmp = new double [7*Np]; - Bq_tmp = new double [7*Np]; - Vel_tmp = new double [3*Np]; + srand(time(NULL)); + double mass_loss =0.f; + double count =0.f; + double *Aq_tmp, *Bq_tmp; + + Aq_tmp = new double [7*Np]; + Bq_tmp = new double [7*Np]; - ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Vel_tmp, Velocity, 3*Np*sizeof(double)); - - //Extract averged velocity - double vx_glb = (Averages->gnb.Px+Averages->gwb.Px)/(Averages->gnb.M+Averages->gwb.M); - double vy_glb = (Averages->gnb.Py+Averages->gwb.Py)/(Averages->gnb.M+Averages->gwb.M); - double vz_glb = (Averages->gnb.Pz+Averages->gwb.Pz)/(Averages->gnb.M+Averages->gwb.M); - double v_mag_glb = sqrt(vx_glb*vx_glb+vy_glb*vy_glb+vz_glb*vz_glb); + ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); + + + for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } - for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ - double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); - double weight = (v_mag_local 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - count += 1.0; - } - mass_loss += random_value*seed_water_in_oil; - } + for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } - for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ - double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); - double weight = (v_mag_local 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - count += 1.0; - } - mass_loss += random_value*seed_water_in_oil; - } + count= sumReduce( Dm->Comm, count); + mass_loss= sumReduce( Dm->Comm, mass_loss); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); - count= sumReduce( Dm->Comm, count); - mass_loss= sumReduce( Dm->Comm, mass_loss); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + // Need to initialize Aq, Bq, Den, Phi directly + //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); - // Need to initialize Aq, Bq, Den, Phi directly - //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); - - return(mass_loss); + return(mass_loss); } -//double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ -// srand(time(NULL)); -// double mass_loss =0.f; -// double count =0.f; -// double *Aq_tmp, *Bq_tmp; -// -// Aq_tmp = new double [7*Np]; -// Bq_tmp = new double [7*Np]; -// -// ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); -// ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); -// -///* for (int k=1; kSDs(i,j,k) < 0.f){ -// // skip -// } -// else if (phase(i,j,k) > 0.f ){ -// phase(i,j,k) -= random_value*seed_water_in_oil; -// mass_loss += random_value*seed_water_in_oil; -// count++; -// } -// else { -// -// } -// } -// } -// } -// */ -// for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ -// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; -// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; -// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; -// double phase_id = (dA - dB) / (dA + dB); -// if (phase_id > 0.0){ -// Aq_tmp[n] -= 0.3333333333333333*random_value; -// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; -// -// Bq_tmp[n] += 0.3333333333333333*random_value; -// Bq_tmp[n+Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; -// } -// mass_loss += random_value*seed_water_in_oil; -// } -// -// for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ -// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; -// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; -// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; -// double phase_id = (dA - dB) / (dA + dB); -// if (phase_id > 0.0){ -// Aq_tmp[n] -= 0.3333333333333333*random_value; -// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; -// -// Bq_tmp[n] += 0.3333333333333333*random_value; -// Bq_tmp[n+Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; -// } -// mass_loss += random_value*seed_water_in_oil; -// } -// -// count = Dm->Comm.sumReduce( count ); -// mass_loss = Dm->Comm.sumReduce( mass_loss ); -// if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); -// -// // Need to initialize Aq, Bq, Den, Phi directly -// //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); -// -// return(mass_loss); -//} - double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); From 91f42ab74f14a44883d2d68e59750f93da1f7018 Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 08:45:06 -0400 Subject: [PATCH 096/270] condition unpack routines on BC for halo --- common/ScaLBL.cpp | 42 ++++++++++++++++++++------- example/Workflow/ComputeSaturation.py | 37 +++++++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) create mode 100755 example/Workflow/ComputeSaturation.py diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 007a7290..12589ecf 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1497,22 +1497,44 @@ void ScaLBL_Communicator::RecvHalo(double *data){ //................................................................................... ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N); - ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); + + if (BoundaryCondition > 0){ + if (kproc != 0){ + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N); + } + if (kproc != nprocz-1){ + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); + } + } + else { + ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); + ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); + } + //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... diff --git a/example/Workflow/ComputeSaturation.py b/example/Workflow/ComputeSaturation.py new file mode 100755 index 00000000..56a34ece --- /dev/null +++ b/example/Workflow/ComputeSaturation.py @@ -0,0 +1,37 @@ +import sys +import numpy as np +import matplotlib.pylab as plt + +FILENAME=sys.argv[1] +Nx=int(sys.argv[2]) +Ny=int(sys.argv[3]) +Nz=int(sys.argv[4]) + +# read the input image +Output = np.fromfile(FILENAME,dtype = np.uint8) +Output.shape = (Nz,Ny,Nx) + +Oil=np.count_nonzero(Output==1) +Water=np.count_nonzero(Output==2) +Sw=Water/(Oil+Water) + +Porosity=1.0-(Oil+Water)/(Nx*Ny*Nz) + +print(FILENAME,"Porosity=", Porosity) + +SaturationProfile=np.zeros(Nz) +PorosityProfile=np.zeros(Nz) +# Compute saturation slice by slice +for idx in range(0, Nz): + Slice = Output[idx,:,:] + Oil=np.count_nonzero(Slice==1) + Water=np.count_nonzero(Slice==2) + SaturationProfile[idx]=Water/(Oil+Water) + PorosityProfile[idx]=(Oil+Water)/(Nx*Ny) + + +plt.figure() +plt.plot(SaturationProfile) +plt.xlabel('Position (z)') +plt.ylabel('Water Saturation') +plt.show() From a82c8995fe64cc9c34e39e5400a53c3e2b687a3e Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 09:27:32 -0400 Subject: [PATCH 097/270] make sure not to remove solid for reflection BC --- common/Domain.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 03d0c5ca..33d6117a 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -592,10 +592,10 @@ void Domain::Decomp( const std::string& Filename ) double sum; double sum_local=0.0; double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && kproc() == 0){ + if (BoundaryCondition > 0 && BoundaryCondition !=5 && kproc() == 0){ if (inlet_layers_z < 4){ inlet_layers_z=4; if(RANK==0){ @@ -611,7 +611,7 @@ void Domain::Decomp( const std::string& Filename ) } } } - if (BoundaryCondition > 0 && kproc() == nprocz-1){ + if (BoundaryCondition > 0 && BoundaryCondition !=5 && kproc() == nprocz-1){ if (outlet_layers_z < 4){ outlet_layers_z=4; if(RANK==nprocs-1){ @@ -1061,10 +1061,10 @@ void Domain::ReadIDs(){ double sum; double sum_local=0.0; double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6)); + if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6)); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && kproc() == 0){ + if (BoundaryCondition > 0 && BoundaryCondition !=5 && kproc() == 0){ if (inlet_layers_z < 4) inlet_layers_z=4; for (int k=0; k 0 && kproc() == nprocz()-1){ + if (BoundaryCondition > 0 && BoundaryCondition !=5 && kproc() == nprocz()-1){ if (outlet_layers_z < 4) outlet_layers_z=4; for (int k=Nz-outlet_layers_z; k Date: Tue, 7 Apr 2020 09:58:32 -0400 Subject: [PATCH 098/270] disable periodic BC override --- models/ColorModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 21a1f597..def4ab4e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -128,21 +128,21 @@ void ScaLBL_ColorModel::ReadParams(string filename){ // Override user-specified boundary condition for specific protocols auto protocol = color_db->getWithDefault( "protocol", "none" ); if (protocol == "seed water"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n"); } domain_db->putScalar( "BC", BoundaryCondition ); } else if (protocol == "open connected oil"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n"); } domain_db->putScalar( "BC", BoundaryCondition ); } else if (protocol == "shell aggregation"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n"); } From af8b2d799aaa27859f257ae75e52651325687814 Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 10:06:54 -0400 Subject: [PATCH 099/270] enable target Ca for reflection BC --- models/ColorModel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index def4ab4e..83ddc5d4 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -567,8 +567,8 @@ void ScaLBL_ColorModel::Run(){ if (color_db->keyExists( "timestep" )){ timestep = color_db->getScalar( "timestep" ); } - if (BoundaryCondition != 0 && SET_CAPILLARY_NUMBER==true){ - if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 \n"); + if (BoundaryCondition != 0 && BoundaryCondition != 5 && SET_CAPILLARY_NUMBER==true){ + if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 or 5 \n"); SET_CAPILLARY_NUMBER=false; } if (analysis_db->keyExists( "seed_water" )){ From 3b006fbc3c8aa7ed81b0e34b6f0a67687e58881e Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 10:38:21 -0400 Subject: [PATCH 100/270] reflect BC for D3Q7 --- common/ScaLBL.cpp | 11 ++++++++++- common/ScaLBL.h | 4 ++++ cpu/D3Q7.cpp | 15 +++++++++++++++ gpu/D3Q7.cu | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 12589ecf..ef9b2341 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1286,7 +1286,16 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); } - + if (BoundaryCondition == 5){ + if (kproc == 0){ + ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Aq, sendCount_z, N); + ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Bq, sendCount_z, N); + } + if (kproc == nprocz-1){ + ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Aq, sendCount_Z, N); + ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Bq, sendCount_Z, N); + } + } //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 92956f1f..dec8b3d1 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -141,6 +141,10 @@ extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np); +extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np); + +extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np); + extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination); diff --git a/cpu/D3Q7.cpp b/cpu/D3Q7.cpp index 344e6851..0940b3b6 100644 --- a/cpu/D3Q7.cpp +++ b/cpu/D3Q7.cpp @@ -72,6 +72,21 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int } } +extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){ + for (int idx=0; idx>>(list, dist, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_Reflection_BC_Z<<>>(list, dist, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} + extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N){ int GRID = count / 512 + 1; dvc_ScaLBL_D3Q7_Unpack <<>>(q, list, start, count, recvbuf, dist, N); From 7636220a4894a1a7c485e289e928408a50efd54e Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 10:43:01 -0400 Subject: [PATCH 101/270] add header for print --- gpu/D3Q7.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/gpu/D3Q7.cu b/gpu/D3Q7.cu index c10a865b..8a551f78 100644 --- a/gpu/D3Q7.cu +++ b/gpu/D3Q7.cu @@ -1,4 +1,5 @@ // GPU Functions for D3Q7 Lattice Boltzmann Methods +#include #define NBLOCKS 560 #define NTHREADS 128 From cfa40bdcba7fb109136dfd619da098ff8dd8596d Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 7 Apr 2020 13:57:29 -0400 Subject: [PATCH 102/270] fix declare --- cpu/D3Q7.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpu/D3Q7.cpp b/cpu/D3Q7.cpp index 0940b3b6..48f71495 100644 --- a/cpu/D3Q7.cpp +++ b/cpu/D3Q7.cpp @@ -73,6 +73,7 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int } extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){ + int n; for (int idx=0; idx Date: Tue, 7 Apr 2020 15:29:33 -0400 Subject: [PATCH 103/270] GPU version ONLY; two-phase greyscale model; save the work --- common/ScaLBL.h | 21 + cpu/D3Q19.cpp | 27 - cpu/Greyscale.cpp | 27 + gpu/D3Q19.cu | 38 - gpu/Greyscale.cu | 38 + gpu/GreyscaleColor.cu | 1590 +++++++++++++++++++++++ models/GreyscaleColorModel.cpp | 864 +++++++----- models/GreyscaleColorModel.h | 31 +- models/GreyscaleModel.cpp | 33 +- models/GreyscaleModel.h | 1 - tests/CMakeLists.txt | 1 + tests/lbpm_greyscaleColor_simulator.cpp | 59 + 12 files changed, 2282 insertions(+), 448 deletions(-) create mode 100644 gpu/GreyscaleColor.cu create mode 100644 tests/lbpm_greyscaleColor_simulator.cpp diff --git a/common/ScaLBL.h b/common/ScaLBL.h index d3833bcb..df87fcc4 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -72,6 +72,27 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); +// GREYSCALE COLOR MODEL + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure); + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure); + +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, diff --git a/cpu/D3Q19.cpp b/cpu/D3Q19.cpp index 2c0e686d..564eb96d 100644 --- a/cpu/D3Q19.cpp +++ b/cpu/D3Q19.cpp @@ -84,33 +84,6 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np) } } - -extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den) -{ - int n; - for (n=0; n>>(dist, Np, Den); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); - } -} extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ dvc_ScaLBL_D3Q19_Swap<<>>(ID, disteven, distodd, Nx, Ny, Nz); diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 0a9a63e0..a3fba989 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -1590,6 +1590,37 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double } } +__global__ void dvc_ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den) +{ + int n; + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist, Np, Den); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); + } +} diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu new file mode 100644 index 00000000..c9586d3c --- /dev/null +++ b/gpu/GreyscaleColor.cu @@ -0,0 +1,1590 @@ +#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double nx,ny,nz,C; + double nA,nB; + double a1,b1,a2,b2,nAB,delta; + double beta=0.95; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; + double Gff_x,Gff_y,Gff_z; + double Gfs_x,Gfs_y,Gfs_z; + + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s0.0)-Gsc*nB*nA_gradx*int(phi<0.0); + Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); + Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + dist[1*Np+n] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + dist[2*Np+n] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[3*Np+n] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[4*Np+n] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[5*Np+n] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[6*Np+n] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + dist[7*Np+n] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + dist[8*Np+n] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + dist[9*Np+n] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + dist[10*Np+n] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + dist[11*Np+n] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + dist[12*Np+n] = fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + dist[13*Np+n] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + dist[14*Np+n] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + dist[15*Np+n] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + dist[16*Np+n] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + dist[17*Np+n] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + dist[18*Np+n] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // Calculate the color gradient + nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); + ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); + nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + double ColorMag = C; + if (C==0.0) ColorMag=1.0; + nx = nx/ColorMag; + ny = ny/ColorMag; + nz = nz/ColorMag; + if (C == 0.0) nx = ny = nz = 0.0; + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double nx,ny,nz,C; + double nA,nB; + double a1,b1,a2,b2,nAB,delta; + double beta=0.95; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; + double Gff_x,Gff_y,Gff_z; + double Gfs_x,Gfs_y,Gfs_z; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + fq = dist[nr1]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// + // fluid-fluid force + Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); + Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); + Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // Calculate the color gradient + nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); + ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); + nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + double ColorMag = C; + if (C==0.0) ColorMag=1.0; + nx = nx/ColorMag; + ny = ny/ColorMag; + nz = nz/ColorMag; + if (C == 0.0) nx = ny = nz = 0.0; + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ + int n; + int S = Np/NBLOCKS/NTHREADS + 1; + double phi; + double nA,nB; + double Den0; + for (int s=0; s>>(dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColor: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColor: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ + dvc_ScaLBL_D3Q19_GreyColorIMRT_Init<<>>(dist,Den,rhoA,rhoB,Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity<<>>(NeighborList, Aq, Bq, Den, start, finish, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleColorDensity: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity<<>>(Aq, Bq, Den, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorDensity: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient<<>>(neighborList, Den, DenGrad, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Gradient: %s \n",cudaGetErrorString(err)); + } +} + diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 11d92c80..421805c5 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -1,7 +1,7 @@ /* Greyscale lattice boltzmann model */ -#include "models/GreyscaleModel.h" +#include "models/GreyscaleColorModel.h" #include "analysis/distance.h" #include "analysis/morphology.h" #include @@ -13,77 +13,90 @@ void DeleteArray( const TYPE *p ) delete [] p; } -ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),Gsc(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); SignDist.fill(0); } -ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ +ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ } -void ScaLBL_GreyscaleModel::ReadParams(string filename){ +void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); - greyscale_db = db->getDatabase( "Greyscale" ); + greyscaleColor_db = db->getDatabase( "GreyscaleColor" ); analysis_db = db->getDatabase( "Analysis" ); vis_db = db->getDatabase( "Visualization" ); // set defaults timestepMax = 100000; - tau = 1.0; - tau_eff = tau; - Den = 1.0;//constant density + tauA = 1.0; + tauB = 1.0; + tauA_eff = tauA;//the effective viscosity of the Darcy term + tauB_eff = tauB; + rhoA = 1.0;//constant molecular mass (after LB scaling) + rhoB = 1.0; tolerance = 0.01; Fx = Fy = Fz = 0.0; Restart=false; din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel - CollisionType = 1; //1: IMRT; 2: BGK // ---------------------- Greyscale Model parameters -----------------------// - if (greyscale_db->keyExists( "timestepMax" )){ - timestepMax = greyscale_db->getScalar( "timestepMax" ); + if (greyscaleColor_db->keyExists( "timestepMax" )){ + timestepMax = greyscaleColor_db->getScalar( "timestepMax" ); } - if (greyscale_db->keyExists( "tau" )){ - tau = greyscale_db->getScalar( "tau" ); + if (greyscaleColor_db->keyExists( "tauA" )){ + tauA = greyscaleColor_db->getScalar( "tauA" ); } - tau_eff = greyscale_db->getWithDefault( "tau_eff", tau ); - if (greyscale_db->keyExists( "Den" )){ - Den = greyscale_db->getScalar( "Den" ); + if (greyscaleColor_db->keyExists( "tauB" )){ + tauB = greyscaleColor_db->getScalar( "tauB" ); } - if (greyscale_db->keyExists( "dp" )){ - dp = greyscale_db->getScalar( "dp" ); + tauA_eff = greyscaleColor_db->getWithDefault( "tauA_eff", tauA); + tauB_eff = greyscaleColor_db->getWithDefault( "tauB_eff", tauB); + if (greyscaleColor_db->keyExists( "rhoA" )){ + rhoA = greyscaleColor_db->getScalar( "rhoA" ); } - if (greyscale_db->keyExists( "F" )){ - Fx = greyscale_db->getVector( "F" )[0]; - Fy = greyscale_db->getVector( "F" )[1]; - Fz = greyscale_db->getVector( "F" )[2]; + if (greyscaleColor_db->keyExists( "rhoB" )){ + rhoB = greyscaleColor_db->getScalar( "rhoB" ); } - if (greyscale_db->keyExists( "Restart" )){ - Restart = greyscale_db->getScalar( "Restart" ); + if (greyscaleColor_db->keyExists( "Gsc" )){ + Gsc = greyscaleColor_db->getScalar( "Gsc" ); } - if (greyscale_db->keyExists( "din" )){ - din = greyscale_db->getScalar( "din" ); + if (greyscaleColor_db->keyExists( "dp" )){ + dp = greyscaleColor_db->getScalar( "dp" ); } - if (greyscale_db->keyExists( "dout" )){ - dout = greyscale_db->getScalar( "dout" ); + if (greyscaleColor_db->keyExists( "F" )){ + Fx = greyscaleColor_db->getVector( "F" )[0]; + Fy = greyscaleColor_db->getVector( "F" )[1]; + Fz = greyscaleColor_db->getVector( "F" )[2]; } - if (greyscale_db->keyExists( "flux" )){ - flux = greyscale_db->getScalar( "flux" ); + if (greyscaleColor_db->keyExists( "Restart" )){ + Restart = greyscaleColor_db->getScalar( "Restart" ); } - if (greyscale_db->keyExists( "tolerance" )){ - tolerance = greyscale_db->getScalar( "tolerance" ); + if (greyscaleColor_db->keyExists( "din" )){ + din = greyscaleColor_db->getScalar( "din" ); } - auto collision = greyscale_db->getWithDefault( "collision", "IMRT" ); - if (collision == "BGK"){ - CollisionType=2; + if (greyscaleColor_db->keyExists( "dout" )){ + dout = greyscaleColor_db->getScalar( "dout" ); } + if (greyscaleColor_db->keyExists( "flux" )){ + flux = greyscaleColor_db->getScalar( "flux" ); + } + if (greyscaleColor_db->keyExists( "tolerance" )){ + tolerance = greyscaleColor_db->getScalar( "tolerance" ); + } + //auto collision = greyscaleColor_db->getWithDefault( "collision", "IMRT" ); + //if (collision == "BGK"){ + // CollisionType=2; + //} // ------------------------------------------------------------------------// //------------------------ Other Domain parameters ------------------------// @@ -94,7 +107,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ // ------------------------------------------------------------------------// } -void ScaLBL_GreyscaleModel::SetDomain(){ +void ScaLBL_GreyscaleColorModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases // domain parameters @@ -125,7 +138,7 @@ void ScaLBL_GreyscaleModel::SetDomain(){ nprocz = Dm->nprocz(); } -void ScaLBL_GreyscaleModel::ReadInput(){ +void ScaLBL_GreyscaleColorModel::ReadInput(){ sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); @@ -174,44 +187,151 @@ void ScaLBL_GreyscaleModel::ReadInput(){ if (rank == 0) cout << "Domain set." << endl; } -/******************************************************** - * AssignComponentLabels * - ********************************************************/ -void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Permeability) +void ScaLBL_GreyscaleColorModel::AssignSolidForce(double *SolidPotential, double *SolidForce){ + + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f/18.f; + double w_edge = 1.f/36.f; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kid[nn] <= 0)||(Mask->id[nn]>=3)){ + double vec_x = double(ii-1); + double vec_y = double(jj-1); + double vec_z = double(kk-1); + double GWNS=SolidPotential[nn]; + phi_x += GWNS*weight*vec_x; + phi_y += GWNS*weight*vec_y; + phi_z += GWNS*weight*vec_z; + } + } + } + } + SolidForce[idx] = phi_x; + SolidForce[idx+Np] = phi_y; + SolidForce[idx+2*Np] = phi_z; + } + } + } + } + delete [] Dst; +} + + +void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double *Permeability, double *SolidPotential) { size_t NLABELS=0; signed char VALUE=0; double POROSITY=0.f; double PERMEABILITY=0.f; + double AFFINITY=0.f; - auto LabelList = greyscale_db->getVector( "ComponentLabels" ); - auto PorosityList = greyscale_db->getVector( "PorosityList" ); - auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + auto LabelList = greyscaleColor_db->getVector( "ComponentLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "ComponentAffinity" ); + auto PorosityList = greyscaleColor_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscaleColor_db->getVector( "PermeabilityList" ); + + //1. Requirement for "ComponentLabels": + // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 + // *label = 1 and 2 are reserved for NW and W phase respectively. + //2. Requirement for "ComponentAffinity": + // *should be in the same length as "ComponentLabels" + // *could leave Affinity=0.0 for label=1 and 2 + //3. Requirement for "PorosityList": + // *for ComponentLables <=0, put porosity value = 0.0; + // *for ComponentLabels >=3, put the corresponding sub-resolution porosity + // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) + //4. Requirement for "PermeabilityList": + // *for ComponentLabels <=2, does not matter, can leave it as 1.0 NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size()){ - ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + if (NLABELS != PorosityList.size() || NLABELS != AffinityList.size() || NLABELS != PermeabilityList.size()){ + ERROR("Error: ComponentLabels, ComponentAffinity, PorosityList and PermeabilityList must all be the same length! \n"); } double label_count[NLABELS]; double label_count_global[NLABELS]; - // Assign the labels for (int idx=0; idx 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability for (int k=1;k0) && (VALUE == LabelList[idx])){ POROSITY=PorosityList[idx]; label_count[idx] += 1.0; idx = NLABELS; - //Mask->id[n] = 0; // set mask to zero since this is an immobile component } } int idx = Map(i,j,k); @@ -227,9 +347,8 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm } } - if (NLABELS != PermeabilityList.size()){ - ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); - } + //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability for (int k=1;k0) && (VALUE == LabelList[idx])){ PERMEABILITY=PermeabilityList[idx]; idx = NLABELS; //Mask->id[n] = 0; // set mask to zero since this is an immobile component @@ -257,6 +376,34 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm } } + //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase + for (int k=0;k=3){ + AFFINITY=AffinityList[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 + } + else{//i.e. label = 1 or 2 + AFFINITY=0.0; + } + idx = NLABELS; + } + } + //NOTE: node_ID = 1 and 2 are reserved + if ((VALUE == 1)||(VALUE == 2)) AFFINITY=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists + SolidPotential[n] = AFFINITY; + } + } + } + // Set Dm to match Mask for (int i=0; iid[i] = Mask->id[i]; @@ -286,8 +433,61 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm } } +void ScaLBL_GreyscaleColorModel::DensityField_Init(){ -void ScaLBL_GreyscaleModel::Create(){ + size_t NLABELS=0; + signed char VALUE=0; + //Read all greyscale node labels and their conrresponding initial saturaiton + //TODO make something default, LabelList=1,2, SwList=0.0, 1.0 + auto LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); + auto SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); + NLABELS=LabelList.size(); + if (NLABELS != SwList.size()){ + ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); + } + + double *Den_temp; + Den_temp=new double [2*Np]; + double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. + double nB=0.5; + + for (int k=1; kid[n]; + if (VALUE>0){ + for (unsigned int idx=0; idx < NLABELS; idx++){ + if (VALUE == LabelList[idx]){ + double Sw = SwList[idx]; + if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); + nB=Sw; + nA=1.0-Sw; + idx = NLABELS; + } + } + if (VALUE==1){//label=1 reserved for NW phase + nA=1.0; + nB=0.0; + } + else if(VALUE==2){//label=2 reserved for W phase + nA=0.0; + nB=1.0; + } + int idx = Map(i,j,k); + Den_temp[idx+0*Np] = nA; + Den_temp[idx+1*Np] = nB; + } + } + } + } + //copy to device + ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] Den_temp; +} + +void ScaLBL_GreyscaleColorModel::Create(){ /* * This function creates the variables needed to run a LBM */ @@ -324,99 +524,81 @@ void ScaLBL_GreyscaleModel::Create(){ neighborSize=18*(Np*sizeof(int)); //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidForce, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); + if (rank==0) printf ("Setting up device neighbor list \n"); fflush(stdout); - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; kLastExterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; - // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); // initialize phi based on PhaseLabel (include solid component labels) double *Poros, *Perm; Poros = new double[Np]; Perm = new double[Np]; - AssignComponentLabels(Poros,Perm); + double *SolidForce_host = new double[3*Np]; + double *SolidPotential_host = new double [Nx*Ny*Nz]; + AssignComponentLabels(Poros,Perm,SolidPotential_host); + AssignSolidForce(SolidPotential_host,SolidForce_host); ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); + ScaLBL_CopyToDevice(SolidForce, SolidForce_host, 3*Np*sizeof(double)); + + ScaLBL_DeviceBarrier(); + //TODO make the following smart pointers + delete [] SolidForce_host; + delete [] SolidPotential_host; + delete [] Poros; + delete [] Perm; } -void ScaLBL_GreyscaleModel::Initialize(){ - if (rank==0) printf ("Initializing distributions \n"); - //TODO: for BGK, you need to consider voxel porosity - // for IMRT, the whole set of feq is different - // if in the future you have different collison mode, need to write two set of initialization functions - if (CollisionType==1){ - ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); - if (rank==0) printf("Collision model: Incompressible MRT.\n"); - } - else if (CollisionType==2){ - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf("Collision model: BGK.\n"); - } - else{ - if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); - ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); - CollisionType=1; - greyscale_db->putScalar( "collision", "IMRT" ); - } - +void ScaLBL_GreyscaleColorModel::Initialize(){ if (Restart == true){ if (rank==0){ - printf("Initializing distributions from Restart! \n"); + printf("Initializing density field and distributions from Restart! \n"); } // Read in the restart file to CPU buffers std::shared_ptr cfq; cfq = std::shared_ptr(new double[19*Np],DeleteArray); + std::shared_ptr cDen; + cDen = std::shared_ptr(new double[2*Np],DeleteArray); FILE *File; File=fopen(LocalRestartFile,"rb"); fread(cfq.get(),sizeof(double),19*Np,File); + fread(cDen.get(),sizeof(double),2*Np,File); fclose(File); // Copy the restart data to the GPU ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); + ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } + else{ + if (rank==0) printf ("Initializing density field \n"); + DensityField_Init();//initialize density field + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); + } } -void ScaLBL_GreyscaleModel::Run(){ +void ScaLBL_GreyscaleColorModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -432,8 +614,8 @@ void ScaLBL_GreyscaleModel::Run(){ if (analysis_db->keyExists( "restart_interval" )){ restart_interval = analysis_db->getScalar( "restart_interval" ); } - if (greyscale_db->keyExists( "timestep" )){ - timestep = greyscale_db->getScalar( "timestep" ); + if (greyscaleColor_db->keyExists( "timestep" )){ + timestep = greyscaleColor_db->getScalar( "timestep" ); } if (rank==0){ @@ -454,213 +636,217 @@ void ScaLBL_GreyscaleModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); auto current_db = db->cloneDatabase(); - double rlx = 1.0/tau; - double rlx_eff = 1.0/tau_eff; double error = 1.0; double flow_rate_previous = 0.0; while (timestep < timestepMax && error > tolerance) { //************************************************************************/ // *************ODD TIMESTEP*************// timestep++; + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, 0, ScaLBL_Comm->LastExterior(), Np); + + //compute Den gradients - component A + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); + //compute Den gradients - component B + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, + tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + +// // Set BCs +// if (BoundaryCondition == 3){ +// ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); +// ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); +// } + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, + tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, 0, ScaLBL_Comm->LastExterior(), Np); + + //compute Den gradients - component A + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); + //compute Den gradients - component B + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, + tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + +// // Set BCs +// if (BoundaryCondition == 3){ +// ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); +// ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); +// } + ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, + tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); //************************************************************************/ - if (timestep%analysis_interval==0){ - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); - //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); - - double count_loc=0; - double count; - double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - //double px_loc,py_loc,pz_loc; - //double px,py,pz; - //double mass_loc,mass_glb; - - //parameters for domain average - int64_t i,j,k,n,imin,jmin,kmin,kmax; - // If external boundary conditions are set, do not average over the inlet and outlet - kmin=1; kmax=Nz-1; - //In case user forgets to specify the inlet/outlet buffer layers for BC>0 - if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; - if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; - - imin=jmin=1; - // If inlet/outlet layers exist use these as default - //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; - //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; - if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer - if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; - -// px_loc = py_loc = pz_loc = 0.f; -// mass_loc = 0.f; +// if (timestep%analysis_interval==0){ +// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); +// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); +// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); +// +// double count_loc=0; +// double count; +// double vax,vay,vaz; +// double vax_loc,vay_loc,vaz_loc; +// //double px_loc,py_loc,pz_loc; +// //double px,py,pz; +// //double mass_loc,mass_glb; +// +// //parameters for domain average +// int64_t i,j,k,n,imin,jmin,kmin,kmax; +// // If external boundary conditions are set, do not average over the inlet and outlet +// kmin=1; kmax=Nz-1; +// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 +// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; +// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; +// +// imin=jmin=1; +// // If inlet/outlet layers exist use these as default +// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; +// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; +// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer +// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; +// +//// px_loc = py_loc = pz_loc = 0.f; +//// mass_loc = 0.f; +//// for (int k=kmin; k 0){ +//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +//// mass_loc += Den*PorosityMap(i,j,k); +//// } +//// } +//// } +//// } +//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// +//// vax = px/mass_glb; +//// vay = py/mass_glb; +//// vaz = pz/mass_glb; +// +// vax_loc = vay_loc = vaz_loc = 0.f; // for (int k=kmin; k 0){ -// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); -// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); -// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); -// mass_loc += Den*PorosityMap(i,j,k); +// vax_loc += Velocity_x(i,j,k); +// vay_loc += Velocity_y(i,j,k); +// vaz_loc += Velocity_z(i,j,k); +// count_loc+=1.0; // } // } // } // } -// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); // -// vax = px/mass_glb; -// vay = py/mass_glb; -// vaz = pz/mass_glb; - - vax_loc = vay_loc = vaz_loc = 0.f; - for (int k=kmin; k 0){ - vax_loc += Velocity_x(i,j,k); - vay_loc += Velocity_y(i,j,k); - vaz_loc += Velocity_z(i,j,k); - count_loc+=1.0; - } - } - } - } - //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); - - vax /= count; - vay /= count; - vaz /= count; - - double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - double dir_x = Fx/force_mag; - double dir_y = Fy/force_mag; - double dir_z = Fz/force_mag; - if (force_mag == 0.0){ - // default to z direction - dir_x = 0.0; - dir_y = 0.0; - dir_z = 1.0; - force_mag = 1.0; - } - //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; - double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); - - error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); - flow_rate_previous = flow_rate; - - //if (rank==0) printf("Computing Minkowski functionals \n"); - Morphology.ComputeScalar(SignDist,0.f); - //Morphology.PrintAll(); - double mu = (tau-0.5)/3.f; - double Vs = Morphology.V(); - double As = Morphology.A(); - double Hs = Morphology.H(); - double Xs = Morphology.X(); - Vs = Dm->Comm.sumReduce( Vs); - As = Dm->Comm.sumReduce( As); - Hs = Dm->Comm.sumReduce( Hs); - Xs = Dm->Comm.sumReduce( Xs); - - double h = Dm->voxel_length; - //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; - double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; - - if (rank==0){ - printf(" AbsPerm = %.5g [micron^2]\n",absperm); - bool WriteHeader=false; - FILE * log_file = fopen("Permeability.csv","r"); - if (log_file != NULL) - fclose(log_file); - else - WriteHeader=true; - log_file = fopen("Permeability.csv","a"); - if (WriteHeader) - fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", - timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); - - fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, - h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); - fclose(log_file); - } - } +// vax = Mask->Comm.sumReduce( vax_loc ); +// vay = Mask->Comm.sumReduce( vay_loc ); +// vaz = Mask->Comm.sumReduce( vaz_loc ); +// count = Mask->Comm.sumReduce( count_loc ); +// +// vax /= count; +// vay /= count; +// vaz /= count; +// +// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); +// double dir_x = Fx/force_mag; +// double dir_y = Fy/force_mag; +// double dir_z = Fz/force_mag; +// if (force_mag == 0.0){ +// // default to z direction +// dir_x = 0.0; +// dir_y = 0.0; +// dir_z = 1.0; +// force_mag = 1.0; +// } +// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; +// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); +// +// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); +// flow_rate_previous = flow_rate; +// +// //if (rank==0) printf("Computing Minkowski functionals \n"); +// Morphology.ComputeScalar(SignDist,0.f); +// //Morphology.PrintAll(); +// double mu = (tau-0.5)/3.f; +// double Vs = Morphology.V(); +// double As = Morphology.A(); +// double Hs = Morphology.H(); +// double Xs = Morphology.X(); +// Vs = Dm->Comm.sumReduce( Vs); +// As = Dm->Comm.sumReduce( As); +// Hs = Dm->Comm.sumReduce( Hs); +// Xs = Dm->Comm.sumReduce( Xs); +// +// double h = Dm->voxel_length; +// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; +// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; +// +// if (rank==0){ +// printf(" AbsPerm = %.5g [micron^2]\n",absperm); +// bool WriteHeader=false; +// FILE * log_file = fopen("Permeability.csv","r"); +// if (log_file != NULL) +// fclose(log_file); +// else +// WriteHeader=true; +// log_file = fopen("Permeability.csv","a"); +// if (WriteHeader) +// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", +// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); +// +// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, +// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); +// fclose(log_file); +// } +// } if (timestep%visualization_interval==0){ VelocityField(); @@ -669,9 +855,9 @@ void ScaLBL_GreyscaleModel::Run(){ if (timestep%restart_interval==0){ //Use rank=0 write out Restart.db if (rank==0) { - greyscale_db->putScalar("timestep",timestep); - greyscale_db->putScalar( "Restart", true ); - current_db->putDatabase("Greyscale", greyscale_db); + greyscaleColor_db->putScalar("timestep",timestep); + greyscaleColor_db->putScalar( "Restart", true ); + current_db->putDatabase("GreyscaleColor", greyscaleColor_db); std::ofstream OutStream("Restart.db"); current_db->print(OutStream, ""); OutStream.close(); @@ -681,17 +867,21 @@ void ScaLBL_GreyscaleModel::Run(){ std::shared_ptr cfq; cfq = std::shared_ptr(new double[19*Np],DeleteArray); ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU + std::shared_ptr cDen; + cDen = std::shared_ptr(new double[2*Np],DeleteArray); + ScaLBL_CopyToHost(cDen.get(),Den,2*Np*sizeof(double));// Copy restart data to the CPU FILE *RESTARTFILE; RESTARTFILE=fopen(LocalRestartFile,"wb"); fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); + fwrite(cDen.get(),sizeof(double),2*Np,RESTARTFILE); fclose(RESTARTFILE); MPI_Barrier(comm); } } PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_greyscale_simulator",1); + PROFILE_SAVE("lbpm_greyscaleColor_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -712,7 +902,7 @@ void ScaLBL_GreyscaleModel::Run(){ // ************************************************************************ } -void ScaLBL_GreyscaleModel::VelocityField(){ +void ScaLBL_GreyscaleColorModel::VelocityField(){ /* Minkowski Morphology(Mask); int SIZE=Np*sizeof(double); @@ -828,7 +1018,7 @@ void ScaLBL_GreyscaleModel::VelocityField(){ } -void ScaLBL_GreyscaleModel::WriteDebug(){ +void ScaLBL_GreyscaleColorModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); @@ -840,26 +1030,26 @@ void ScaLBL_GreyscaleModel::WriteDebug(){ // fwrite(PhaseField.data(),8,N,OUTFILE); // fclose(OUTFILE); // -// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A.%05i.raw",rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B.%05i.raw",rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); -// -// ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); -// FILE *PFILE; -// sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); -// PFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,PFILE); -// fclose(PFILE); + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); FILE *VELX_FILE; @@ -882,17 +1072,17 @@ void ScaLBL_GreyscaleModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); - ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); - FILE *POROS_FILE; - sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); - POROS_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,POROS_FILE); - fclose(POROS_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); - FILE *PERM_FILE; - sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); - PERM_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PERM_FILE); - fclose(PERM_FILE); +// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); +// FILE *POROS_FILE; +// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); +// POROS_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,POROS_FILE); +// fclose(POROS_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); +// FILE *PERM_FILE; +// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); +// PERM_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PERM_FILE); +// fclose(PERM_FILE); } diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index a99925b1..52841c4a 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -1,5 +1,5 @@ /* -Implementation of color lattice boltzmann model +Implementation of multicomponent greyscale lattice boltzmann model */ #include #include @@ -16,10 +16,10 @@ Implementation of color lattice boltzmann model #include "ProfilerApp.h" #include "threadpool/thread_pool.h" -class ScaLBL_GreyscaleModel{ +class ScaLBL_GreyscaleColorModel{ public: - ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleModel(); + ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleColorModel(); // functions in they should be run void ReadParams(string filename); @@ -35,15 +35,15 @@ public: bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; - int CollisionType; - double tau; - double tau_eff; - double Den;//constant density + double tauA,tauB; + double tauA_eff,tauB_eff; + double rhoA,rhoB; double tolerance; double Fx,Fy,Fz,flux; double din,dout; double dp;//solid particle diameter, unit in voxel double GreyPorosity; + double Gsc; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -56,18 +56,21 @@ public: // input database std::shared_ptr db; std::shared_ptr domain_db; - std::shared_ptr greyscale_db; + std::shared_ptr greyscaleColor_db; std::shared_ptr analysis_db; std::shared_ptr vis_db; signed char *id; int *NeighborList; - int *dvcMap; - double *fq; + double *fq,*Aq,*Bq; + double *Den; double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; double *Pressure_dvc; + double *SolidForce; + double *DenGradA; + double *DenGradB; IntArray Map; DoubleArray SignDist; DoubleArray Velocity_x; @@ -86,7 +89,9 @@ private: char LocalRankFilename[40]; char LocalRestartFile[40]; - void AssignComponentLabels(double *Porosity, double *Permeablity); - + void AssignComponentLabels(double *Porosity, double *Permeablity, double *SolidPotential); + void AssignSolidForce(double *SolidPotential, double *SolidForce); + void DensityField_Init(); + }; diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 11d92c80..1ef65204 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -324,7 +324,6 @@ void ScaLBL_GreyscaleModel::Create(){ neighborSize=18*(Np*sizeof(int)); //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); @@ -332,38 +331,8 @@ void ScaLBL_GreyscaleModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); + if (rank==0) printf ("Setting up device neighbor list \n"); fflush(stdout); - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; kLastExterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; - // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); // initialize phi based on PhaseLabel (include solid component labels) diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index a99925b1..3e883b16 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -62,7 +62,6 @@ public: signed char *id; int *NeighborList; - int *dvcMap; double *fq; double *Permeability;//grey voxel permeability double *Porosity; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8b14a9dc..56eda802 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,6 +4,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_greyscaleColor_simulator.cpp b/tests/lbpm_greyscaleColor_simulator.cpp new file mode 100644 index 00000000..b836e03c --- /dev/null +++ b/tests/lbpm_greyscaleColor_simulator.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI.h" +#include "models/GreyscaleColorModel.h" +//#define WRITE_SURFACES + +using namespace std; + + +int main(int argc, char **argv) +{ + //***************************************** + // ***** MPI STUFF **************** + //***************************************** + // Initialize MPI + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { + // parallel domain size (# of sub-domains) + int nprocx,nprocy,nprocz; + int iproc,jproc,kproc; + + if (rank == 0){ + printf("****************************************\n"); + printf("Running Greyscale Two-Phase Calculation \n"); + printf("****************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm); + auto filename = argv[1]; + GreyscaleColor.ReadParams(filename); + GreyscaleColor.SetDomain(); // this reads in the domain + GreyscaleColor.ReadInput(); + GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleColor.Run(); + //GreyscaleColor.VelocityField(); + GreyscaleColor.WriteDebug(); + } + // **************************************************** + MPI_Barrier(comm); + MPI_Finalize(); + // **************************************************** +} From ce09bb9ad527475690d6297879341678eac1fb37 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 7 Apr 2020 22:17:29 -0400 Subject: [PATCH 104/270] save the work --- models/GreyscaleColorModel.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 421805c5..2a01f45b 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -437,10 +437,20 @@ void ScaLBL_GreyscaleColorModel::DensityField_Init(){ size_t NLABELS=0; signed char VALUE=0; - //Read all greyscale node labels and their conrresponding initial saturaiton - //TODO make something default, LabelList=1,2, SwList=0.0, 1.0 - auto LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); - auto SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); + + if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ + auto LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); + } + else{ + vector LabelList{1,2}; + } + if (greyscaleColor_db->keyExists( "GreyNodeSw" )){ + auto SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); + } + else{ + vector SwList{0.0,1.0}; + } + NLABELS=LabelList.size(); if (NLABELS != SwList.size()){ ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); From 606e81d6849a38036f8332baabad29b61f77ebbc Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 8 Apr 2020 16:48:39 -0400 Subject: [PATCH 105/270] GPU version: save the work --- gpu/GreyscaleColor.cu | 18 ++++++++++++------ models/GreyscaleColorModel.cpp | 15 +++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index c9586d3c..dd24c995 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -357,9 +357,12 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// // fluid-fluid force - Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); - Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); - Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); +// Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); +// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); +// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); + Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); + Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); // fluid-solid force Gfs_x = (nA-nB)*SolidForce[n+0*Np]; Gfs_y = (nA-nB)*SolidForce[n+1*Np]; @@ -978,9 +981,12 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// // fluid-fluid force - Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); - Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); - Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); +// Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); +// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); +// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); + Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); + Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); // fluid-solid force Gfs_x = (nA-nB)*SolidForce[n+0*Np]; Gfs_y = (nA-nB)*SolidForce[n+1*Np]; diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 2a01f45b..5eefb899 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -438,18 +438,17 @@ void ScaLBL_GreyscaleColorModel::DensityField_Init(){ size_t NLABELS=0; signed char VALUE=0; + vector LabelList{1,2}; + vector SwList{0.0,1.0}; + if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ - auto LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); + LabelList.clear(); + LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); } - else{ - vector LabelList{1,2}; - } if (greyscaleColor_db->keyExists( "GreyNodeSw" )){ - auto SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); + SwList.clear(); + SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); } - else{ - vector SwList{0.0,1.0}; - } NLABELS=LabelList.size(); if (NLABELS != SwList.size()){ From bdf8539f408405ff9612d054d9981d35de1e0368 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 10 Apr 2020 15:03:15 -0400 Subject: [PATCH 106/270] debugging strange mass conservation issue --- analysis/SubPhase.cpp | 6 +- common/ScaLBL.cpp | 14 +- models/ColorModel.cpp | 49 +++---- tests/TestMassConservationD3Q7.cpp | 206 +++++++++++++++-------------- tests/lbpm_color_simulator.cpp | 2 +- 5 files changed, 143 insertions(+), 134 deletions(-) diff --git a/analysis/SubPhase.cpp b/analysis/SubPhase.cpp index 76541ffd..7ef8194b 100644 --- a/analysis/SubPhase.cpp +++ b/analysis/SubPhase.cpp @@ -280,7 +280,7 @@ void SubPhase::Basic(){ dir_y = 0.0; dir_z = 1.0; } - if (Dm->BoundaryCondition > 0 ){ + if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){ // compute the pressure drop double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0; double length = ((Nz-2)*Dm->nprocz()); @@ -376,8 +376,8 @@ void SubPhase::Full(){ // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; - if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; - if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; + if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == 0) kmin=4; + if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; imin=jmin=1; // If inlet layers exist use these as default diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index ef9b2341..07aa3f1d 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1286,7 +1286,7 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N); } - if (BoundaryCondition == 5){ +/* if (BoundaryCondition == 5){ if (kproc == 0){ ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Aq, sendCount_z, N); ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Bq, sendCount_z, N); @@ -1296,6 +1296,7 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){ ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Bq, sendCount_Z, N); } } + */ //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... @@ -1543,10 +1544,15 @@ void ScaLBL_Communicator::RecvHalo(double *data){ ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N); ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N); } - //................................................................................... Lock=false; // unlock the communicator after communications complete //................................................................................... + if (BoundaryCondition == 5 && kproc == 0){ + ScaLBL_CopySlice_z(data,Nx,Ny,Nz,1,0); + } + if (BoundaryCondition == 5 && kproc == nprocz-1){ + ScaLBL_CopySlice_z(data,Nx,Ny,Nz,Nz-2,Nz-1); + } } void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray ®data){ @@ -1583,7 +1589,7 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == 0) { if (BoundaryCondition == 5){ - ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,1,0); + //ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,1,0); } else { // Set the phase indicator field and density on the z inlet @@ -1596,7 +1602,7 @@ void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){ if (kproc == nprocz-1){ if (BoundaryCondition == 5){ - ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,Nz-2,Nz-1); + //ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,Nz-2,Nz-1); } else { // Set the phase indicator field and density on the Z outlet diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 83ddc5d4..a0f339c6 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -470,7 +470,8 @@ void ScaLBL_ColorModel::Initialize(){ ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + // establish reservoirs for external bC + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4 ){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -743,7 +744,7 @@ void ScaLBL_ColorModel::Run(){ //************************************************************************ PROFILE_STOP("Update"); - if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ + if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ printf("%i %f \n",timestep,din); } // Run the analysis @@ -1159,7 +1160,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -1447,7 +1448,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta // 7. Re-initialize phase field and density ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -1516,25 +1517,25 @@ void ScaLBL_ColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); -// FILE *CGX_FILE; -// sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); -// CGX_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGX_FILE); -// fclose(CGX_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); -// FILE *CGY_FILE; -// sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); -// CGY_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGY_FILE); -// fclose(CGY_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); -// FILE *CGZ_FILE; -// sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); -// CGZ_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGZ_FILE); -// fclose(CGZ_FILE); +/* ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); + FILE *CGX_FILE; + sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); + CGX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGX_FILE); + fclose(CGX_FILE); + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); + FILE *CGY_FILE; + sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); + CGY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGY_FILE); + fclose(CGY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); + FILE *CGZ_FILE; + sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); + CGZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGZ_FILE); + fclose(CGZ_FILE); +*/ } diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index bbfe8cae..35e42c1c 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -69,10 +69,11 @@ int main(int argc, char **argv) // Initialize MPI int rank,nprocs; MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) + int CleanCheck = 0; if (rank == 0){ printf("********************************************************\n"); @@ -84,68 +85,68 @@ int main(int argc, char **argv) } } { - auto filename = argv[1]; - ScaLBL_ColorModel CM(rank,nprocs,comm); - CM.ReadParams(filename); - CM.SetDomain(); - int i,j,k,n; - int Nx,Ny,Nz,N,Np; - Nx = CM.Nx; - Ny = CM.Ny; - Nz = CM.Nz; - N = Nx*Ny*Nz; + auto filename = argv[1]; + ScaLBL_ColorModel CM(rank,nprocs,comm); + CM.ReadParams(filename); + CM.SetDomain(); + int i,j,k,n; + int Nx,Ny,Nz,N,Np; + Nx = CM.Nx; + Ny = CM.Ny; + Nz = CM.Nz; + N = Nx*Ny*Nz; - //CM.ReadInput(); - double radius=0.4*double(Nx); - InitializeBubble(CM,radius); - CM.Create(); // creating the model will create data structure to match the pore structure and allocate variables - CM.Initialize(); // initializing the model will set initial conditions for variables - //CM.Run(); - //CM.WriteDebug(); + //CM.ReadInput(); + double radius=0.4*double(Nx); + InitializeBubble(CM,radius); + CM.Create(); // creating the model will create data structure to match the pore structure and allocate variables + CM.Initialize(); // initializing the model will set initial conditions for variables + //CM.Run(); + //CM.WriteDebug(); - CM.timestepMax = 10; - CM.Run(); + CM.timestepMax = 10; + CM.Run(); - Np = CM.Np; - double *DenOriginal, *DenFinal; - DenOriginal = new double [2*Np]; - DenFinal = new double [2*Np]; + Np = CM.Np; + double *DenOriginal, *DenFinal; + DenOriginal = new double [2*Np]; + DenFinal = new double [2*Np]; - // Run the odd timestep - ScaLBL_CopyToHost(DenOriginal,CM.Den,2*Np*sizeof(double)); - /* + // Run the odd timestep + ScaLBL_CopyToHost(DenOriginal,CM.Den,2*Np*sizeof(double)); + /* CM.ScaLBL_Comm->BiSendD3Q7AA(CM.Aq,CM.Bq); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_PhaseField(CM.NeighborList, CM.dvcMap, CM.Aq, CM.Bq, CM.Den, CM.Phi, CM.ScaLBL_Comm->FirstInterior(), CM.ScaLBL_Comm->LastInterior(), CM.Np); CM.ScaLBL_Comm->BiRecvD3Q7AA(CM.Aq,CM.Bq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); ScaLBL_D3Q7_AAodd_PhaseField(CM.NeighborList, CM.dvcMap, CM.Aq, CM.Bq, CM.Den, CM.Phi, 0, CM.ScaLBL_Comm->LastExterior(), CM.Np); - */ + */ - CM.timestepMax = 2; - CM.Run(); - int D3Q7[7][3]={{0,0,0},{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}}; - // Compare and make sure mass is conserved at every lattice site - auto Error = new double[N]; - auto A_q = new double[7*Np]; - //auto B_q = new double[7*Np]; - bool CleanCheck = true; - double original,final, sum_q; - double total_mass_A_0 = 0.0; - double total_mass_B_0= 0.0; - double total_mass_A_1 = 0.0; - double total_mass_B_1= 0.0; - int count_negative_A = 0; - int count_negative_B = 0; - ScaLBL_CopyToHost(DenFinal,CM.Den,2*Np*sizeof(double)); - ScaLBL_CopyToHost(A_q,CM.Aq,7*Np*sizeof(double)); - for (i=0; i-1){ - //printf("idx=%i\n",idx); + CM.timestepMax = 2; + CM.timestep = 0; + CM.Run(); + int D3Q7[7][3]={{0,0,0},{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}}; + // Compare and make sure mass is conserved at every lattice site + auto Error = new double[N]; + auto A_q = new double[7*Np]; + //auto B_q = new double[7*Np]; + double original,final, sum_q; + double total_mass_A_0 = 0.0; + double total_mass_B_0= 0.0; + double total_mass_A_1 = 0.0; + double total_mass_B_1= 0.0; + int count_negative_A = 0; + int count_negative_B = 0; + ScaLBL_CopyToHost(DenFinal,CM.Den,2*Np*sizeof(double)); + ScaLBL_CopyToHost(A_q,CM.Aq,7*Np*sizeof(double)); + for (i=0; i-1){ + //printf("idx=%i\n",idx); final = DenFinal[idx]; if (final < 0.0) count_negative_A++; original = DenOriginal[idx]; @@ -153,60 +154,61 @@ int main(int argc, char **argv) total_mass_A_1 += final; sum_q = A_q[idx]; for (int q=1; q<7; q++){ - int Cqx = D3Q7[q][0]; - int Cqy = D3Q7[q][1]; - int Cqz = D3Q7[q][2]; - int iq = CM.Map(i-Cqx,j-Cqy,k-Cqz); - if (iq < Np && iq > -1){ - sum_q += A_q[q*Np+iq]; - } - else if (q%2==0){ - sum_q += A_q[(q-1)*Np+idx]; - } - else{ - sum_q += A_q[(q+1)*Np+idx]; - } + int Cqx = D3Q7[q][0]; + int Cqy = D3Q7[q][1]; + int Cqz = D3Q7[q][2]; + int iq = CM.Map(i-Cqx,j-Cqy,k-Cqz); + if (iq < Np && iq > -1){ + sum_q += A_q[q*Np+iq]; + } + else if (q%2==0){ + sum_q += A_q[(q-1)*Np+idx]; + } + else{ + sum_q += A_q[(q+1)*Np+idx]; + } } Error[n] = sum_q - original; - - /*if (fabs(DenFinal[idx] - DenOriginal[idx]) > 1e-15){ - //if (CM.Dm->id[n] == 0) printf("Solid phase! \n"); - //if (CM.Dm->id[n] == 1) printf("Wetting phase! \n"); - //if (CM.Dm->id[n] == 2) printf("Non-wetting phase! \n"); - printf("Mass not conserved: WP density, site=%i,%i,%i, original = %f, final = %f \n",i,j,k,original,final); - CleanCheck=false; - Error[n] += final-original; - }*/ + + if (fabs(DenFinal[idx] - DenOriginal[idx]) > 1e-15){ + //if (CM.Dm->id[n] == 0) printf("Solid phase! \n"); + //if (CM.Dm->id[n] == 1) printf("Wetting phase! \n"); + //if (CM.Dm->id[n] == 2) printf("Non-wetting phase! \n"); + //printf("Mass not conserved: WP density, site=%i,%i,%i, original = %f, final = %f \n",i,j,k,original,final); + CleanCheck=false; + Error[n] += final-original; + } final = DenFinal[Np+idx]; if (final < 0.0) count_negative_B++; original = DenOriginal[Np+idx]; total_mass_B_0 += original; total_mass_B_1 += final; - /*if (fabs(DenFinal[Np+idx] - DenOriginal[Np+idx]) > 1e-15){ - //if (CM.Dm->id[n] == 0) printf("Solid phase! \n"); - //if (CM.Dm->id[n] == 1) printf("Wetting phase! \n"); - //if (CM.Dm->id[n] == 2) printf("Non-wetting phase! \n"); - printf("Mass not conserved: NWP density, site=%i,%i,%i, original = %f, final = %f \n",i,j,k,original,final); - CleanCheck=false; - Error[n] += final-original; - }*/ + if (fabs(DenFinal[Np+idx] - DenOriginal[Np+idx]) > 1e-15){ + //if (CM.Dm->id[n] == 0) printf("Solid phase! \n"); + //if (CM.Dm->id[n] == 1) printf("Wetting phase! \n"); + //if (CM.Dm->id[n] == 2) printf("Non-wetting phase! \n"); + //printf("Mass not conserved: NWP density, site=%i,%i,%i, original = %f, final = %f \n",i,j,k,original,final); + CleanCheck=false; + Error[n] += final-original; + + } + } } } } - } - printf("Negative density values for A = %i \n",count_negative_A); - printf("Negative density values for B = %i \n",count_negative_B); - printf("Global mass difference A = %.5g\n",total_mass_A_1-total_mass_A_0); - printf("Global mass difference B = %.5g\n",total_mass_B_1-total_mass_B_0); + printf("Negative density values for A = %i \n",count_negative_A); + printf("Negative density values for B = %i \n",count_negative_B); + printf("Global mass difference A = %.5g\n",total_mass_A_1-total_mass_A_0); + printf("Global mass difference B = %.5g\n",total_mass_B_1-total_mass_B_0); - if (count_negative_A > 0 ||count_negative_B > 0) CleanCheck=1; - if (fabs(total_mass_A_1-total_mass_A_0) > 1.0e-15||fabs(total_mass_B_1-total_mass_B_0) > 1.0e-15 ) CleanCheck=2; + if (count_negative_A > 0 ||count_negative_B > 0) CleanCheck=1; + if (fabs(total_mass_A_1-total_mass_A_0) > 1.0e-13||fabs(total_mass_B_1-total_mass_B_0) > 1.0e-13 ) CleanCheck=2; - /* - FILE *OUTFILE; - OUTFILE = fopen("error.raw","wb"); - fwrite(Error,8,N,OUTFILE); - fclose(OUTFILE); + FILE *OUTFILE; + OUTFILE = fopen("error.raw","wb"); + fwrite(Error,8,N,OUTFILE); + fclose(OUTFILE); + /* if (rank==0) printf("Checking that the correct velocity is retained \n"); // Swap convention is observed -- velocity is negative @@ -256,15 +258,15 @@ int main(int argc, char **argv) } } } -*/ - if (CleanCheck){ - if (rank==0) printf("Test passed: mass conservation for D3Q7 \n"); - } - else { - if (rank==0) printf("Test failed!: mass conservation for D3Q7 \n"); + */ + if (CleanCheck == 0){ + if (rank==0) printf("Test passed: mass conservation for D3Q7 \n"); + } + else { + if (rank==0) printf("Test failed!: mass conservation for D3Q7 \n"); + } } -} // **************************************************** MPI_Barrier(comm); MPI_Finalize(); diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 1f63c653..79b2a718 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -58,7 +58,7 @@ int main(int argc, char **argv) ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables ColorModel.Initialize(); // initializing the model will set initial conditions for variables ColorModel.Run(); - //ColorModel.WriteDebug(); + ColorModel.WriteDebug(); PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); From d1d92ea6bbc793fe66e007fffb3f79e3aec75a74 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 10 Apr 2020 21:31:44 -0400 Subject: [PATCH 107/270] debug mass conservation test --- tests/TestMassConservationD3Q7.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index 35e42c1c..6186fd60 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -41,7 +41,7 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) int jglobal= j+(Ny-2)*ColorModel.Mask->jproc(); int kglobal= k+(Nz-2)*ColorModel.Mask->kproc(); // Initialize phase position field for parallel bubble test - if (jglobal < 40){ + if (kglobal < 40){ ColorModel.Mask->id[n] = 0; } else if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) @@ -183,15 +183,15 @@ int main(int argc, char **argv) original = DenOriginal[Np+idx]; total_mass_B_0 += original; total_mass_B_1 += final; - if (fabs(DenFinal[Np+idx] - DenOriginal[Np+idx]) > 1e-15){ + /*if (fabs(DenFinal[Np+idx] - DenOriginal[Np+idx]) > 1e-15){ //if (CM.Dm->id[n] == 0) printf("Solid phase! \n"); //if (CM.Dm->id[n] == 1) printf("Wetting phase! \n"); //if (CM.Dm->id[n] == 2) printf("Non-wetting phase! \n"); //printf("Mass not conserved: NWP density, site=%i,%i,%i, original = %f, final = %f \n",i,j,k,original,final); CleanCheck=false; Error[n] += final-original; - - } + } + */ } } } From 984e7b504eb3429320ec5a39eef762adb2946131 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 12 Apr 2020 23:38:24 -0400 Subject: [PATCH 108/270] GPU: save the work; mass transport using chemical potential --- common/ScaLBL.h | 12 + gpu/GreyscaleColor.cu | 1383 +++++++++++++++++++++++++++++++- models/GreyscaleColorModel.cpp | 67 +- models/GreyscaleColorModel.h | 4 + 4 files changed, 1454 insertions(+), 12 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index df87fcc4..e9561be3 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -84,6 +84,16 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dis double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure); + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure); + extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); @@ -94,6 +104,8 @@ extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index dd24c995..edfb903f 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -1266,6 +1266,1292 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double } } +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double nA,nB; + double nA_sum_tmp,nB_sum_tmp; + double a1,b1,a2,b2; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; + double Gff_x,Gff_y,Gff_z; + double Gfs_x,Gfs_y,Gfs_z; + double Gsc = 1.f/3.f*sqrt(kappa*lambda); + double nA_lap,nB_lap; + double chem_a,chem_b; + double rlx_massA,rlx_massB; + + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + dist[1*Np+n] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + dist[2*Np+n] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[3*Np+n] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[4*Np+n] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[5*Np+n] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[6*Np+n] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + dist[7*Np+n] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + dist[8*Np+n] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + dist[9*Np+n] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + dist[10*Np+n] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + dist[11*Np+n] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + dist[12*Np+n] = fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + dist[13*Np+n] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + dist[14*Np+n] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + dist[15*Np+n] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + dist[16*Np+n] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + dist[17*Np+n] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + dist[18*Np+n] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // calcuale chemical potential + chem_a = lambda*(nA*nA*nA-1.5*nA*nA+0.5*nA)-kappa*nA_lap; + chem_b = lambda*(nB*nB*nB-1.5*nB*nB+0.5*nB)-kappa*nB_lap; + nA_sum_tmp = 0.0; + nB_sum_tmp = 0.0; + rlx_massA = 3.f-sqrt(3.f); + rlx_massB = 3.f-sqrt(3.f); + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + a1 = Aq[1*Np+n]; + b1 = Bq[1*Np+n]; + a2 = Aq[2*Np+n]; + b2 = Bq[2*Np+n]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*ux)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*ux)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*ux)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*ux)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + a1 = Aq[3*Np+n]; + b1 = Bq[3*Np+n]; + a2 = Aq[4*Np+n]; + b2 = Bq[4*Np+n]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uy)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uy)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uy)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uy)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + a1 = Aq[5*Np+n]; + b1 = Bq[5*Np+n]; + a2 = Aq[6*Np+n]; + b2 = Bq[6*Np+n]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uz)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uz)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uz)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uz)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + a1=Aq[n]; + b1=Bq[n]; + Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-nA_sum_tmp); + Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-nB_sum_tmp); + + + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double nA,nB; + double nA_sum_tmp,nB_sum_tmp; + double a1,b1,a2,b2; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; + double Gff_x,Gff_y,Gff_z; + double Gfs_x,Gfs_y,Gfs_z; + double Gsc = 1.f/3.f*sqrt(kappa*lambda); + double nA_lap,nB_lap; + double chem_a,chem_b; + double rlx_massA,rlx_massB; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + fq = dist[nr1]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// + // fluid-fluid force + Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); + Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); + Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // calcuale chemical potential + chem_a = lambda*(nA*nA*nA-1.5*nA*nA+0.5*nA)-kappa*nA_lap; + chem_b = lambda*(nB*nB*nB-1.5*nB*nB+0.5*nB)-kappa*nB_lap; + nA_sum_tmp = 0.0; + nB_sum_tmp = 0.0; + rlx_massA = 3.f-sqrt(3.f); + rlx_massB = 3.f-sqrt(3.f); + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + a1 = Aq[nr2]; + b1 = Bq[nr2]; + a2 = Aq[nr1]; + b2 = Bq[nr1]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*ux)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*ux)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*ux)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*ux)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + a1 = Aq[nr4]; + b1 = Bq[nr4]; + a2 = Aq[nr3]; + b2 = Bq[nr3]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uy)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uy)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uy)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uy)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + a1 = Aq[nr6]; + b1 = Bq[nr6]; + a2 = Aq[nr5]; + b2 = Bq[nr5]; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uz)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uz)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uz)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uz)); + nA_sum_tmp += a1+a2; + nB_sum_tmp += b1+b2; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + a1=Aq[n]; + b1=Bq[n]; + Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-nA_sum_tmp); + Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-nB_sum_tmp); + + + } + } +} + __global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ int n; int S = Np/NBLOCKS/NTHREADS + 1; @@ -1510,9 +2796,9 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, doub m18 = Den[nn]*int(n!=nn); //............Compute the Color Gradient................................... - nx = 1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = 1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = 1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx = 0.3333333333333333*1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny = 0.3333333333333333*1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz = 0.3333333333333333*1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); DenGrad[n] = nx; DenGrad[Np+n] = ny; @@ -1521,6 +2807,61 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, doub } } +__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ + + int n,nn; + // distributions + double m1,m2,m3,m4,m5,m6,m7,m8,m9; + double m10,m11,m12,m13,m14,m15,m16,m17,m18; + double lap; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist, Aq, Bq, Den, DenGradA, DenGradB,DenLapA,DenLapB, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB,DenLapA,DenLapB, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); + } +} + + extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np){ dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, start, finish, Np); cudaError_t err = cudaGetLastError(); @@ -1594,3 +2964,10 @@ extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double * } } +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ + dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian<<>>(neighborList, Den, DenLap, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Laplacian: %s \n",cudaGetErrorString(err)); + } +} diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 5eefb899..4b1164e1 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -14,7 +14,7 @@ void DeleteArray( const TYPE *p ) } ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),Gsc(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),Gsc(0),gamma(0),kappa(0),lambda(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -48,6 +48,10 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel + Gsc = 1.0; + gamma = 1.0; + kappa = 1.0; + lambda = 1.0; // ---------------------- Greyscale Model parameters -----------------------// if (greyscaleColor_db->keyExists( "timestepMax" )){ @@ -70,6 +74,15 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ if (greyscaleColor_db->keyExists( "Gsc" )){ Gsc = greyscaleColor_db->getScalar( "Gsc" ); } + if (greyscaleColor_db->keyExists( "gamma" )){ + gamma = greyscaleColor_db->getScalar( "gamma" ); + } + if (greyscaleColor_db->keyExists( "kappa" )){ + kappa = greyscaleColor_db->getScalar( "kappa" ); + } + if (greyscaleColor_db->keyExists( "lambda" )){ + lambda = greyscaleColor_db->getScalar( "lambda" ); + } if (greyscaleColor_db->keyExists( "dp" )){ dp = greyscaleColor_db->getScalar( "dp" ); } @@ -544,6 +557,8 @@ void ScaLBL_GreyscaleColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &SolidForce, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenLapA, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenLapB, sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device neighbor list \n"); @@ -670,9 +685,24 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + //compute Den laplacian - component A + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0*Np],DenLapA); + + //compute Den laplacian - component B + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[1*Np],DenLapB); + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); +// ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, +// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -681,8 +711,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, - tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); +// ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, +// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -707,9 +739,24 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + //compute Den laplacian - component A + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0*Np],DenLapA); + + //compute Den laplacian - component B + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[1*Np],DenLapB); + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); +// ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, +// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -718,8 +765,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } - ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, - tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); +// ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, +// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 52841c4a..85bbb56f 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -44,6 +44,8 @@ public: double dp;//solid particle diameter, unit in voxel double GreyPorosity; double Gsc; + double gamma; + double kappa,lambda; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -71,6 +73,8 @@ public: double *SolidForce; double *DenGradA; double *DenGradB; + double *DenLapA; + double *DenLapB; IntArray Map; DoubleArray SignDist; DoubleArray Velocity_x; From d6a8647ee1d0fd4f8ef30a079b7526ede60aa0c4 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 17 Apr 2020 12:21:29 -0400 Subject: [PATCH 109/270] cannot exclude inlet / outlet without screwing up topology --- analysis/SubPhase.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/analysis/SubPhase.cpp b/analysis/SubPhase.cpp index 7ef8194b..2a5e3350 100644 --- a/analysis/SubPhase.cpp +++ b/analysis/SubPhase.cpp @@ -161,12 +161,12 @@ void SubPhase::Basic(){ // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; imin=jmin=1; - // If inlet/outlet layers exist use these as default + /*// If inlet/outlet layers exist use these as default if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; - + */ nb.reset(); wb.reset(); double count_w = 0.0; @@ -376,16 +376,17 @@ void SubPhase::Full(){ // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; - if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == 0) kmin=4; + /*if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == 0) kmin=4; if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; - + */ imin=jmin=1; - // If inlet layers exist use these as default + /*// If inlet layers exist use these as default + * NOTE -- excluding inlet / outlet will screw up topological averages!!! if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; - + */ nd.reset(); nc.reset(); wd.reset(); wc.reset(); iwn.reset(); iwnc.reset(); Dm->CommunicateMeshHalo(Phi); From 297ea4cb63ab1ec610606722308d00ddafd9b987 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 17 Apr 2020 16:23:06 -0400 Subject: [PATCH 110/270] update the chemical potential approach and save work --- common/ScaLBL.h | 28 +- gpu/GreyscaleColor.cu | 511 ++++++++++++++++++++++++--------- models/GreyscaleColorModel.cpp | 252 +++++++++++----- models/GreyscaleColorModel.h | 20 +- 4 files changed, 583 insertions(+), 228 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index e9561be3..7bee2d1b 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -84,28 +84,34 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dis double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure); +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure); +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, + double *Pressure, double rhoA,double rhoB, int Np); + +extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi, double *PressTensor, double *PhiLap, + double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index edfb903f..b773ab4a 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -3,6 +3,69 @@ #define NBLOCKS 1024 #define NTHREADS 256 +__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Poros,double *Velocity, + double *Pressure, double rhoA,double rhoB, int N){ + + int n; + double ux,uy,uz,u_mag; + double pressure; + double porosity; + double rho0; + double phi; + double nA,nB; + + int S = N/NBLOCKS/NTHREADS + 1; + for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + //Load pressure gradient + px=PressureGrad[0*Np+n]; + py=PressureGrad[1*Np+n]; + pz=PressureGrad[2*Np+n]; + + //Load pressure tensor gradient + //For reference full list of PressTensorGrad + //PressTensorGrad[n+0*Np] = Pxx_x + //PressTensorGrad[n+1*Np] = Pxx_y + //PressTensorGrad[n+2*Np] = Pxx_z + //PressTensorGrad[n+3*Np] = Pyy_x + //PressTensorGrad[n+4*Np] = Pyy_y + //PressTensorGrad[n+5*Np] = Pyy_z + //PressTensorGrad[n+6*Np] = Pzz_x + //PressTensorGrad[n+7*Np] = Pzz_y + //PressTensorGrad[n+8*Np] = Pzz_z + //PressTensorGrad[n+9*Np] = Pxy_x + //PressTensorGrad[n+10*Np] = Pxy_y + //PressTensorGrad[n+11*Np] = Pxy_z + //PressTensorGrad[n+12*Np] = Pyz_x + //PressTensorGrad[n+13*Np] = Pyz_y + //PressTensorGrad[n+14*Np] = Pyz_z + //PressTensorGrad[n+15*Np] = Pxz_x + //PressTensorGrad[n+16*Np] = Pxz_y + //PressTensorGrad[n+17*Np] = Pxz_z + Pxx_x = PressTensorGrad[0*Np+n]; + Pyy_y = PressTensorGrad[4*Np+n]; + Pzz_z = PressTensorGrad[8*Np+n]; + Pxy_x = PressTensorGrad[9*Np+n]; + Pxz_x = PressTensorGrad[15*Np+n]; + Pxy_y = PressTensorGrad[10*Np+n]; + Pyz_y = PressTensorGrad[13*Np+n]; + Pyz_z = PressTensorGrad[14*Np+n]; + Pxz_z = PressTensorGrad[17*Np+n]; + //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... + //TODO double check if you need porosity as a fre-factor + Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); + Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); + Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; // local density rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); @@ -1623,20 +1728,6 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double m18 -= fq; //---------------------------------------------------------------------// - //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// - // fluid-fluid force - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); @@ -1810,10 +1901,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double //-----------------------Mass transport------------------------// // calcuale chemical potential - chem_a = lambda*(nA*nA*nA-1.5*nA*nA+0.5*nA)-kappa*nA_lap; - chem_b = lambda*(nB*nB*nB-1.5*nB*nB+0.5*nB)-kappa*nB_lap; - nA_sum_tmp = 0.0; - nB_sum_tmp = 0.0; + chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; + chem_b = lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; rlx_massA = 3.f-sqrt(3.f); rlx_massB = 3.f-sqrt(3.f); @@ -1824,12 +1913,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double b1 = Bq[1*Np+n]; a2 = Aq[2*Np+n]; b2 = Bq[2*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*ux)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*ux)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*ux)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*ux)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); Aq[1*Np+n] = a1; Bq[1*Np+n] = b1; @@ -1843,12 +1930,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double b1 = Bq[3*Np+n]; a2 = Aq[4*Np+n]; b2 = Bq[4*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uy)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uy)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uy)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uy)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); Aq[3*Np+n] = a1; Bq[3*Np+n] = b1; @@ -1861,12 +1946,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double b1 = Bq[5*Np+n]; a2 = Aq[6*Np+n]; b2 = Bq[6*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uz)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uz)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uz)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uz)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); Aq[5*Np+n] = a1; Bq[5*Np+n] = b1; @@ -1878,18 +1961,18 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double // Stationary value - distribution 0 a1=Aq[n]; b1=Bq[n]; - Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-nA_sum_tmp); - Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-nB_sum_tmp); + Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); + Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); } } } -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure){ +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; double vx,vy,vz,v_mag; @@ -1910,17 +1993,19 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou double mu_eff;//effective kinematic viscosity for Darcy term double rho0; double phi; + double phi_lap;//laplacian of phase field double nA,nB; - double nA_sum_tmp,nB_sum_tmp; double a1,b1,a2,b2; - double nA_gradx,nA_grady,nA_gradz; - double nB_gradx,nB_grady,nB_gradz; - double Gff_x,Gff_y,Gff_z; double Gfs_x,Gfs_y,Gfs_z; - double Gsc = 1.f/3.f*sqrt(kappa*lambda); - double nA_lap,nB_lap; + double Gff_x,Gff_y,Gff_z; double chem_a,chem_b; double rlx_massA,rlx_massB; + // *---------------------------------Pressure Tensor Gradient------------------------------------*// + double Pxx_x,Pyy_y,Pzz_z; + double Pxy_x,Pxy_y; + double Pyz_y,Pyz_z; + double Pxz_x,Pxz_z; + double px,py,pz; //pressure gradient const double mrt_V1=0.05263157894736842; const double mrt_V2=0.012531328320802; @@ -1945,17 +2030,58 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // read the component number densities nA = Den[n]; nB = Den[Np + n]; - nA_gradx = DenGradA[n+0*Np]; - nA_grady = DenGradA[n+1*Np]; - nA_gradz = DenGradA[n+2*Np]; - nB_gradx = DenGradB[n+0*Np]; - nB_grady = DenGradB[n+1*Np]; - nB_gradz = DenGradB[n+2*Np]; - nA_lap = DenLapA[n];//laplacian of denA - nB_lap = DenLapB[n]; - // compute phase indicator field phi=(nA-nB)/(nA+nB); + // load laplacian of phase field + phi_lap = PhiLap[n]; + // Load voxel porosity and perm + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + //Load pressure gradient + px=PressureGrad[0*Np+n]; + py=PressureGrad[1*Np+n]; + pz=PressureGrad[2*Np+n]; + + //Load pressure tensor gradient + //For reference full list of PressTensorGrad + //PressTensorGrad[n+0*Np] = Pxx_x + //PressTensorGrad[n+1*Np] = Pxx_y + //PressTensorGrad[n+2*Np] = Pxx_z + //PressTensorGrad[n+3*Np] = Pyy_x + //PressTensorGrad[n+4*Np] = Pyy_y + //PressTensorGrad[n+5*Np] = Pyy_z + //PressTensorGrad[n+6*Np] = Pzz_x + //PressTensorGrad[n+7*Np] = Pzz_y + //PressTensorGrad[n+8*Np] = Pzz_z + //PressTensorGrad[n+9*Np] = Pxy_x + //PressTensorGrad[n+10*Np] = Pxy_y + //PressTensorGrad[n+11*Np] = Pxy_z + //PressTensorGrad[n+12*Np] = Pyz_x + //PressTensorGrad[n+13*Np] = Pyz_y + //PressTensorGrad[n+14*Np] = Pyz_z + //PressTensorGrad[n+15*Np] = Pxz_x + //PressTensorGrad[n+16*Np] = Pxz_y + //PressTensorGrad[n+17*Np] = Pxz_z + Pxx_x = PressTensorGrad[0*Np+n]; + Pyy_y = PressTensorGrad[4*Np+n]; + Pzz_z = PressTensorGrad[8*Np+n]; + Pxy_x = PressTensorGrad[9*Np+n]; + Pxz_x = PressTensorGrad[15*Np+n]; + Pxy_y = PressTensorGrad[10*Np+n]; + Pyz_y = PressTensorGrad[13*Np+n]; + Pyz_z = PressTensorGrad[14*Np+n]; + Pxz_z = PressTensorGrad[17*Np+n]; + //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... + //TODO double check if you need porosity as a fre-factor + Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); + Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); + Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; // local density rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); @@ -2258,20 +2384,6 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou m18 -= fq; //---------------------------------------------------------------------// - //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// - // fluid-fluid force - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); @@ -2464,10 +2576,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //-----------------------Mass transport------------------------// // calcuale chemical potential - chem_a = lambda*(nA*nA*nA-1.5*nA*nA+0.5*nA)-kappa*nA_lap; - chem_b = lambda*(nB*nB*nB-1.5*nB*nB+0.5*nB)-kappa*nB_lap; - nA_sum_tmp = 0.0; - nB_sum_tmp = 0.0; + chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; + chem_b = lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; rlx_massA = 3.f-sqrt(3.f); rlx_massB = 3.f-sqrt(3.f); @@ -2478,12 +2588,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou b1 = Bq[nr2]; a2 = Aq[nr1]; b2 = Bq[nr1]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*ux)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*ux)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*ux)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*ux)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); // q = 1 //nread = neighborList[n+Np]; @@ -2500,12 +2608,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou b1 = Bq[nr4]; a2 = Aq[nr3]; b2 = Bq[nr3]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uy)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uy)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uy)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uy)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); // q = 3 //nread = neighborList[n+3*Np]; @@ -2523,12 +2629,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou b1 = Bq[nr6]; a2 = Aq[nr5]; b2 = Bq[nr5]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5+nA*4.5*uz)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5+nB*4.5*uz)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*(gamma*chem_a*4.5-nA*4.5*uz)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*(gamma*chem_b*4.5-nB*4.5*uz)); - nA_sum_tmp += a1+a2; - nB_sum_tmp += b1+b2; + a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); + b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); + a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); + b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); // q = 5 //nread = neighborList[n+5*Np]; @@ -2544,8 +2648,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // Stationary value - distribution 0 a1=Aq[n]; b1=Bq[n]; - Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-nA_sum_tmp); - Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-nB_sum_tmp); + Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); + Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); } @@ -2590,7 +2694,7 @@ __global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, d } } -__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ int idx; double nA,nB; @@ -2617,11 +2721,13 @@ __global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, doub Bq[4*Np+idx]=0.1111111111111111*nB; Bq[5*Np+idx]=0.1111111111111111*nB; Bq[6*Np+idx]=0.1111111111111111*nB; + + Phi[idx] = nA-nB; } } } -__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *neighborList, double *Aq, double *Bq, double *Den, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *neighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ int n,nread; double fq,nA,nB; @@ -2682,11 +2788,13 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *neighborList, // save the number densities Den[n] = nA; Den[Np+n] = nB; + // save the phase field + Phi[n] = nA-nB; } } } -__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ int n; double fq,nA,nB; @@ -2741,6 +2849,8 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double // save the number densities Den[n] = nA; Den[Np+n] = nB; + // save the phase field + Phi[n] = nA-nB; } } } @@ -2863,6 +2973,115 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, dou } } +__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi, double *PressTensor, double *PhiLap, + double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ + //**GreyscaleColor model related parameters: + //kappaA, kappaB: characterize interfacial tension + //lambdaA, lambdaB: characterize bulk free energy + //nA: concentration of liquid 1; + //nB: concentration of liquid 2; + //nA = 0.5*(1+phi/chi) + //nB = 0.5*(1-phi/chi) + //nA+nB=1 + //chi: a scaling factor, is set to 1.0 for now. + + int nn,n; + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double nx,ny,nz;//Color gradient + double nA,nB;//ELBM parameters: concentration of liquid 1 and 2 + double phi;//phase field + double pb;//thermodynamic bulk fluid pressure + double Lphi;//Laplacian of phase field + double C;//squared magnitude of the gradient of phase field + double chi = 1.0;//legacy ELBM parameter, scale the phase field; may be useful in the future; + double kappa = 0.25*(kappaA+kappaB)/(chi*chi);//the effective surface tension coefficient + double Pxx,Pyy,Pzz,Pxy,Pyz,Pxz;//Pressure tensor + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist, Aq, Bq, Den, DenGradA, DenGradB,DenLapA,DenLapB, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Aq, Bq, Den, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2905,13 +3124,14 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *DenLapA, double *DenLapB,double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappa,double lambda,double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure){ +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB,DenLapA,DenLapB, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, SolidForce, start, finish, Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, + Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2920,8 +3140,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double } -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, start, finish, Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); @@ -2936,9 +3156,9 @@ extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, doubl } } -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity<<>>(NeighborList, Aq, Bq, Den, start, finish, Np); + dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity<<>>(NeighborList, Aq, Bq, Den, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2946,9 +3166,9 @@ extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, doubl } } -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity<<>>(Aq, Bq, Den, start, finish, Np); + dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity<<>>(Aq, Bq, Den, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorDensity: %s \n",cudaGetErrorString(err)); @@ -2971,3 +3191,22 @@ extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Laplacian: %s \n",cudaGetErrorString(err)); } } + +extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, + double *Pressure, double rhoA,double rhoB, int Np){ + + dvc_ScaLBL_D3Q19_GreyscaleColor_Pressure<<>>(dist, Den, Porosity, Velocity, Pressure, rhoA, rhoB, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Pressure: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi, double *PressTensor, double *PhiLap, + double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ + dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor<<>>(neighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_PressureTensor: %s \n",cudaGetErrorString(err)); + } +} diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 4b1164e1..8360e2dc 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -14,7 +14,8 @@ void DeleteArray( const TYPE *p ) } ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),Gsc(0),gamma(0),kappa(0),lambda(0), +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0), +rhoA(0),rhoB(0),gamma(0),kappaA(0),kappaB(0),lambdaA(0),lambdaB(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -48,10 +49,12 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel - Gsc = 1.0; - gamma = 1.0; - kappa = 1.0; - lambda = 1.0; + //Gsc = 1.0; + gamma = 1.0;//may also have gammaA and gammaB; + kappaA = 1.0e-3; + kappaB = 1.0e-3; + lambdaA = 1.0e-3; + lambdaB = 1.0e-3; // ---------------------- Greyscale Model parameters -----------------------// if (greyscaleColor_db->keyExists( "timestepMax" )){ @@ -71,17 +74,23 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ if (greyscaleColor_db->keyExists( "rhoB" )){ rhoB = greyscaleColor_db->getScalar( "rhoB" ); } - if (greyscaleColor_db->keyExists( "Gsc" )){ - Gsc = greyscaleColor_db->getScalar( "Gsc" ); - } +// if (greyscaleColor_db->keyExists( "Gsc" )){ +// Gsc = greyscaleColor_db->getScalar( "Gsc" ); +// } if (greyscaleColor_db->keyExists( "gamma" )){ gamma = greyscaleColor_db->getScalar( "gamma" ); } - if (greyscaleColor_db->keyExists( "kappa" )){ - kappa = greyscaleColor_db->getScalar( "kappa" ); + if (greyscaleColor_db->keyExists( "kappaA" )){ + kappaA = greyscaleColor_db->getScalar( "kappaA" ); } - if (greyscaleColor_db->keyExists( "lambda" )){ - lambda = greyscaleColor_db->getScalar( "lambda" ); + if (greyscaleColor_db->keyExists( "kappaB" )){ + kappaB = greyscaleColor_db->getScalar( "kappaB" ); + } + if (greyscaleColor_db->keyExists( "lambdaA" )){ + lambdaA = greyscaleColor_db->getScalar( "lambdaA" ); + } + if (greyscaleColor_db->keyExists( "lambdaB" )){ + lambdaB = greyscaleColor_db->getScalar( "lambdaB" ); } if (greyscaleColor_db->keyExists( "dp" )){ dp = greyscaleColor_db->getScalar( "dp" ); @@ -550,15 +559,20 @@ void ScaLBL_GreyscaleColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &PressureGrad, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &PhiLap, sizeof(double)*Np);//laplacian of phase field ScaLBL_AllocateDeviceMemory((void **) &SolidForce, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenLapA, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenLapB, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &PressTensor, 6*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &PressTensorGrad, 18*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &DenLapA, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &DenLapB, sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device neighbor list \n"); @@ -608,17 +622,30 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + //TODO need to initialize velocity field ! + //this is required for calculating the pressure_dvc + //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity } else{ if (rank==0) printf ("Initializing density field \n"); DensityField_Init();//initialize density field - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); + + //Velocity also needs initialization + if (rank==0) printf ("Initializing velocity field \n"); + double *vel_init; + vel_init = new double [3*Np]; + for (int i=0;i<3*Np;i++) vel_init[i]=0.0; + ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] vel_init; } } @@ -669,40 +696,77 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - //compute Den gradients - component A - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); - //compute Den gradients - component B - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + // Update local pressure + ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + // Compute pressure gradient + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Pressure_dvc); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad);//TODO need to fix RecvGrad missing send function - see ScaLBL.cpp + ScaLBL_DeviceBarrier(); + // Compute Pressure Tensor + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + ScaLBL_Comm->RecvHalo(Phi); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + /* Compute gradient of the pressure tensor */ + // call the recv Grad function once per tensor element + // 1st tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); + // 2nd tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); + // 3rd tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); + // 4th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); + // 5th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); + // 6th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); - //compute Den laplacian - component A - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0*Np],DenLapA); +// //compute Den gradients - component A +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_Comm->SendHalo(&Den[0*Np]); +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); +// ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); +// //compute Den gradients - component B +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_Comm->SendHalo(&Den[1*Np]); +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); +// ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); - //compute Den laplacian - component B - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[1*Np],DenLapB); ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL // ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, + Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -713,8 +777,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ // } // ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, SolidForce, + 0, ScaLBL_Comm->LastExterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, + Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -723,40 +789,76 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - //compute Den gradients - component A - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); - //compute Den gradients - component B - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); + // Update local pressure + ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + // Compute pressure gradient + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Pressure_dvc); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad);//TODO need to fix RecvGrad missing send function - see ScaLBL.cpp + ScaLBL_DeviceBarrier(); + // Compute Pressure Tensor + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + ScaLBL_Comm->RecvHalo(Phi); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + /* Compute gradient of the pressure tensor */ + // call the recv Grad function once per tensor element + // 1st tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); + // 2nd tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); + // 3rd tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); + // 4th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); + // 5th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); + // 6th tensor element + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); - //compute Den laplacian - component A - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[0*Np], DenLapA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0*Np],DenLapA); - - //compute Den laplacian - component B - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, &Den[1*Np], DenLapB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[1*Np],DenLapB); +// //compute Den gradients - component A +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_Comm->SendHalo(&Den[0*Np]); +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); +// ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); +// //compute Den gradients - component B +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_Comm->SendHalo(&Den[1*Np]); +// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); +// ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL // ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, SolidForce, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, + Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -767,8 +869,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ // } // ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, DenGradA, DenGradB, DenLapA,DenLapB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappa,lambda, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc); + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, SolidForce, + 0, ScaLBL_Comm->LastExterior(), Np, + tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, + Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 85bbb56f..374920d0 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -43,9 +43,10 @@ public: double din,dout; double dp;//solid particle diameter, unit in voxel double GreyPorosity; - double Gsc; + //double Gsc; double gamma; - double kappa,lambda; + double kappaA,kappaB; + double lambdaA,lambdaB; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -69,12 +70,17 @@ public: double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; - double *Pressure_dvc; double *SolidForce; - double *DenGradA; - double *DenGradB; - double *DenLapA; - double *DenLapB; + double *Pressure_dvc; + double *PressureGrad;// gradiant of pressure + double *PressTensor;//pressure tensor + double *PressTensorGrad;// gradient of pressure tensor + double *Phi; + double *PhiLap;//laplacian of phase field phi +// double *DenGradA; +// double *DenGradB; +// double *DenLapA; +// double *DenLapB; IntArray Map; DoubleArray SignDist; DoubleArray Velocity_x; From 67f5076fa39e9170c67bf7319f262373a8177cee Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 17 Apr 2020 16:32:09 -0400 Subject: [PATCH 111/270] fix bug in recvGrad --- common/ScaLBL.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6f2966e7..4c187f0e 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1068,6 +1068,7 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ ScaLBL_Gradient_Unpack(1.0,-1,0,0,dvcRecvDist_x,0,recvCount_x,recvbuf_x,phi,grad,N); ScaLBL_Gradient_Unpack(0.5,-1,-1,0,dvcRecvDist_x,recvCount_x,recvCount_x,recvbuf_x,phi,grad,N); ScaLBL_Gradient_Unpack(0.5,-1,1,0,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N); + ScaLBL_Gradient_Unpack(0.5,-1,0,-1,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N); ScaLBL_Gradient_Unpack(0.5,-1,0,1,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N); //................................................................................... //...Packing for X face(1,7,9,11,13)................................ From 8dc9aed0abc0ca518a902f61dc38d60445e93e96 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 17 Apr 2020 17:55:00 -0400 Subject: [PATCH 112/270] fix mass conservation test --- tests/TestMassConservationD3Q7.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/TestMassConservationD3Q7.cpp b/tests/TestMassConservationD3Q7.cpp index 6186fd60..17c50d19 100644 --- a/tests/TestMassConservationD3Q7.cpp +++ b/tests/TestMassConservationD3Q7.cpp @@ -202,7 +202,7 @@ int main(int argc, char **argv) printf("Global mass difference B = %.5g\n",total_mass_B_1-total_mass_B_0); if (count_negative_A > 0 ||count_negative_B > 0) CleanCheck=1; - if (fabs(total_mass_A_1-total_mass_A_0) > 1.0e-13||fabs(total_mass_B_1-total_mass_B_0) > 1.0e-13 ) CleanCheck=2; + if (fabs(total_mass_A_1-total_mass_A_0) > 1.0e-8 || fabs(total_mass_B_1-total_mass_B_0) > 1.0e-8) CleanCheck=2; FILE *OUTFILE; OUTFILE = fopen("error.raw","wb"); From 67896fcbe291d792950545747aeeee4602856e9b Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 17 Apr 2020 18:35:47 -0400 Subject: [PATCH 113/270] remove debug dump --- tests/lbpm_color_simulator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 79b2a718..1f63c653 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -58,7 +58,7 @@ int main(int argc, char **argv) ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables ColorModel.Initialize(); // initializing the model will set initial conditions for variables ColorModel.Run(); - ColorModel.WriteDebug(); + //ColorModel.WriteDebug(); PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); From b495c9916c2be425fe739c899299e5de38fc0674 Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 17 Apr 2020 19:12:08 -0400 Subject: [PATCH 114/270] seed water morphdelta negative by default --- models/ColorModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a0f339c6..189f0059 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -546,18 +546,18 @@ void ScaLBL_ColorModel::Run(){ USE_MORPH = true; } else if (protocol == "seed water"){ - morph_delta = 0.05; + morph_delta = -0.05; seed_water = 0.01; USE_SEED = true; USE_MORPH = true; } else if (protocol == "open connected oil"){ - morph_delta = 0.05; + morph_delta = -0.05; USE_MORPH = true; USE_MORPHOPEN_OIL = true; } else if (protocol == "shell aggregation"){ - morph_delta = 0.05; + morph_delta = -0.05; USE_MORPH = true; } if (color_db->keyExists( "capillary_number" )){ From 4f75ddd89ab7d6b5426bf65c0b646b0617a4727d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 18 Apr 2020 10:56:28 -0400 Subject: [PATCH 115/270] fix a few bugs; but wait to see if that works --- gpu/GreyscaleColor.cu | 24 +++++++++++++----------- models/GreyscaleColorModel.cpp | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index b773ab4a..1f939acc 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -1902,7 +1902,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double //-----------------------Mass transport------------------------// // calcuale chemical potential chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; - chem_b = lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; + chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; rlx_massA = 3.f-sqrt(3.f); rlx_massB = 3.f-sqrt(3.f); @@ -2577,7 +2577,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //-----------------------Mass transport------------------------// // calcuale chemical potential chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; - chem_b = lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; + chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; rlx_massA = 3.f-sqrt(3.f); rlx_massB = 3.f-sqrt(3.f); @@ -2906,9 +2906,9 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, doub m18 = Den[nn]*int(n!=nn); //............Compute the Color Gradient................................... - nx = 0.3333333333333333*1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = 0.3333333333333333*1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = 0.3333333333333333*1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); DenGrad[n] = nx; DenGrad[Np+n] = ny; @@ -2967,7 +2967,9 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, dou nn = neighborList[n+16*Np]%Np; m18 = Den[nn]*int(n!=nn); - lap = 2.0*0.3333333333333333*1.f/18.f*(m1+m2+m3+m4+m5+m6-6*Den[n]+0.5*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18-12*Den[n])); + + + lap = 1.f/3.f*(m1+m2+m3+m4+m5+m6-6*Den[n]+0.5*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18-12*Den[n])); DenLap[n] = lap; } } @@ -3042,21 +3044,21 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborLis m18 = Phi[nn]*int(n!=nn); //............Compute the Color Gradient................................... - nx = 0.3333333333333333*1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = 0.3333333333333333*1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = 0.3333333333333333*1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); C = nx*nx+ny*ny+nz*nz; // Laplacian of phase field //Lphi = 0.3333333333333333*(m1+m2+m3+m4+m5+m6)+ // 0.16666666666666666*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18) - 4.0*phi; phi = Phi[n]; - Lphi = 2.0*0.3333333333333333*1.f/18.f*(m1+m2+m3+m4+m5+m6-6*phi+0.5*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18-12*phi)); + Lphi = 1.f/3.f*(m1+m2+m3+m4+m5+m6-6*phi+0.5*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18-12*phi)); //bulk pressure p_b nA = 0.5*(1.0+phi/chi); nB = 0.5*(1.0-phi/chi); pb = -((1.0-nA)*(1.0-nA)*nA*nA*lambdaA)*0.5 - ((1.0-nB)*(1.0-nB)*nB*nB*lambdaB)*0.5 + - (nA - nB)*chi*(((0.5*nA-1.5*nA*nA+nA*nA*nA)*lambdaA)/chi + ((0.5*nB-1.5*nB*nB+nB*nB*nB)*lambdaB)/chi); + (nA - nB)*chi*(((0.5*nA-1.5*nA*nA+nA*nA*nA)*lambdaA)/chi - ((0.5*nB-1.5*nB*nB+nB*nB*nB)*lambdaB)/chi); //Pressure tensors Pxx=pb-kappa*phi*Lphi-0.5*kappa*C + kappa*nx*nx ; diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 8360e2dc..26c6d783 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -707,7 +707,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad);//TODO need to fix RecvGrad missing send function - see ScaLBL.cpp + ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor ScaLBL_Comm->SendHalo(Phi); @@ -800,7 +800,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad);//TODO need to fix RecvGrad missing send function - see ScaLBL.cpp + ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor ScaLBL_Comm->SendHalo(Phi); From dd3c177deed9112378e2934e138520b433333350 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 18 Apr 2020 22:46:44 -0400 Subject: [PATCH 116/270] change the mass transport formulation;no longer solve Aq and Bq, only solve for phase field --- common/ScaLBL.cpp | 101 ++ common/ScaLBL.h | 23 +- gpu/GreyscaleColor.cu | 2861 ++++++++++++++++++++++++-------- models/GreyscaleColorModel.cpp | 49 +- models/GreyscaleColorModel.h | 3 +- 5 files changed, 2318 insertions(+), 719 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 4c187f0e..e0f3879c 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1135,6 +1135,107 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ } +void ScaLBL_Communicator::SendD3Q7AA(double *Aq){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + if (Lock==true){ + ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + } + else{ + Lock=true; + } + // assign tag of 19 to D3Q19 communication + sendtag = recvtag = 7; + ScaLBL_DeviceBarrier(); + // Pack the distributions + //...Packing for x face(2,8,10,12,14)................................ + ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N); + + req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); + req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); + + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N); + + req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); + req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); + + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N); + + req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); + req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); + + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N); + + req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); + req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); + + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N); + + req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); + req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); + + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N); + + //................................................................................... + // Send all the distributions + req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); + req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); + +} + +void ScaLBL_Communicator::RecvD3Q7AA(double *Aq){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + //................................................................................... + // Wait for completion of D3Q19 communication + MPI_COMM_SCALBL.waitAll(6,req1); + ScaLBL_DeviceBarrier(); + + //................................................................................... + // NOTE: AA Routine writes to opposite + // Unpack the distributions on the device + //................................................................................... + //...Unpacking for x face(2,8,10,12,14)................................ + ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,Aq,N); + //................................................................................... + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,Aq,N); + //................................................................................... + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,Aq,N); + //................................................................................... + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N); + //................................................................................... + + if (BoundaryCondition > 0 && kproc == 0){ + // don't unpack little z + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); + } + else if (BoundaryCondition > 0 && kproc == nprocz-1){ + // don't unpack big z + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); + } + else { + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); + } + + //................................................................................... + Lock=false; // unlock the communicator after communications complete + //................................................................................... + +} + void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 7bee2d1b..d3251740 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -84,17 +84,28 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dis double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np); +//extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *Phi, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); @@ -102,6 +113,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, doubl extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Den, double *Phi, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Den, double *Phi, int start, int finish, int Np); + extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); @@ -223,6 +238,8 @@ public: void RecvD3Q19AA(double *dist); // void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); // void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); + void SendD3Q7AA(double *Aq); + void RecvD3Q7AA(double *Aq); void BiSendD3Q7AA(double *Aq, double *Bq); void BiRecvD3Q7AA(double *Aq, double *Bq); void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 1f939acc..489b8a6b 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -1329,438 +1329,470 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double } } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - int n; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho0; - double phi; - double phi_lap;//laplacian of phase field - double nA,nB; - double a1,b1,a2,b2; - double Gfs_x,Gfs_y,Gfs_z; - double Gff_x,Gff_y,Gff_z; - double chem_a,chem_b; - double rlx_massA,rlx_massB; - // *---------------------------------Pressure Tensor Gradient------------------------------------*// - double Pxx_x,Pyy_y,Pzz_z; - double Pxy_x,Pxy_y; - double Pyz_y,Pyz_z; - double Pxz_x,Pxz_z; - double px,py,pz; //pressure gradient - - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - //Load pressure gradient - px=PressureGrad[0*Np+n]; - py=PressureGrad[1*Np+n]; - pz=PressureGrad[2*Np+n]; - - //Load pressure tensor gradient - //For reference full list of PressTensorGrad - //PressTensorGrad[n+0*Np] = Pxx_x - //PressTensorGrad[n+1*Np] = Pxx_y - //PressTensorGrad[n+2*Np] = Pxx_z - //PressTensorGrad[n+3*Np] = Pyy_x - //PressTensorGrad[n+4*Np] = Pyy_y - //PressTensorGrad[n+5*Np] = Pyy_z - //PressTensorGrad[n+6*Np] = Pzz_x - //PressTensorGrad[n+7*Np] = Pzz_y - //PressTensorGrad[n+8*Np] = Pzz_z - //PressTensorGrad[n+9*Np] = Pxy_x - //PressTensorGrad[n+10*Np] = Pxy_y - //PressTensorGrad[n+11*Np] = Pxy_z - //PressTensorGrad[n+12*Np] = Pyz_x - //PressTensorGrad[n+13*Np] = Pyz_y - //PressTensorGrad[n+14*Np] = Pyz_z - //PressTensorGrad[n+15*Np] = Pxz_x - //PressTensorGrad[n+16*Np] = Pxz_y - //PressTensorGrad[n+17*Np] = Pxz_z - Pxx_x = PressTensorGrad[0*Np+n]; - Pyy_y = PressTensorGrad[4*Np+n]; - Pzz_z = PressTensorGrad[8*Np+n]; - Pxy_x = PressTensorGrad[9*Np+n]; - Pxz_x = PressTensorGrad[15*Np+n]; - Pxy_y = PressTensorGrad[10*Np+n]; - Pyz_y = PressTensorGrad[13*Np+n]; - Pyz_z = PressTensorGrad[14*Np+n]; - Pxz_z = PressTensorGrad[17*Np+n]; - //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... - //TODO double check if you need porosity as a fre-factor - Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); - Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); - Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - // local density - rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - // local relaxation time - tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); - rlx_setA = 1.f/tau; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); - mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity - - - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = dist[n]; - m1 = -30.0*fq; - m2 = 12.0*fq; - - // q=1 - fq = dist[2*Np+n]; - pressure = fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // f2 = dist[10*Np+n]; - fq = dist[1*Np+n]; - pressure += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - fq = dist[4*Np+n]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - fq = dist[3*Np+n]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - fq = dist[6*Np+n]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - fq = dist[5*Np+n]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - fq = dist[8*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - fq = dist[7*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - fq = dist[10*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - fq = dist[9*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - fq = dist[12*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - fq = dist[11*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - fq = dist[14*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - fq = dist[13*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - fq = dist[16*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - fq = dist[15*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - fq = dist[18*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - fq = dist[17*Np+n]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - +//__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ +// int n; +// double vx,vy,vz,v_mag; +// double ux,uy,uz,u_mag; +// double pressure;//defined for this incompressible model +// // conserved momemnts +// double jx,jy,jz; +// // non-conserved moments +// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; +// double fq; +// // currently disable 'GeoFun' +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double c0, c1; //Guo's model parameters +// double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) +// double tau,tau_eff,rlx_setA,rlx_setB; +// double mu_eff;//effective kinematic viscosity for Darcy term +// double rho0; +// double phi; +// double phi_lap;//laplacian of phase field +// double nA,nB; +// double a1,b1,a2,b2; +// double Gfs_x,Gfs_y,Gfs_z; +// double Gff_x,Gff_y,Gff_z; +// double chem_a,chem_b; +// double rlx_massA,rlx_massB; +// // *---------------------------------Pressure Tensor Gradient------------------------------------*// +// double Pxx_x,Pyy_y,Pzz_z; +// double Pxy_x,Pxy_y; +// double Pyz_y,Pyz_z; +// double Pxz_x,Pxz_z; +// double px,py,pz; //pressure gradient +// +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); +// +// //Load pressure gradient +// px=PressureGrad[0*Np+n]; +// py=PressureGrad[1*Np+n]; +// pz=PressureGrad[2*Np+n]; +// +// //Load pressure tensor gradient +// //For reference full list of PressTensorGrad +// //PressTensorGrad[n+0*Np] = Pxx_x +// //PressTensorGrad[n+1*Np] = Pxx_y +// //PressTensorGrad[n+2*Np] = Pxx_z +// //PressTensorGrad[n+3*Np] = Pyy_x +// //PressTensorGrad[n+4*Np] = Pyy_y +// //PressTensorGrad[n+5*Np] = Pyy_z +// //PressTensorGrad[n+6*Np] = Pzz_x +// //PressTensorGrad[n+7*Np] = Pzz_y +// //PressTensorGrad[n+8*Np] = Pzz_z +// //PressTensorGrad[n+9*Np] = Pxy_x +// //PressTensorGrad[n+10*Np] = Pxy_y +// //PressTensorGrad[n+11*Np] = Pxy_z +// //PressTensorGrad[n+12*Np] = Pyz_x +// //PressTensorGrad[n+13*Np] = Pyz_y +// //PressTensorGrad[n+14*Np] = Pyz_z +// //PressTensorGrad[n+15*Np] = Pxz_x +// //PressTensorGrad[n+16*Np] = Pxz_y +// //PressTensorGrad[n+17*Np] = Pxz_z +// Pxx_x = PressTensorGrad[0*Np+n]; +// Pyy_y = PressTensorGrad[4*Np+n]; +// Pzz_z = PressTensorGrad[8*Np+n]; +// Pxy_x = PressTensorGrad[9*Np+n]; +// Pxz_x = PressTensorGrad[15*Np+n]; +// Pxy_y = PressTensorGrad[10*Np+n]; +// Pyz_y = PressTensorGrad[13*Np+n]; +// Pyz_z = PressTensorGrad[14*Np+n]; +// Pxz_z = PressTensorGrad[17*Np+n]; +// //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... +// //TODO double check if you need porosity as a fre-factor +// Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); +// Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); +// Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); +// // fluid-solid force +// Gfs_x = (nA-nB)*SolidForce[n+0*Np]; +// Gfs_y = (nA-nB)*SolidForce[n+1*Np]; +// Gfs_z = (nA-nB)*SolidForce[n+2*Np]; +// +// // local density +// rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); +// // local relaxation time +// tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); +// rlx_setA = 1.f/tau; +// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); +// tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); +// mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity +// +// +// //........................................................................ +// // READ THE DISTRIBUTIONS +// // (read from opposite array due to previous swap operation) +// //........................................................................ +// // q=0 +// fq = dist[n]; +// m1 = -30.0*fq; +// m2 = 12.0*fq; +// +// // q=1 +// fq = dist[2*Np+n]; +// pressure = fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jx = fq; +// m4 = -4.0*fq; +// m9 = 2.0*fq; +// m10 = -4.0*fq; +// +// // f2 = dist[10*Np+n]; +// fq = dist[1*Np+n]; +// pressure += fq; +// m1 -= 11.0*(fq); +// m2 -= 4.0*(fq); +// jx -= fq; +// m4 += 4.0*(fq); +// m9 += 2.0*(fq); +// m10 -= 4.0*(fq); +// +// // q=3 +// fq = dist[4*Np+n]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy = fq; +// m6 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 = fq; +// m12 = -2.0*fq; +// +// // q = 4 +// fq = dist[3*Np+n]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy -= fq; +// m6 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 += fq; +// m12 -= 2.0*fq; +// +// // q=5 +// fq = dist[6*Np+n]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz = fq; +// m8 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q = 6 +// fq = dist[5*Np+n]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz -= fq; +// m8 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q=7 +// fq = dist[8*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 = fq; +// m16 = fq; +// m17 = -fq; +// +// // q = 8 +// fq = dist[7*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 += fq; +// m16 -= fq; +// m17 += fq; +// +// // q=9 +// fq = dist[10*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 += fq; +// m17 += fq; +// +// // q = 10 +// fq = dist[9*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 -= fq; +// m17 -= fq; +// +// // q=11 +// fq = dist[12*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 = fq; +// m16 -= fq; +// m18 = fq; +// +// // q=12 +// fq = dist[11*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 += fq; +// m16 += fq; +// m18 -= fq; +// +// // q=13 +// fq = dist[14*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 -= fq; +// m18 -= fq; +// +// // q=14 +// fq = dist[13*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 += fq; +// m18 += fq; +// +// // q=15 +// fq = dist[16*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 = fq; +// m17 += fq; +// m18 -= fq; +// +// // q=16 +// fq = dist[15*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 += fq; +// m17 -= fq; +// m18 += fq; +// +// // q=17 +// fq = dist[18*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 += fq; +// m18 += fq; +// +// // q=18 +// fq = dist[17*Np+n]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 -= fq; +// m18 -= fq; +// //---------------------------------------------------------------------// +// +// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(perm); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); +// vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); +// vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux*ux+uy*uy+uz*uz); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); +// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); +// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); +// if (porosity==1.0){ +// Fx=rho0*(Gx + Gff_x + Gfs_x); +// Fy=rho0*(Gy + Gff_y + Gfs_y); +// Fz=rho0*(Gz + Gff_z + Gfs_z); +// } +// +// //Calculate pressure for Incompressible-MRT model +// pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); +// +//// //..............carry out relaxation process............................................... +//// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +//// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +//// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +//// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +//// jx = jx + Fx; +//// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +//// jy = jy + Fy; +//// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +//// jz = jz + Fz; +//// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +//// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +//// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +//// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +//// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +//// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +//// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +//// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +//// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +//// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +//// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +//// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +//// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +//// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +//// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +//// m16 = m16 + rlx_setB*( - m16); +//// m17 = m17 + rlx_setB*( - m17); +//// m18 = m18 + rlx_setB*( - m18); +//// //....................................................................................................... +// +// //-------------------- IMRT collison where body force has NO higher-order terms -------------// // //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); // jx = jx + Fx; // m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -1770,206 +1802,861 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double // jz = jz + Fz; // m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); // m16 = m16 + rlx_setB*( - m16); // m17 = m17 + rlx_setB*( - m17); // m18 = m18 + rlx_setB*( - m18); // //....................................................................................................... +// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; +// dist[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); +// dist[1*Np+n] = fq; +// +// // q=2 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); +// dist[2*Np+n] = fq; +// +// // q = 3 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// dist[3*Np+n] = fq; +// +// // q = 4 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// dist[4*Np+n] = fq; +// +// // q = 5 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// dist[5*Np+n] = fq; +// +// // q = 6 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// dist[6*Np+n] = fq; +// +// // q = 7 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); +// dist[7*Np+n] = fq; +// +// // q = 8 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); +// dist[8*Np+n] = fq; +// +// // q = 9 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); +// dist[9*Np+n] = fq; +// +// // q = 10 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); +// dist[10*Np+n] = fq; +// +// // q = 11 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); +// dist[11*Np+n] = fq; +// +// // q = 12 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); +// dist[12*Np+n] = fq; +// +// // q = 13 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); +// dist[13*Np+n] = fq; +// +// // q= 14 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); +// dist[14*Np+n] = fq; +// +// // q = 15 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); +// dist[15*Np+n] = fq; +// +// // q = 16 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); +// dist[16*Np+n] = fq; +// +// // q = 17 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); +// dist[17*Np+n] = fq; +// +// // q = 18 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); +// dist[18*Np+n] = fq; +// //........................................................................ +// +// //Update velocity on device +// Velocity[0*Np+n] = ux; +// Velocity[1*Np+n] = uy; +// Velocity[2*Np+n] = uz; +// //Update pressure on device +// Pressure[n] = pressure; +// +// //-----------------------Mass transport------------------------// +// // calcuale chemical potential +// chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; +// chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; +// rlx_massA = 3.f-sqrt(3.f); +// rlx_massB = 3.f-sqrt(3.f); +// +// //............................................... +// // q = 0,2,4 +// // Cq = {1,0,0}, {0,1,0}, {0,0,1} +// a1 = Aq[1*Np+n]; +// b1 = Bq[1*Np+n]; +// a2 = Aq[2*Np+n]; +// b2 = Bq[2*Np+n]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); +// +// Aq[1*Np+n] = a1; +// Bq[1*Np+n] = b1; +// Aq[2*Np+n] = a2; +// Bq[2*Np+n] = b2; +// +// //............................................... +// // q = 2 +// // Cq = {0,1,0} +// a1 = Aq[3*Np+n]; +// b1 = Bq[3*Np+n]; +// a2 = Aq[4*Np+n]; +// b2 = Bq[4*Np+n]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); +// +// Aq[3*Np+n] = a1; +// Bq[3*Np+n] = b1; +// Aq[4*Np+n] = a2; +// Bq[4*Np+n] = b2; +// //............................................... +// // q = 4 +// // Cq = {0,0,1} +// a1 = Aq[5*Np+n]; +// b1 = Bq[5*Np+n]; +// a2 = Aq[6*Np+n]; +// b2 = Bq[6*Np+n]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); +// +// Aq[5*Np+n] = a1; +// Bq[5*Np+n] = b1; +// Aq[6*Np+n] = a2; +// Bq[6*Np+n] = b2; +// //............................................... +// +// // Instantiate mass transport distributions +// // Stationary value - distribution 0 +// a1=Aq[n]; +// b1=Bq[n]; +// Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); +// Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); +// +// +// } +// } +//} - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... +//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ +// +// int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; +// double vx,vy,vz,v_mag; +// double ux,uy,uz,u_mag; +// double pressure;//defined for this incompressible model +// // conserved momemnts +// double jx,jy,jz; +// // non-conserved moments +// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; +// double fq; +// // currently disable 'GeoFun' +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double c0, c1; //Guo's model parameters +// double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) +// double tau,tau_eff,rlx_setA,rlx_setB; +// double mu_eff;//effective kinematic viscosity for Darcy term +// double rho0; +// double phi; +// double phi_lap;//laplacian of phase field +// double nA,nB; +// double a1,b1,a2,b2; +// double Gfs_x,Gfs_y,Gfs_z; +// double Gff_x,Gff_y,Gff_z; +// double chem_a,chem_b; +// double rlx_massA,rlx_massB; +// // *---------------------------------Pressure Tensor Gradient------------------------------------*// +// double Pxx_x,Pyy_y,Pzz_z; +// double Pxy_x,Pxy_y; +// double Pyz_y,Pyz_z; +// double Pxz_x,Pxz_z; +// double px,py,pz; //pressure gradient +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); +// +// //Load pressure gradient +// px=PressureGrad[0*Np+n]; +// py=PressureGrad[1*Np+n]; +// pz=PressureGrad[2*Np+n]; +// +// //Load pressure tensor gradient +// //For reference full list of PressTensorGrad +// //PressTensorGrad[n+0*Np] = Pxx_x +// //PressTensorGrad[n+1*Np] = Pxx_y +// //PressTensorGrad[n+2*Np] = Pxx_z +// //PressTensorGrad[n+3*Np] = Pyy_x +// //PressTensorGrad[n+4*Np] = Pyy_y +// //PressTensorGrad[n+5*Np] = Pyy_z +// //PressTensorGrad[n+6*Np] = Pzz_x +// //PressTensorGrad[n+7*Np] = Pzz_y +// //PressTensorGrad[n+8*Np] = Pzz_z +// //PressTensorGrad[n+9*Np] = Pxy_x +// //PressTensorGrad[n+10*Np] = Pxy_y +// //PressTensorGrad[n+11*Np] = Pxy_z +// //PressTensorGrad[n+12*Np] = Pyz_x +// //PressTensorGrad[n+13*Np] = Pyz_y +// //PressTensorGrad[n+14*Np] = Pyz_z +// //PressTensorGrad[n+15*Np] = Pxz_x +// //PressTensorGrad[n+16*Np] = Pxz_y +// //PressTensorGrad[n+17*Np] = Pxz_z +// Pxx_x = PressTensorGrad[0*Np+n]; +// Pyy_y = PressTensorGrad[4*Np+n]; +// Pzz_z = PressTensorGrad[8*Np+n]; +// Pxy_x = PressTensorGrad[9*Np+n]; +// Pxz_x = PressTensorGrad[15*Np+n]; +// Pxy_y = PressTensorGrad[10*Np+n]; +// Pyz_y = PressTensorGrad[13*Np+n]; +// Pyz_z = PressTensorGrad[14*Np+n]; +// Pxz_z = PressTensorGrad[17*Np+n]; +// //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... +// //TODO double check if you need porosity as a fre-factor +// Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); +// Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); +// Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); +// // fluid-solid force +// Gfs_x = (nA-nB)*SolidForce[n+0*Np]; +// Gfs_y = (nA-nB)*SolidForce[n+1*Np]; +// Gfs_z = (nA-nB)*SolidForce[n+2*Np]; +// +// // local density +// rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); +// // local relaxation time +// tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); +// rlx_setA = 1.f/tau; +// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); +// tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); +// mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity +// +// //........................................................................ +// // READ THE DISTRIBUTIONS +// // (read from opposite array due to previous swap operation) +// //........................................................................ +// // q=0 +// fq = dist[n]; +// m1 = -30.0*fq; +// m2 = 12.0*fq; +// +// // q=1 +// nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist) +// fq = dist[nr1]; // reading the f1 data into register fq +// pressure = fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jx = fq; +// m4 = -4.0*fq; +// m9 = 2.0*fq; +// m10 = -4.0*fq; +// +// // q=2 +// nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) +// fq = dist[nr2]; // reading the f2 data into register fq +// pressure += fq; +// m1 -= 11.0*(fq); +// m2 -= 4.0*(fq); +// jx -= fq; +// m4 += 4.0*(fq); +// m9 += 2.0*(fq); +// m10 -= 4.0*(fq); +// +// // q=3 +// nr3 = neighborList[n+2*Np]; // neighbor 4 +// fq = dist[nr3]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy = fq; +// m6 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 = fq; +// m12 = -2.0*fq; +// +// // q = 4 +// nr4 = neighborList[n+3*Np]; // neighbor 3 +// fq = dist[nr4]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy -= fq; +// m6 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 += fq; +// m12 -= 2.0*fq; +// +// // q=5 +// nr5 = neighborList[n+4*Np]; +// fq = dist[nr5]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz = fq; +// m8 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q = 6 +// nr6 = neighborList[n+5*Np]; +// fq = dist[nr6]; +// pressure += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz -= fq; +// m8 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q=7 +// nread = neighborList[n+6*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 = fq; +// m16 = fq; +// m17 = -fq; +// +// // q = 8 +// nread = neighborList[n+7*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 += fq; +// m16 -= fq; +// m17 += fq; +// +// // q=9 +// nread = neighborList[n+8*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 += fq; +// m17 += fq; +// +// // q = 10 +// nread = neighborList[n+9*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 -= fq; +// m17 -= fq; +// +// // q=11 +// nread = neighborList[n+10*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 = fq; +// m16 -= fq; +// m18 = fq; +// +// // q=12 +// nread = neighborList[n+11*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 += fq; +// m16 += fq; +// m18 -= fq; +// +// // q=13 +// nread = neighborList[n+12*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 -= fq; +// m18 -= fq; +// +// // q=14 +// nread = neighborList[n+13*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 += fq; +// m18 += fq; +// +// // q=15 +// nread = neighborList[n+14*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 = fq; +// m17 += fq; +// m18 -= fq; +// +// // q=16 +// nread = neighborList[n+15*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 += fq; +// m17 -= fq; +// m18 += fq; +// +// // q=17 +// nread = neighborList[n+16*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 += fq; +// m18 += fq; +// +// // q=18 +// nread = neighborList[n+17*Np]; +// fq = dist[nread]; +// pressure += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 -= fq; +// m18 -= fq; +// //---------------------------------------------------------------------// +// +// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(perm); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); +// vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); +// vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux*ux+uy*uy+uz*uz); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); +// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); +// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); +// if (porosity==1.0){ +// Fx=rho0*(Gx + Gff_x + Gfs_x); +// Fy=rho0*(Gy + Gff_y + Gfs_y); +// Fz=rho0*(Gz + Gff_z + Gfs_z); +// } +// +// //Calculate pressure for Incompressible-MRT model +// pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); +// +//// //..............carry out relaxation process............................................... +//// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +//// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +//// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +//// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +//// jx = jx + Fx; +//// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +//// jy = jy + Fy; +//// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +//// jz = jz + Fz; +//// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +//// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +//// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +//// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +//// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +//// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +//// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +//// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +//// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +//// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +//// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +//// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +//// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +//// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +//// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +//// m16 = m16 + rlx_setB*( - m16); +//// m17 = m17 + rlx_setB*( - m17); +//// m18 = m18 + rlx_setB*( - m18); +//// //....................................................................................................... +// +// //-------------------- IMRT collison where body force has NO higher-order terms -------------// +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... +// +// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; +// dist[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); +// //nread = neighborList[n+Np]; +// dist[nr2] = fq; +// +// // q=2 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); +// //nread = neighborList[n]; +// dist[nr1] = fq; +// +// // q = 3 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// //nread = neighborList[n+3*Np]; +// dist[nr4] = fq; +// +// // q = 4 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// //nread = neighborList[n+2*Np]; +// dist[nr3] = fq; +// +// // q = 5 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// //nread = neighborList[n+5*Np]; +// dist[nr6] = fq; +// +// // q = 6 +// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// //nread = neighborList[n+4*Np]; +// dist[nr5] = fq; +// +// // q = 7 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); +// nread = neighborList[n+7*Np]; +// dist[nread] = fq; +// +// // q = 8 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); +// nread = neighborList[n+6*Np]; +// dist[nread] = fq; +// +// // q = 9 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); +// nread = neighborList[n+9*Np]; +// dist[nread] = fq; +// +// // q = 10 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); +// nread = neighborList[n+8*Np]; +// dist[nread] = fq; +// +// // q = 11 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); +// nread = neighborList[n+11*Np]; +// dist[nread] = fq; +// +// // q = 12 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); +// nread = neighborList[n+10*Np]; +// dist[nread]= fq; +// +// // q = 13 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); +// nread = neighborList[n+13*Np]; +// dist[nread] = fq; +// +// // q= 14 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); +// nread = neighborList[n+12*Np]; +// dist[nread] = fq; +// +// // q = 15 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); +// nread = neighborList[n+15*Np]; +// dist[nread] = fq; +// +// // q = 16 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); +// nread = neighborList[n+14*Np]; +// dist[nread] = fq; +// +// // q = 17 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); +// nread = neighborList[n+17*Np]; +// dist[nread] = fq; +// +// // q = 18 +// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); +// nread = neighborList[n+16*Np]; +// dist[nread] = fq; +// //........................................................................ +// +// //Update velocity on device +// Velocity[0*Np+n] = ux; +// Velocity[1*Np+n] = uy; +// Velocity[2*Np+n] = uz; +// //Update pressure on device +// Pressure[n] = pressure; +// +// //-----------------------Mass transport------------------------// +// // calcuale chemical potential +// chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; +// chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; +// rlx_massA = 3.f-sqrt(3.f); +// rlx_massB = 3.f-sqrt(3.f); +// +// //............................................... +// // q = 0,2,4 +// // Cq = {1,0,0}, {0,1,0}, {0,0,1} +// a1 = Aq[nr2]; +// b1 = Bq[nr2]; +// a2 = Aq[nr1]; +// b2 = Bq[nr1]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); +// +// // q = 1 +// //nread = neighborList[n+Np]; +// Aq[nr2] = a1; +// Bq[nr2] = b1; +// // q=2 +// //nread = neighborList[n]; +// Aq[nr1] = a2; +// Bq[nr1] = b2; +// +// //............................................... +// // Cq = {0,1,0} +// a1 = Aq[nr4]; +// b1 = Bq[nr4]; +// a2 = Aq[nr3]; +// b2 = Bq[nr3]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); +// +// // q = 3 +// //nread = neighborList[n+3*Np]; +// Aq[nr4] = a1; +// Bq[nr4] = b1; +// // q = 4 +// //nread = neighborList[n+2*Np]; +// Aq[nr3] = a2; +// Bq[nr3] = b2; +// +// //............................................... +// // q = 4 +// // Cq = {0,0,1} +// a1 = Aq[nr6]; +// b1 = Bq[nr6]; +// a2 = Aq[nr5]; +// b2 = Bq[nr5]; +// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); +// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); +// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); +// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); +// +// // q = 5 +// //nread = neighborList[n+5*Np]; +// Aq[nr6] = a1; +// Bq[nr6] = b1; +// // q = 6 +// //nread = neighborList[n+4*Np]; +// Aq[nr5] = a2; +// Bq[nr5] = b2; +// //............................................... +// +// // Instantiate mass transport distributions +// // Stationary value - distribution 0 +// a1=Aq[n]; +// b1=Bq[n]; +// Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); +// Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); +// +// +// } +// } +//} - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - dist[1*Np+n] = fq; - - // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - dist[2*Np+n] = fq; - - // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[3*Np+n] = fq; - - // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[4*Np+n] = fq; - - // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[5*Np+n] = fq; - - // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[6*Np+n] = fq; - - // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - dist[7*Np+n] = fq; - - // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - dist[8*Np+n] = fq; - - // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - dist[9*Np+n] = fq; - - // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - dist[10*Np+n] = fq; - - // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - dist[11*Np+n] = fq; - - // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - dist[12*Np+n] = fq; - - // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - dist[13*Np+n] = fq; - - // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - dist[14*Np+n] = fq; - - // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - dist[15*Np+n] = fq; - - // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - dist[16*Np+n] = fq; - - // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - dist[17*Np+n] = fq; - - // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - dist[18*Np+n] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // calcuale chemical potential - chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; - chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; - rlx_massA = 3.f-sqrt(3.f); - rlx_massB = 3.f-sqrt(3.f); - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - a1 = Aq[1*Np+n]; - b1 = Bq[1*Np+n]; - a2 = Aq[2*Np+n]; - b2 = Bq[2*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); - - Aq[1*Np+n] = a1; - Bq[1*Np+n] = b1; - Aq[2*Np+n] = a2; - Bq[2*Np+n] = b2; - - //............................................... - // q = 2 - // Cq = {0,1,0} - a1 = Aq[3*Np+n]; - b1 = Bq[3*Np+n]; - a2 = Aq[4*Np+n]; - b2 = Bq[4*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); - - Aq[3*Np+n] = a1; - Bq[3*Np+n] = b1; - Aq[4*Np+n] = a2; - Bq[4*Np+n] = b2; - //............................................... - // q = 4 - // Cq = {0,0,1} - a1 = Aq[5*Np+n]; - b1 = Bq[5*Np+n]; - a2 = Aq[6*Np+n]; - b2 = Bq[6*Np+n]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); - - Aq[5*Np+n] = a1; - Bq[5*Np+n] = b1; - Aq[6*Np+n] = a2; - Bq[6*Np+n] = b2; - //............................................... - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - a1=Aq[n]; - b1=Bq[n]; - Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); - Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); - - - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ @@ -1995,11 +2682,13 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou double phi; double phi_lap;//laplacian of phase field double nA,nB; - double a1,b1,a2,b2; + //double a1,b1,a2,b2; double Gfs_x,Gfs_y,Gfs_z; double Gff_x,Gff_y,Gff_z; - double chem_a,chem_b; - double rlx_massA,rlx_massB; + double chem; + //double rlx_massA,rlx_massB; + double rlx_phi; + double a1,a2;//PDF of phase field // *---------------------------------Pressure Tensor Gradient------------------------------------*// double Pxx_x,Pyy_y,Pzz_z; double Pxy_x,Pxy_y; @@ -2030,8 +2719,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // read the component number densities nA = Den[n]; nB = Den[Np + n]; - // compute phase indicator field - phi=(nA-nB)/(nA+nB); + // read phase field + phi = Phi[n]; // load laplacian of phase field phi_lap = PhiLap[n]; // Load voxel porosity and perm @@ -2576,82 +3265,678 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //-----------------------Mass transport------------------------// // calcuale chemical potential - chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; - chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; - rlx_massA = 3.f-sqrt(3.f); - rlx_massB = 3.f-sqrt(3.f); + chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; + rlx_phi = 3.f-sqrt(3.f); //............................................... // q = 0,2,4 // Cq = {1,0,0}, {0,1,0}, {0,0,1} - a1 = Aq[nr2]; - b1 = Bq[nr2]; - a2 = Aq[nr1]; - b2 = Bq[nr1]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); + a1 = Cq[nr2]; + a2 = Cq[nr1]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); // q = 1 //nread = neighborList[n+Np]; - Aq[nr2] = a1; - Bq[nr2] = b1; + Cq[nr2] = a1; // q=2 //nread = neighborList[n]; - Aq[nr1] = a2; - Bq[nr1] = b2; + Cq[nr1] = a2; //............................................... // Cq = {0,1,0} - a1 = Aq[nr4]; - b1 = Bq[nr4]; - a2 = Aq[nr3]; - b2 = Bq[nr3]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); + a1 = Cq[nr4]; + a2 = Cq[nr3]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); // q = 3 //nread = neighborList[n+3*Np]; - Aq[nr4] = a1; - Bq[nr4] = b1; + Cq[nr4] = a1; // q = 4 //nread = neighborList[n+2*Np]; - Aq[nr3] = a2; - Bq[nr3] = b2; + Cq[nr3] = a2; //............................................... // q = 4 // Cq = {0,0,1} - a1 = Aq[nr6]; - b1 = Bq[nr6]; - a2 = Aq[nr5]; - b2 = Bq[nr5]; - a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); - b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); - a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); - b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); + a1 = Cq[nr6]; + a2 = Cq[nr5]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); // q = 5 //nread = neighborList[n+5*Np]; - Aq[nr6] = a1; - Bq[nr6] = b1; + Cq[nr6] = a1; // q = 6 //nread = neighborList[n+4*Np]; - Aq[nr5] = a2; - Bq[nr5] = b2; + Cq[nr5] = a2; //............................................... // Instantiate mass transport distributions // Stationary value - distribution 0 - a1=Aq[n]; - b1=Bq[n]; - Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); - Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); + a1=Cq[n]; + Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(a1-3.0*gamma*chem); + + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double phi_lap;//laplacian of phase field + double nA,nB; + //double a1,b1,a2,b2; + double Gfs_x,Gfs_y,Gfs_z; + double Gff_x,Gff_y,Gff_z; + double chem; + //double rlx_massA,rlx_massB; + double rlx_phi; + double a1,a2;//PDF of phase field + // *---------------------------------Pressure Tensor Gradient------------------------------------*// + double Pxx_x,Pyy_y,Pzz_z; + double Pxy_x,Pxy_y; + double Pyz_y,Pyz_z; + double Pxz_x,Pxz_z; + double px,py,pz; //pressure gradient + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + //Load pressure gradient + px=PressureGrad[0*Np+n]; + py=PressureGrad[1*Np+n]; + pz=PressureGrad[2*Np+n]; + + //Load pressure tensor gradient + //For reference full list of PressTensorGrad + //PressTensorGrad[n+0*Np] = Pxx_x + //PressTensorGrad[n+1*Np] = Pxx_y + //PressTensorGrad[n+2*Np] = Pxx_z + //PressTensorGrad[n+3*Np] = Pyy_x + //PressTensorGrad[n+4*Np] = Pyy_y + //PressTensorGrad[n+5*Np] = Pyy_z + //PressTensorGrad[n+6*Np] = Pzz_x + //PressTensorGrad[n+7*Np] = Pzz_y + //PressTensorGrad[n+8*Np] = Pzz_z + //PressTensorGrad[n+9*Np] = Pxy_x + //PressTensorGrad[n+10*Np] = Pxy_y + //PressTensorGrad[n+11*Np] = Pxy_z + //PressTensorGrad[n+12*Np] = Pyz_x + //PressTensorGrad[n+13*Np] = Pyz_y + //PressTensorGrad[n+14*Np] = Pyz_z + //PressTensorGrad[n+15*Np] = Pxz_x + //PressTensorGrad[n+16*Np] = Pxz_y + //PressTensorGrad[n+17*Np] = Pxz_z + Pxx_x = PressTensorGrad[0*Np+n]; + Pyy_y = PressTensorGrad[4*Np+n]; + Pzz_z = PressTensorGrad[8*Np+n]; + Pxy_x = PressTensorGrad[9*Np+n]; + Pxz_x = PressTensorGrad[15*Np+n]; + Pxy_y = PressTensorGrad[10*Np+n]; + Pyz_y = PressTensorGrad[13*Np+n]; + Pyz_z = PressTensorGrad[14*Np+n]; + Pxz_z = PressTensorGrad[17*Np+n]; + //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... + //TODO double check if you need porosity as a fre-factor + Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); + Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); + Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + // local density + rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); + // local relaxation time + tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); + rlx_setA = 1.f/tau; + rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); + mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity + + + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + // q=0 + fq = dist[n]; + m1 = -30.0*fq; + m2 = 12.0*fq; + + // q=1 + fq = dist[2*Np+n]; + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // f2 = dist[10*Np+n]; + fq = dist[1*Np+n]; + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + fq = dist[4*Np+n]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + fq = dist[3*Np+n]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + fq = dist[6*Np+n]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + fq = dist[5*Np+n]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + fq = dist[8*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + fq = dist[7*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + fq = dist[10*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + fq = dist[9*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + fq = dist[12*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + fq = dist[11*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + fq = dist[14*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + fq = dist[13*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + fq = dist[16*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + fq = dist[15*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + fq = dist[18*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + fq = dist[17*Np+n]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + dist[1*Np+n] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + dist[2*Np+n] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[3*Np+n] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[4*Np+n] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[5*Np+n] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[6*Np+n] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + dist[7*Np+n] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + dist[8*Np+n] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + dist[9*Np+n] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + dist[10*Np+n] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + dist[11*Np+n] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + dist[12*Np+n] = fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + dist[13*Np+n] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + dist[14*Np+n] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + dist[15*Np+n] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + dist[16*Np+n] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + dist[17*Np+n] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + dist[18*Np+n] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // calcuale chemical potential + chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; + rlx_phi = 3.f-sqrt(3.f); + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + a1 = Cq[1*Np+n]; + a2 = Cq[2*Np+n]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); + + Cq[1*Np+n] = a1; + Cq[2*Np+n] = a2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + a1 = Cq[3*Np+n]; + a2 = Cq[4*Np+n]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); + + Cq[3*Np+n] = a1; + Cq[4*Np+n] = a2; + //............................................... + // q = 4 + // Cq = {0,0,1} + a1 = Cq[5*Np+n]; + a2 = Cq[6*Np+n]; + a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); + a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); + + Cq[5*Np+n] = a1; + Cq[6*Np+n] = a2; + //............................................... + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + a1=Cq[n]; + Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(a1-3.0*gamma*chem); } } } @@ -2694,9 +3979,43 @@ __global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, d } } -__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ +//__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ +// int idx; +// double nA,nB; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s>>(dist, Aq, Bq, Den, SolidForce, start, finish, Np, +// tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); +// +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); +// } +//} +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ +// +// dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, SolidForce, start, finish, Np, +// tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, +// Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); +// +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); +// } +//} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Aq, Bq, Den, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Cq, Phi, Den, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); @@ -3126,12 +4567,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Cq, Phi, Den, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); @@ -3141,9 +4582,16 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double } } +//extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ +// dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, Phi, start, finish, Np); +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); +// } +//} -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, Phi, start, finish, Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *Phi, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Cq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); @@ -3177,6 +4625,25 @@ extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, } } +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Den, double *Phi, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi<<>>(NeighborList, Cq, Den, Phi, start, finish, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleColorPhi: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Den, double *Phi, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi<<>>(Cq, Den, Phi, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorPhi: %s \n",cudaGetErrorString(err)); + } +} + extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient<<>>(neighborList, Den, DenGrad, start, finish, Np); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 26c6d783..6dc77451 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -561,8 +561,9 @@ void ScaLBL_GreyscaleColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PressureGrad, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Cq, 7*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PhiLap, sizeof(double)*Np);//laplacian of phase field @@ -622,8 +623,10 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); //TODO need to initialize velocity field ! //this is required for calculating the pressure_dvc @@ -632,8 +635,10 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ else{ if (rank==0) printf ("Initializing density field \n"); DensityField_Init();//initialize density field - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); @@ -695,11 +700,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ timestep++; // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + //ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL + //ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + //ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); @@ -763,7 +772,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL // ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, Den, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -777,7 +786,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // } // ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Aq, Bq, Den, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, Den, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -788,11 +797,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ timestep++; // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + //ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL + //ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + //ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); @@ -855,7 +868,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL // ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, Den, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -869,7 +882,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // } // ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, // tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Aq, Bq, Den, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, Den, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 374920d0..0e5a4af0 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -65,7 +65,8 @@ public: signed char *id; int *NeighborList; - double *fq,*Aq,*Bq; + //double *fq,*Aq,*Bq; + double *fq,*Cq; double *Den; double *Permeability;//grey voxel permeability double *Porosity; From 389c60a06f9e5caf07c79ef0a253fe890fc90685 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 19 Apr 2020 22:21:43 -0400 Subject: [PATCH 117/270] GPU version; save the work; mass is not conserved --- common/ScaLBL.h | 2 +- gpu/GreyscaleColor.cu | 288 +++++++++++++++++++++++---------- models/GreyscaleColorModel.cpp | 25 ++- models/GreyscaleColorModel.h | 2 +- 4 files changed, 222 insertions(+), 95 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index d3251740..4557adb3 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -105,7 +105,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); //extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 489b8a6b..74fe1923 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -3266,7 +3266,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //-----------------------Mass transport------------------------// // calcuale chemical potential chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; - rlx_phi = 3.f-sqrt(3.f); + //rlx_phi = 3.f-sqrt(3.f); + rlx_phi = 1.0; //............................................... // q = 0,2,4 @@ -3898,7 +3899,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double //-----------------------Mass transport------------------------// // calcuale chemical potential chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; - rlx_phi = 3.f-sqrt(3.f); + //rlx_phi = 3.f-sqrt(3.f); + rlx_phi = 1.0; //............................................... // q = 0,2,4 @@ -4012,11 +4014,13 @@ __global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, d // } //} -__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *Phi, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, + int start, int finish, int Np){ int idx; double nA,nB; double phi; - + double phi_lap;//laplacian of the phase field + double chem;//chemical potential int S = Np/NBLOCKS/NTHREADS + 1; for (int s=0; s>>(Den, Cq, Phi, start, finish, Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 6dc77451..6ac69b91 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -455,7 +455,7 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double } } -void ScaLBL_GreyscaleColorModel::DensityField_Init(){ +void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ size_t NLABELS=0; signed char VALUE=0; @@ -482,6 +482,10 @@ void ScaLBL_GreyscaleColorModel::DensityField_Init(){ double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. double nB=0.5; + double *Phi_temp; + Phi_temp=new double [Np]; + double phi = 0.0; + for (int k=1; k1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); nB=Sw; nA=1.0-Sw; + phi = nA-nB; idx = NLABELS; } } if (VALUE==1){//label=1 reserved for NW phase nA=1.0; nB=0.0; + phi = nA-nB; } else if(VALUE==2){//label=2 reserved for W phase nA=0.0; nB=1.0; + phi = nA-nB; } int idx = Map(i,j,k); Den_temp[idx+0*Np] = nA; Den_temp[idx+1*Np] = nB; + Phi_temp[idx] = phi; } } } } //copy to device ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); + ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); ScaLBL_DeviceBarrier(); delete [] Den_temp; } @@ -625,8 +634,10 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); //TODO need to initialize velocity field ! //this is required for calculating the pressure_dvc @@ -634,11 +645,13 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ } else{ if (rank==0) printf ("Initializing density field \n"); - DensityField_Init();//initialize density field + Density_and_Phase_Init();//initialize density field //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 0e5a4af0..8b0cd85e 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -102,7 +102,7 @@ private: void AssignComponentLabels(double *Porosity, double *Permeablity, double *SolidPotential); void AssignSolidForce(double *SolidPotential, double *SolidForce); - void DensityField_Init(); + void Density_and_Phase_Init(); }; From 74e858f005f167ef5cca6422f7e02e5e8839e21e Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 19 Apr 2020 22:49:33 -0400 Subject: [PATCH 118/270] fix dumb bugs --- models/GreyscaleColorModel.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 6ac69b91..1680939b 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -344,9 +344,9 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double //Populate the poroisty map, NOTE only for node_ID > 0, i.e. open or grey nodes //For node_ID <= 0: these are solid nodes of various wettability - for (int k=1;k 0, i.e. open or grey nodes //For node_ID <= 0: these are solid nodes of various wettability - for (int k=1;kid[n]; if (VALUE>0){ @@ -525,6 +525,7 @@ void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); ScaLBL_DeviceBarrier(); delete [] Den_temp; + delete [] Phi_temp; } void ScaLBL_GreyscaleColorModel::Create(){ From cc61cb940d99e4037e2dac073be0d8ca9d7c1d2d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 19 Apr 2020 23:35:37 -0400 Subject: [PATCH 119/270] fix another dumb bug --- gpu/GreyscaleColor.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 74fe1923..b9519cd7 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -4029,6 +4029,7 @@ __global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, doub nA = Den[idx]; nB = Den[idx+Np]; phi = nA-nB; + phi_lap = PhiLap[idx]; chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; Cq[1*Np+idx]=0.5*gamma*chem; From eaf9e828ea289dce4125ce584f5d00979a3d12ff Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 24 Apr 2020 16:20:52 -0400 Subject: [PATCH 120/270] GPU only; save the work; greyscaleFE works only conditionally; --- common/ScaLBL.cpp | 27 +- common/ScaLBL.h | 21 +- gpu/GreyscaleColor.cu | 1834 ++++---------------------------- models/GreyscaleColorModel.cpp | 171 ++- 4 files changed, 280 insertions(+), 1773 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index e0f3879c..0feb5558 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1151,41 +1151,39 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq){ //...Packing for x face(2,8,10,12,14)................................ ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N); - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 2*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 2*recvCount_X,rank_X,recvtag); + req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); + req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); //...Packing for X face(1,7,9,11,13)................................ ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N); - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 2*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 2*recvCount_x,rank_x,recvtag); + req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); + req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); //...Packing for y face(4,8,9,16,18)................................. ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N); - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 2*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 2*recvCount_Y,rank_Y,recvtag); + req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); + req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N); - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 2*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 2*recvCount_y,rank_y,recvtag); + req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); + req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); //...Packing for z face(6,12,13,16,17)................................ ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N); - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 2*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 2*recvCount_Z,rank_Z,recvtag); + req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); + req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); //...Packing for Z face(5,11,14,15,18)................................ ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N); + req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); + req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); //................................................................................... - // Send all the distributions - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 2*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 2*recvCount_z,rank_z,recvtag); - } void ScaLBL_Communicator::RecvD3Q7AA(double *Aq){ @@ -1194,6 +1192,7 @@ void ScaLBL_Communicator::RecvD3Q7AA(double *Aq){ //................................................................................... // Wait for completion of D3Q19 communication MPI_COMM_SCALBL.waitAll(6,req1); + MPI_COMM_SCALBL.waitAll(6,req2); ScaLBL_DeviceBarrier(); //................................................................................... diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 4557adb3..849b8252 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -84,27 +84,16 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dis double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); -//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, -// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, -// double Gx, double Gy, double Gz, -// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -// -//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, -// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, -// double Gx, double Gy, double Gz, -// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -//extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np); extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); @@ -113,9 +102,9 @@ extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, doubl extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Phi, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); @@ -124,7 +113,7 @@ extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, double *Pressure, double rhoA,double rhoB, int Np); -extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi, double *PressTensor, double *PhiLap, +extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); // MRT MODEL diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index b9519cd7..4841e212 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -1329,1334 +1329,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double } } -//__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, -// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, -// double Gx, double Gy, double Gz, -// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ -// int n; -// double vx,vy,vz,v_mag; -// double ux,uy,uz,u_mag; -// double pressure;//defined for this incompressible model -// // conserved momemnts -// double jx,jy,jz; -// // non-conserved moments -// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; -// double fq; -// // currently disable 'GeoFun' -// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) -// double porosity; -// double perm;//voxel permeability -// double c0, c1; //Guo's model parameters -// double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) -// double tau,tau_eff,rlx_setA,rlx_setB; -// double mu_eff;//effective kinematic viscosity for Darcy term -// double rho0; -// double phi; -// double phi_lap;//laplacian of phase field -// double nA,nB; -// double a1,b1,a2,b2; -// double Gfs_x,Gfs_y,Gfs_z; -// double Gff_x,Gff_y,Gff_z; -// double chem_a,chem_b; -// double rlx_massA,rlx_massB; -// // *---------------------------------Pressure Tensor Gradient------------------------------------*// -// double Pxx_x,Pyy_y,Pzz_z; -// double Pxy_x,Pxy_y; -// double Pyz_y,Pyz_z; -// double Pxz_x,Pxz_z; -// double px,py,pz; //pressure gradient -// -// -// const double mrt_V1=0.05263157894736842; -// const double mrt_V2=0.012531328320802; -// const double mrt_V3=0.04761904761904762; -// const double mrt_V4=0.004594820384294068; -// const double mrt_V5=0.01587301587301587; -// const double mrt_V6=0.0555555555555555555555555; -// const double mrt_V7=0.02777777777777778; -// const double mrt_V8=0.08333333333333333; -// const double mrt_V9=0.003341687552213868; -// const double mrt_V10=0.003968253968253968; -// const double mrt_V11=0.01388888888888889; -// const double mrt_V12=0.04166666666666666; -// -// -// int S = Np/NBLOCKS/NTHREADS + 1; -// for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); -// -// //Load pressure gradient -// px=PressureGrad[0*Np+n]; -// py=PressureGrad[1*Np+n]; -// pz=PressureGrad[2*Np+n]; -// -// //Load pressure tensor gradient -// //For reference full list of PressTensorGrad -// //PressTensorGrad[n+0*Np] = Pxx_x -// //PressTensorGrad[n+1*Np] = Pxx_y -// //PressTensorGrad[n+2*Np] = Pxx_z -// //PressTensorGrad[n+3*Np] = Pyy_x -// //PressTensorGrad[n+4*Np] = Pyy_y -// //PressTensorGrad[n+5*Np] = Pyy_z -// //PressTensorGrad[n+6*Np] = Pzz_x -// //PressTensorGrad[n+7*Np] = Pzz_y -// //PressTensorGrad[n+8*Np] = Pzz_z -// //PressTensorGrad[n+9*Np] = Pxy_x -// //PressTensorGrad[n+10*Np] = Pxy_y -// //PressTensorGrad[n+11*Np] = Pxy_z -// //PressTensorGrad[n+12*Np] = Pyz_x -// //PressTensorGrad[n+13*Np] = Pyz_y -// //PressTensorGrad[n+14*Np] = Pyz_z -// //PressTensorGrad[n+15*Np] = Pxz_x -// //PressTensorGrad[n+16*Np] = Pxz_y -// //PressTensorGrad[n+17*Np] = Pxz_z -// Pxx_x = PressTensorGrad[0*Np+n]; -// Pyy_y = PressTensorGrad[4*Np+n]; -// Pzz_z = PressTensorGrad[8*Np+n]; -// Pxy_x = PressTensorGrad[9*Np+n]; -// Pxz_x = PressTensorGrad[15*Np+n]; -// Pxy_y = PressTensorGrad[10*Np+n]; -// Pyz_y = PressTensorGrad[13*Np+n]; -// Pyz_z = PressTensorGrad[14*Np+n]; -// Pxz_z = PressTensorGrad[17*Np+n]; -// //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... -// //TODO double check if you need porosity as a fre-factor -// Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); -// Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); -// Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); -// // fluid-solid force -// Gfs_x = (nA-nB)*SolidForce[n+0*Np]; -// Gfs_y = (nA-nB)*SolidForce[n+1*Np]; -// Gfs_z = (nA-nB)*SolidForce[n+2*Np]; -// -// // local density -// rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); -// // local relaxation time -// tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); -// rlx_setA = 1.f/tau; -// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); -// tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); -// mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity -// -// -// //........................................................................ -// // READ THE DISTRIBUTIONS -// // (read from opposite array due to previous swap operation) -// //........................................................................ -// // q=0 -// fq = dist[n]; -// m1 = -30.0*fq; -// m2 = 12.0*fq; -// -// // q=1 -// fq = dist[2*Np+n]; -// pressure = fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jx = fq; -// m4 = -4.0*fq; -// m9 = 2.0*fq; -// m10 = -4.0*fq; -// -// // f2 = dist[10*Np+n]; -// fq = dist[1*Np+n]; -// pressure += fq; -// m1 -= 11.0*(fq); -// m2 -= 4.0*(fq); -// jx -= fq; -// m4 += 4.0*(fq); -// m9 += 2.0*(fq); -// m10 -= 4.0*(fq); -// -// // q=3 -// fq = dist[4*Np+n]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jy = fq; -// m6 = -4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 = fq; -// m12 = -2.0*fq; -// -// // q = 4 -// fq = dist[3*Np+n]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jy -= fq; -// m6 += 4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 += fq; -// m12 -= 2.0*fq; -// -// // q=5 -// fq = dist[6*Np+n]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jz = fq; -// m8 = -4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 -= fq; -// m12 += 2.0*fq; -// -// // q = 6 -// fq = dist[5*Np+n]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jz -= fq; -// m8 += 4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 -= fq; -// m12 += 2.0*fq; -// -// // q=7 -// fq = dist[8*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jy += fq; -// m6 += fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 = fq; -// m16 = fq; -// m17 = -fq; -// -// // q = 8 -// fq = dist[7*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jy -= fq; -// m6 -= fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 += fq; -// m16 -= fq; -// m17 += fq; -// -// // q=9 -// fq = dist[10*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jy -= fq; -// m6 -= fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 -= fq; -// m16 += fq; -// m17 += fq; -// -// // q = 10 -// fq = dist[9*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jy += fq; -// m6 += fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 -= fq; -// m16 -= fq; -// m17 -= fq; -// -// // q=11 -// fq = dist[12*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jz += fq; -// m8 += fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 = fq; -// m16 -= fq; -// m18 = fq; -// -// // q=12 -// fq = dist[11*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jz -= fq; -// m8 -= fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 += fq; -// m16 += fq; -// m18 -= fq; -// -// // q=13 -// fq = dist[14*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jz -= fq; -// m8 -= fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 -= fq; -// m16 -= fq; -// m18 -= fq; -// -// // q=14 -// fq = dist[13*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jz += fq; -// m8 += fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 -= fq; -// m16 += fq; -// m18 += fq; -// -// // q=15 -// fq = dist[16*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy += fq; -// m6 += fq; -// jz += fq; -// m8 += fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 = fq; -// m17 += fq; -// m18 -= fq; -// -// // q=16 -// fq = dist[15*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy -= fq; -// m6 -= fq; -// jz -= fq; -// m8 -= fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 += fq; -// m17 -= fq; -// m18 += fq; -// -// // q=17 -// fq = dist[18*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy += fq; -// m6 += fq; -// jz -= fq; -// m8 -= fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 -= fq; -// m17 += fq; -// m18 += fq; -// -// // q=18 -// fq = dist[17*Np+n]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy -= fq; -// m6 -= fq; -// jz += fq; -// m8 += fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 -= fq; -// m17 -= fq; -// m18 -= fq; -// //---------------------------------------------------------------------// -// -// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); -// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes -// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); -// c1 = porosity*0.5*GeoFun/sqrt(perm); -// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes -// -// vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); -// vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); -// vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); -// v_mag=sqrt(vx*vx+vy*vy+vz*vz); -// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); -// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); -// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); -// u_mag=sqrt(ux*ux+uy*uy+uz*uz); -// -// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium -// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); -// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); -// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); -// if (porosity==1.0){ -// Fx=rho0*(Gx + Gff_x + Gfs_x); -// Fy=rho0*(Gy + Gff_y + Gfs_y); -// Fz=rho0*(Gz + Gff_z + Gfs_z); -// } -// -// //Calculate pressure for Incompressible-MRT model -// pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); -// -//// //..............carry out relaxation process............................................... -//// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -//// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -//// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -//// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -//// jx = jx + Fx; -//// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -//// jy = jy + Fy; -//// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -//// jz = jz + Fz; -//// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -//// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -//// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -//// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -//// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -//// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -//// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -//// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -//// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -//// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -//// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -//// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -//// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -//// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -//// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -//// m16 = m16 + rlx_setB*( - m16); -//// m17 = m17 + rlx_setB*( - m17); -//// m18 = m18 + rlx_setB*( - m18); -//// //....................................................................................................... -// -// //-------------------- IMRT collison where body force has NO higher-order terms -------------// -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... -// -// //.................inverse transformation...................................................... -// // q=0 -// fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; -// dist[n] = fq; -// -// // q = 1 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); -// dist[1*Np+n] = fq; -// -// // q=2 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); -// dist[2*Np+n] = fq; -// -// // q = 3 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); -// dist[3*Np+n] = fq; -// -// // q = 4 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); -// dist[4*Np+n] = fq; -// -// // q = 5 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); -// dist[5*Np+n] = fq; -// -// // q = 6 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); -// dist[6*Np+n] = fq; -// -// // q = 7 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); -// dist[7*Np+n] = fq; -// -// // q = 8 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); -// dist[8*Np+n] = fq; -// -// // q = 9 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); -// dist[9*Np+n] = fq; -// -// // q = 10 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); -// dist[10*Np+n] = fq; -// -// // q = 11 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); -// dist[11*Np+n] = fq; -// -// // q = 12 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); -// dist[12*Np+n] = fq; -// -// // q = 13 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); -// dist[13*Np+n] = fq; -// -// // q= 14 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); -// dist[14*Np+n] = fq; -// -// // q = 15 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); -// dist[15*Np+n] = fq; -// -// // q = 16 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); -// dist[16*Np+n] = fq; -// -// // q = 17 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); -// dist[17*Np+n] = fq; -// -// // q = 18 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); -// dist[18*Np+n] = fq; -// //........................................................................ -// -// //Update velocity on device -// Velocity[0*Np+n] = ux; -// Velocity[1*Np+n] = uy; -// Velocity[2*Np+n] = uz; -// //Update pressure on device -// Pressure[n] = pressure; -// -// //-----------------------Mass transport------------------------// -// // calcuale chemical potential -// chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; -// chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; -// rlx_massA = 3.f-sqrt(3.f); -// rlx_massB = 3.f-sqrt(3.f); -// -// //............................................... -// // q = 0,2,4 -// // Cq = {1,0,0}, {0,1,0}, {0,0,1} -// a1 = Aq[1*Np+n]; -// b1 = Bq[1*Np+n]; -// a2 = Aq[2*Np+n]; -// b2 = Bq[2*Np+n]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); -// -// Aq[1*Np+n] = a1; -// Bq[1*Np+n] = b1; -// Aq[2*Np+n] = a2; -// Bq[2*Np+n] = b2; -// -// //............................................... -// // q = 2 -// // Cq = {0,1,0} -// a1 = Aq[3*Np+n]; -// b1 = Bq[3*Np+n]; -// a2 = Aq[4*Np+n]; -// b2 = Bq[4*Np+n]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); -// -// Aq[3*Np+n] = a1; -// Bq[3*Np+n] = b1; -// Aq[4*Np+n] = a2; -// Bq[4*Np+n] = b2; -// //............................................... -// // q = 4 -// // Cq = {0,0,1} -// a1 = Aq[5*Np+n]; -// b1 = Bq[5*Np+n]; -// a2 = Aq[6*Np+n]; -// b2 = Bq[6*Np+n]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); -// -// Aq[5*Np+n] = a1; -// Bq[5*Np+n] = b1; -// Aq[6*Np+n] = a2; -// Bq[6*Np+n] = b2; -// //............................................... -// -// // Instantiate mass transport distributions -// // Stationary value - distribution 0 -// a1=Aq[n]; -// b1=Bq[n]; -// Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); -// Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); -// -// -// } -// } -//} - -//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, -// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, -// double Gx, double Gy, double Gz, -// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ -// -// int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; -// double vx,vy,vz,v_mag; -// double ux,uy,uz,u_mag; -// double pressure;//defined for this incompressible model -// // conserved momemnts -// double jx,jy,jz; -// // non-conserved moments -// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; -// double fq; -// // currently disable 'GeoFun' -// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) -// double porosity; -// double perm;//voxel permeability -// double c0, c1; //Guo's model parameters -// double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) -// double tau,tau_eff,rlx_setA,rlx_setB; -// double mu_eff;//effective kinematic viscosity for Darcy term -// double rho0; -// double phi; -// double phi_lap;//laplacian of phase field -// double nA,nB; -// double a1,b1,a2,b2; -// double Gfs_x,Gfs_y,Gfs_z; -// double Gff_x,Gff_y,Gff_z; -// double chem_a,chem_b; -// double rlx_massA,rlx_massB; -// // *---------------------------------Pressure Tensor Gradient------------------------------------*// -// double Pxx_x,Pyy_y,Pzz_z; -// double Pxy_x,Pxy_y; -// double Pyz_y,Pyz_z; -// double Pxz_x,Pxz_z; -// double px,py,pz; //pressure gradient -// -// const double mrt_V1=0.05263157894736842; -// const double mrt_V2=0.012531328320802; -// const double mrt_V3=0.04761904761904762; -// const double mrt_V4=0.004594820384294068; -// const double mrt_V5=0.01587301587301587; -// const double mrt_V6=0.0555555555555555555555555; -// const double mrt_V7=0.02777777777777778; -// const double mrt_V8=0.08333333333333333; -// const double mrt_V9=0.003341687552213868; -// const double mrt_V10=0.003968253968253968; -// const double mrt_V11=0.01388888888888889; -// const double mrt_V12=0.04166666666666666; -// -// int S = Np/NBLOCKS/NTHREADS + 1; -// for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); -// -// //Load pressure gradient -// px=PressureGrad[0*Np+n]; -// py=PressureGrad[1*Np+n]; -// pz=PressureGrad[2*Np+n]; -// -// //Load pressure tensor gradient -// //For reference full list of PressTensorGrad -// //PressTensorGrad[n+0*Np] = Pxx_x -// //PressTensorGrad[n+1*Np] = Pxx_y -// //PressTensorGrad[n+2*Np] = Pxx_z -// //PressTensorGrad[n+3*Np] = Pyy_x -// //PressTensorGrad[n+4*Np] = Pyy_y -// //PressTensorGrad[n+5*Np] = Pyy_z -// //PressTensorGrad[n+6*Np] = Pzz_x -// //PressTensorGrad[n+7*Np] = Pzz_y -// //PressTensorGrad[n+8*Np] = Pzz_z -// //PressTensorGrad[n+9*Np] = Pxy_x -// //PressTensorGrad[n+10*Np] = Pxy_y -// //PressTensorGrad[n+11*Np] = Pxy_z -// //PressTensorGrad[n+12*Np] = Pyz_x -// //PressTensorGrad[n+13*Np] = Pyz_y -// //PressTensorGrad[n+14*Np] = Pyz_z -// //PressTensorGrad[n+15*Np] = Pxz_x -// //PressTensorGrad[n+16*Np] = Pxz_y -// //PressTensorGrad[n+17*Np] = Pxz_z -// Pxx_x = PressTensorGrad[0*Np+n]; -// Pyy_y = PressTensorGrad[4*Np+n]; -// Pzz_z = PressTensorGrad[8*Np+n]; -// Pxy_x = PressTensorGrad[9*Np+n]; -// Pxz_x = PressTensorGrad[15*Np+n]; -// Pxy_y = PressTensorGrad[10*Np+n]; -// Pyz_y = PressTensorGrad[13*Np+n]; -// Pyz_z = PressTensorGrad[14*Np+n]; -// Pxz_z = PressTensorGrad[17*Np+n]; -// //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... -// //TODO double check if you need porosity as a fre-factor -// Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); -// Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); -// Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); -// // fluid-solid force -// Gfs_x = (nA-nB)*SolidForce[n+0*Np]; -// Gfs_y = (nA-nB)*SolidForce[n+1*Np]; -// Gfs_z = (nA-nB)*SolidForce[n+2*Np]; -// -// // local density -// rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); -// // local relaxation time -// tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); -// rlx_setA = 1.f/tau; -// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); -// tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); -// mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity -// -// //........................................................................ -// // READ THE DISTRIBUTIONS -// // (read from opposite array due to previous swap operation) -// //........................................................................ -// // q=0 -// fq = dist[n]; -// m1 = -30.0*fq; -// m2 = 12.0*fq; -// -// // q=1 -// nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist) -// fq = dist[nr1]; // reading the f1 data into register fq -// pressure = fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jx = fq; -// m4 = -4.0*fq; -// m9 = 2.0*fq; -// m10 = -4.0*fq; -// -// // q=2 -// nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) -// fq = dist[nr2]; // reading the f2 data into register fq -// pressure += fq; -// m1 -= 11.0*(fq); -// m2 -= 4.0*(fq); -// jx -= fq; -// m4 += 4.0*(fq); -// m9 += 2.0*(fq); -// m10 -= 4.0*(fq); -// -// // q=3 -// nr3 = neighborList[n+2*Np]; // neighbor 4 -// fq = dist[nr3]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jy = fq; -// m6 = -4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 = fq; -// m12 = -2.0*fq; -// -// // q = 4 -// nr4 = neighborList[n+3*Np]; // neighbor 3 -// fq = dist[nr4]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jy -= fq; -// m6 += 4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 += fq; -// m12 -= 2.0*fq; -// -// // q=5 -// nr5 = neighborList[n+4*Np]; -// fq = dist[nr5]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jz = fq; -// m8 = -4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 -= fq; -// m12 += 2.0*fq; -// -// // q = 6 -// nr6 = neighborList[n+5*Np]; -// fq = dist[nr6]; -// pressure += fq; -// m1 -= 11.0*fq; -// m2 -= 4.0*fq; -// jz -= fq; -// m8 += 4.0*fq; -// m9 -= fq; -// m10 += 2.0*fq; -// m11 -= fq; -// m12 += 2.0*fq; -// -// // q=7 -// nread = neighborList[n+6*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jy += fq; -// m6 += fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 = fq; -// m16 = fq; -// m17 = -fq; -// -// // q = 8 -// nread = neighborList[n+7*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jy -= fq; -// m6 -= fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 += fq; -// m16 -= fq; -// m17 += fq; -// -// // q=9 -// nread = neighborList[n+8*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jy -= fq; -// m6 -= fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 -= fq; -// m16 += fq; -// m17 += fq; -// -// // q = 10 -// nread = neighborList[n+9*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jy += fq; -// m6 += fq; -// m9 += fq; -// m10 += fq; -// m11 += fq; -// m12 += fq; -// m13 -= fq; -// m16 -= fq; -// m17 -= fq; -// -// // q=11 -// nread = neighborList[n+10*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jz += fq; -// m8 += fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 = fq; -// m16 -= fq; -// m18 = fq; -// -// // q=12 -// nread = neighborList[n+11*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jz -= fq; -// m8 -= fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 += fq; -// m16 += fq; -// m18 -= fq; -// -// // q=13 -// nread = neighborList[n+12*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx += fq; -// m4 += fq; -// jz -= fq; -// m8 -= fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 -= fq; -// m16 -= fq; -// m18 -= fq; -// -// // q=14 -// nread = neighborList[n+13*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jx -= fq; -// m4 -= fq; -// jz += fq; -// m8 += fq; -// m9 += fq; -// m10 += fq; -// m11 -= fq; -// m12 -= fq; -// m15 -= fq; -// m16 += fq; -// m18 += fq; -// -// // q=15 -// nread = neighborList[n+14*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy += fq; -// m6 += fq; -// jz += fq; -// m8 += fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 = fq; -// m17 += fq; -// m18 -= fq; -// -// // q=16 -// nread = neighborList[n+15*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy -= fq; -// m6 -= fq; -// jz -= fq; -// m8 -= fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 += fq; -// m17 -= fq; -// m18 += fq; -// -// // q=17 -// nread = neighborList[n+16*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy += fq; -// m6 += fq; -// jz -= fq; -// m8 -= fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 -= fq; -// m17 += fq; -// m18 += fq; -// -// // q=18 -// nread = neighborList[n+17*Np]; -// fq = dist[nread]; -// pressure += fq; -// m1 += 8.0*fq; -// m2 += fq; -// jy -= fq; -// m6 -= fq; -// jz += fq; -// m8 += fq; -// m9 -= 2.0*fq; -// m10 -= 2.0*fq; -// m14 -= fq; -// m17 -= fq; -// m18 -= fq; -// //---------------------------------------------------------------------// -// -// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); -// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes -// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); -// c1 = porosity*0.5*GeoFun/sqrt(perm); -// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes -// -// vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); -// vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); -// vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); -// v_mag=sqrt(vx*vx+vy*vy+vz*vz); -// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); -// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); -// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); -// u_mag=sqrt(ux*ux+uy*uy+uz*uz); -// -// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium -// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); -// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); -// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); -// if (porosity==1.0){ -// Fx=rho0*(Gx + Gff_x + Gfs_x); -// Fy=rho0*(Gy + Gff_y + Gfs_y); -// Fz=rho0*(Gz + Gff_z + Gfs_z); -// } -// -// //Calculate pressure for Incompressible-MRT model -// pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); -// -//// //..............carry out relaxation process............................................... -//// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -//// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -//// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -//// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -//// jx = jx + Fx; -//// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -//// jy = jy + Fy; -//// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -//// jz = jz + Fz; -//// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -//// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -//// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -//// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -//// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -//// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -//// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -//// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -//// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -//// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -//// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -//// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -//// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -//// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -//// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -//// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -//// m16 = m16 + rlx_setB*( - m16); -//// m17 = m17 + rlx_setB*( - m17); -//// m18 = m18 + rlx_setB*( - m18); -//// //....................................................................................................... -// -// //-------------------- IMRT collison where body force has NO higher-order terms -------------// -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... -// -// -// //.................inverse transformation...................................................... -// // q=0 -// fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; -// dist[n] = fq; -// -// // q = 1 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); -// //nread = neighborList[n+Np]; -// dist[nr2] = fq; -// -// // q=2 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); -// //nread = neighborList[n]; -// dist[nr1] = fq; -// -// // q = 3 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); -// //nread = neighborList[n+3*Np]; -// dist[nr4] = fq; -// -// // q = 4 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); -// //nread = neighborList[n+2*Np]; -// dist[nr3] = fq; -// -// // q = 5 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); -// //nread = neighborList[n+5*Np]; -// dist[nr6] = fq; -// -// // q = 6 -// fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); -// //nread = neighborList[n+4*Np]; -// dist[nr5] = fq; -// -// // q = 7 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); -// nread = neighborList[n+7*Np]; -// dist[nread] = fq; -// -// // q = 8 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); -// nread = neighborList[n+6*Np]; -// dist[nread] = fq; -// -// // q = 9 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); -// nread = neighborList[n+9*Np]; -// dist[nread] = fq; -// -// // q = 10 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); -// nread = neighborList[n+8*Np]; -// dist[nread] = fq; -// -// // q = 11 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); -// nread = neighborList[n+11*Np]; -// dist[nread] = fq; -// -// // q = 12 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); -// nread = neighborList[n+10*Np]; -// dist[nread]= fq; -// -// // q = 13 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); -// nread = neighborList[n+13*Np]; -// dist[nread] = fq; -// -// // q= 14 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); -// nread = neighborList[n+12*Np]; -// dist[nread] = fq; -// -// // q = 15 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); -// nread = neighborList[n+15*Np]; -// dist[nread] = fq; -// -// // q = 16 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); -// nread = neighborList[n+14*Np]; -// dist[nread] = fq; -// -// // q = 17 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); -// nread = neighborList[n+17*Np]; -// dist[nread] = fq; -// -// // q = 18 -// fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); -// nread = neighborList[n+16*Np]; -// dist[nread] = fq; -// //........................................................................ -// -// //Update velocity on device -// Velocity[0*Np+n] = ux; -// Velocity[1*Np+n] = uy; -// Velocity[2*Np+n] = uz; -// //Update pressure on device -// Pressure[n] = pressure; -// -// //-----------------------Mass transport------------------------// -// // calcuale chemical potential -// chem_a = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-0.25*kappaA*phi_lap; -// chem_b = -lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*kappaB*phi_lap; -// rlx_massA = 3.f-sqrt(3.f); -// rlx_massB = 3.f-sqrt(3.f); -// -// //............................................... -// // q = 0,2,4 -// // Cq = {1,0,0}, {0,1,0}, {0,0,1} -// a1 = Aq[nr2]; -// b1 = Bq[nr2]; -// a2 = Aq[nr1]; -// b2 = Bq[nr1]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*ux)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*ux)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*ux)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*ux)); -// -// // q = 1 -// //nread = neighborList[n+Np]; -// Aq[nr2] = a1; -// Bq[nr2] = b1; -// // q=2 -// //nread = neighborList[n]; -// Aq[nr1] = a2; -// Bq[nr1] = b2; -// -// //............................................... -// // Cq = {0,1,0} -// a1 = Aq[nr4]; -// b1 = Bq[nr4]; -// a2 = Aq[nr3]; -// b2 = Bq[nr3]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uy)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uy)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uy)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uy)); -// -// // q = 3 -// //nread = neighborList[n+3*Np]; -// Aq[nr4] = a1; -// Bq[nr4] = b1; -// // q = 4 -// //nread = neighborList[n+2*Np]; -// Aq[nr3] = a2; -// Bq[nr3] = b2; -// -// //............................................... -// // q = 4 -// // Cq = {0,0,1} -// a1 = Aq[nr6]; -// b1 = Bq[nr6]; -// a2 = Aq[nr5]; -// b2 = Bq[nr5]; -// a1 = (1.0-rlx_massA)*a1+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a+nA*uz)); -// b1 = (1.0-rlx_massB)*b1+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b+nB*uz)); -// a2 = (1.0-rlx_massA)*a2+rlx_massA*(0.1111111111111111*4.5*(gamma*chem_a-nA*uz)); -// b2 = (1.0-rlx_massB)*b2+rlx_massB*(0.1111111111111111*4.5*(gamma*chem_b-nB*uz)); -// -// // q = 5 -// //nread = neighborList[n+5*Np]; -// Aq[nr6] = a1; -// Bq[nr6] = b1; -// // q = 6 -// //nread = neighborList[n+4*Np]; -// Aq[nr5] = a2; -// Bq[nr5] = b2; -// //............................................... -// -// // Instantiate mass transport distributions -// // Stationary value - distribution 0 -// a1=Aq[n]; -// b1=Bq[n]; -// Aq[n] = (1.0-rlx_massA)*a1+rlx_massA*(nA-3.0*gamma*chem_a); -// Bq[n] = (1.0-rlx_massB)*b1+rlx_massB*(nB-3.0*gamma*chem_b); -// -// -// } -// } -//} - -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ @@ -2678,15 +1351,13 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) double tau,tau_eff,rlx_setA,rlx_setB; double mu_eff;//effective kinematic viscosity for Darcy term - double rho0; + double rho,rho0; double phi; double phi_lap;//laplacian of phase field double nA,nB; - //double a1,b1,a2,b2; double Gfs_x,Gfs_y,Gfs_z; double Gff_x,Gff_y,Gff_z; double chem; - //double rlx_massA,rlx_massB; double rlx_phi; double a1,a2;//PDF of phase field // *---------------------------------Pressure Tensor Gradient------------------------------------*// @@ -2716,11 +1387,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou if ( n 10Np => odd part of dist) fq = dist[nr1]; // reading the f1 data into register fq - pressure = fq; + rho += fq; m1 -= 11.0*fq; m2 -= 4.0*fq; jx = fq; @@ -2804,7 +1475,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=2 nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) fq = dist[nr2]; // reading the f2 data into register fq - pressure += fq; + rho += fq; m1 -= 11.0*(fq); m2 -= 4.0*(fq); jx -= fq; @@ -2815,7 +1486,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=3 nr3 = neighborList[n+2*Np]; // neighbor 4 fq = dist[nr3]; - pressure += fq; + rho += fq; m1 -= 11.0*fq; m2 -= 4.0*fq; jy = fq; @@ -2828,7 +1499,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q = 4 nr4 = neighborList[n+3*Np]; // neighbor 3 fq = dist[nr4]; - pressure += fq; + rho += fq; m1 -= 11.0*fq; m2 -= 4.0*fq; jy -= fq; @@ -2841,7 +1512,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=5 nr5 = neighborList[n+4*Np]; fq = dist[nr5]; - pressure += fq; + rho += fq; m1 -= 11.0*fq; m2 -= 4.0*fq; jz = fq; @@ -2854,7 +1525,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q = 6 nr6 = neighborList[n+5*Np]; fq = dist[nr6]; - pressure += fq; + rho += fq; m1 -= 11.0*fq; m2 -= 4.0*fq; jz -= fq; @@ -2867,7 +1538,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=7 nread = neighborList[n+6*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx += fq; @@ -2885,7 +1556,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q = 8 nread = neighborList[n+7*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx -= fq; @@ -2903,7 +1574,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=9 nread = neighborList[n+8*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx += fq; @@ -2921,7 +1592,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q = 10 nread = neighborList[n+9*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx -= fq; @@ -2939,7 +1610,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=11 nread = neighborList[n+10*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx += fq; @@ -2957,7 +1628,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=12 nread = neighborList[n+11*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx -= fq; @@ -2975,7 +1646,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=13 nread = neighborList[n+12*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx += fq; @@ -2993,7 +1664,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=14 nread = neighborList[n+13*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jx -= fq; @@ -3011,7 +1682,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=15 nread = neighborList[n+14*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jy += fq; @@ -3027,7 +1698,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=16 nread = neighborList[n+15*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jy -= fq; @@ -3043,7 +1714,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=17 nread = neighborList[n+16*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jy += fq; @@ -3059,7 +1730,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // q=18 nread = neighborList[n+17*Np]; fq = dist[nread]; - pressure += fq; + rho += fq; m1 += 8.0*fq; m2 += fq; jy -= fq; @@ -3099,159 +1770,127 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou } //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + pressure=rho/3.0; -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); + m10 = m10 + rlx_setA*( - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*( - m12); + m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); //....................................................................................................... //.................inverse transformation...................................................... // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; dist[n] = fq; // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); //nread = neighborList[n+Np]; dist[nr2] = fq; // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); //nread = neighborList[n]; dist[nr1] = fq; // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); //nread = neighborList[n+3*Np]; dist[nr4] = fq; // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); //nread = neighborList[n+2*Np]; dist[nr3] = fq; // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); //nread = neighborList[n+5*Np]; dist[nr6] = fq; // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); //nread = neighborList[n+4*Np]; dist[nr5] = fq; // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); nread = neighborList[n+7*Np]; dist[nread] = fq; // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); nread = neighborList[n+6*Np]; dist[nread] = fq; // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); nread = neighborList[n+9*Np]; dist[nread] = fq; // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); nread = neighborList[n+8*Np]; dist[nread] = fq; // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); nread = neighborList[n+11*Np]; dist[nread] = fq; // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); nread = neighborList[n+10*Np]; dist[nread]= fq; // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); nread = neighborList[n+13*Np]; dist[nread] = fq; // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); nread = neighborList[n+12*Np]; dist[nread] = fq; // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); nread = neighborList[n+15*Np]; dist[nread] = fq; // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); nread = neighborList[n+14*Np]; dist[nread] = fq; // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); nread = neighborList[n+17*Np]; dist[nread] = fq; // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); nread = neighborList[n+16*Np]; dist[nread] = fq; //........................................................................ @@ -3265,17 +1904,19 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //-----------------------Mass transport------------------------// // calcuale chemical potential - chem = lambdaA*(nA*nA*nA-1.5*nA*nA+0.5*nA)-lambdaB*(nB*nB*nB-1.5*nB*nB+0.5*nB)-0.25*(kappaA+kappaB)*phi_lap; + chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; //rlx_phi = 3.f-sqrt(3.f); rlx_phi = 1.0; //............................................... // q = 0,2,4 // Cq = {1,0,0}, {0,1,0}, {0,0,1} - a1 = Cq[nr2]; - a2 = Cq[nr1]; - a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); - a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); + //a1 = Cq[nr2]; + //a2 = Cq[nr1]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); // q = 1 //nread = neighborList[n+Np]; @@ -3286,10 +1927,12 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //............................................... // Cq = {0,1,0} - a1 = Cq[nr4]; - a2 = Cq[nr3]; - a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); - a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); + //a1 = Cq[nr4]; + //a2 = Cq[nr3]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); // q = 3 //nread = neighborList[n+3*Np]; @@ -3301,10 +1944,12 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou //............................................... // q = 4 // Cq = {0,0,1} - a1 = Cq[nr6]; - a2 = Cq[nr5]; - a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); - a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); + //a1 = Cq[nr6]; + //a2 = Cq[nr5]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); // q = 5 //nread = neighborList[n+5*Np]; @@ -3316,14 +1961,15 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou // Instantiate mass transport distributions // Stationary value - distribution 0 - a1=Cq[n]; - Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(a1-3.0*gamma*chem); + //a1=Cq[n]; + //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); + Cq[n] = phi-3.0*gamma*chem; } } } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ @@ -3344,15 +1990,13 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) double tau,tau_eff,rlx_setA,rlx_setB; double mu_eff;//effective kinematic viscosity for Darcy term - double rho0; + double rho,rho0; double phi; double phi_lap;//laplacian of phase field double nA,nB; - //double a1,b1,a2,b2; double Gfs_x,Gfs_y,Gfs_z; double Gff_x,Gff_y,Gff_z; double chem; - //double rlx_massA,rlx_massB; double rlx_phi; double a1,a2;//PDF of phase field // *---------------------------------Pressure Tensor Gradient------------------------------------*// @@ -3384,12 +2028,10 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double if ( n>>(dist, Aq, Bq, Den, SolidForce, start, finish, Np, -// tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); -// -// cudaError_t err = cudaGetLastError(); -// if (cudaSuccess != err){ -// printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); -// } -//} -// -//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,double *SolidForce, int start, int finish, int Np, -// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, -// double Gx, double Gy, double Gz, -// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ -// -// dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Aq, Bq, Den, SolidForce, start, finish, Np, -// tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, -// Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); -// -// cudaError_t err = cudaGetLastError(); -// if (cudaSuccess != err){ -// printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); -// } -//} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Cq, Phi, Den, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Cq, Phi, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); @@ -4682,12 +3232,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *Den,double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Cq, Phi, Den, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Cq, Phi, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); @@ -4697,16 +3247,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double } } -//extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Aq, double *Bq, double *Phi, int start, int finish, int Np){ -// dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Aq, Bq, Phi, start, finish, Np); -// cudaError_t err = cudaGetLastError(); -// if (cudaSuccess != err){ -// printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); -// } -//} - -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Den, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); +extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Phi, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); @@ -4740,9 +3282,9 @@ extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, } } -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Den, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi<<>>(NeighborList, Cq, Den, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi<<>>(NeighborList, Cq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -4750,9 +3292,9 @@ extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *C } } -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Den, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi<<>>(Cq, Den, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi<<>>(Cq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorPhi: %s \n",cudaGetErrorString(err)); @@ -4786,9 +3328,9 @@ extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, } } -extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi, double *PressTensor, double *PhiLap, +extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor<<>>(neighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); + dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor<<>>(neighborList,Phi,Pressure,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_PressureTensor: %s \n",cudaGetErrorString(err)); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 1680939b..1827b961 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -74,9 +74,6 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ if (greyscaleColor_db->keyExists( "rhoB" )){ rhoB = greyscaleColor_db->getScalar( "rhoB" ); } -// if (greyscaleColor_db->keyExists( "Gsc" )){ -// Gsc = greyscaleColor_db->getScalar( "Gsc" ); -// } if (greyscaleColor_db->keyExists( "gamma" )){ gamma = greyscaleColor_db->getScalar( "gamma" ); } @@ -477,8 +474,8 @@ void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); } - double *Den_temp; - Den_temp=new double [2*Np]; +// double *Den_temp; +// Den_temp=new double [2*Np]; double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. double nB=0.5; @@ -513,18 +510,18 @@ void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ phi = nA-nB; } int idx = Map(i,j,k); - Den_temp[idx+0*Np] = nA; - Den_temp[idx+1*Np] = nB; + //Den_temp[idx+0*Np] = nA; + //Den_temp[idx+1*Np] = nB; Phi_temp[idx] = phi; } } } } //copy to device - ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); + //ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); ScaLBL_DeviceBarrier(); - delete [] Den_temp; + //delete [] Den_temp; delete [] Phi_temp; } @@ -566,20 +563,20 @@ void ScaLBL_GreyscaleColorModel::Create(){ //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Cq, 7*sizeof(double)*Np);//phase field distribution ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PressureGrad, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Cq, 7*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PhiLap, sizeof(double)*Np);//laplacian of phase field ScaLBL_AllocateDeviceMemory((void **) &SolidForce, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PressTensor, 6*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &PressTensorGrad, 18*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); //ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); //ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); //ScaLBL_AllocateDeviceMemory((void **) &DenLapA, sizeof(double)*Np); @@ -613,6 +610,7 @@ void ScaLBL_GreyscaleColorModel::Create(){ void ScaLBL_GreyscaleColorModel::Initialize(){ if (Restart == true){ + //TODO: Restart funtion is currently not working; need updates if (rank==0){ printf("Initializing density field and distributions from Restart! \n"); } @@ -635,10 +633,10 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); + //ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); //TODO need to initialize velocity field ! //this is required for calculating the pressure_dvc @@ -651,20 +649,21 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyColorIMRT_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyColorIMRT_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); + //ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); + ScaLBL_D3Q19_Init(fq, Np); - //Velocity also needs initialization - if (rank==0) printf ("Initializing velocity field \n"); - double *vel_init; - vel_init = new double [3*Np]; - for (int i=0;i<3*Np;i++) vel_init[i]=0.0; - ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] vel_init; + //Velocity also needs initialization (for old incompressible momentum transport) + //if (rank==0) printf ("Initializing velocity field \n"); + //double *vel_init; + //vel_init = new double [3*Np]; + //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; + //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); + //ScaLBL_DeviceBarrier(); + //delete [] vel_init; } } @@ -714,18 +713,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ timestep++; // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) - //ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - //ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - //ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure - ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + //ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); // Compute pressure gradient ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); @@ -733,11 +729,13 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor - ScaLBL_Comm->SendHalo(Phi); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); - ScaLBL_Comm->RecvHalo(Phi); - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + //NOTE send and recv halo causes problems - it errorneously changes Phi + //ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + //ScaLBL_Comm->RecvHalo(Phi); + //ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + /* Compute gradient of the pressure tensor */ // call the recv Grad function once per tensor element // 1st tensor element @@ -771,22 +769,9 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); -// //compute Den gradients - component A -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); -// ScaLBL_Comm->SendHalo(&Den[0*Np]); -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); -// ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); -// //compute Den gradients - component B -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); -// ScaLBL_Comm->SendHalo(&Den[1*Np]); -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); -// ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL -// ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, -// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, Den, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -798,31 +783,27 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } -// ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, -// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, Den, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + // *************EVEN TIMESTEP*************// timestep++; // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) - //ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - //ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - //ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure - ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + //ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); // Compute pressure gradient ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); @@ -830,11 +811,12 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor - ScaLBL_Comm->SendHalo(Phi); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); - ScaLBL_Comm->RecvHalo(Phi); - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + //ScaLBL_Comm->SendHalo(Phi); + //NOTE send and recv halo causes problems - it errorneously changes Phi + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + //ScaLBL_Comm->RecvHalo(Phi); + //ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); /* Compute gradient of the pressure tensor */ // call the recv Grad function once per tensor element // 1st tensor element @@ -868,21 +850,9 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); -// //compute Den gradients - component A -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); -// ScaLBL_Comm->SendHalo(&Den[0*Np]); -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[0*Np], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); -// ScaLBL_Comm->RecvGrad(&Den[0*Np],DenGradA); -// //compute Den gradients - component B -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); -// ScaLBL_Comm->SendHalo(&Den[1*Np]); -// ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &Den[1*Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); -// ScaLBL_Comm->RecvGrad(&Den[1*Np],DenGradB); ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL -// ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, -// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, Den, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -894,9 +864,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } -// ScaLBL_D3Q19_AAeven_GreyscaleColor(fq, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, -// tauA,tauB,tauA_eff,tauB_eff,rhoA,rhoB,Gsc,Fx,Fy,Fz,Porosity,Permeability,Velocity,Pressure_dvc); - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, Den, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -1219,19 +1187,20 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ // fwrite(PhaseField.data(),8,N,OUTFILE); // fclose(OUTFILE); // - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - FILE *AFILE; - sprintf(LocalRankFilename,"A.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - FILE *BFILE; - sprintf(LocalRankFilename,"B.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); FILE *PFILE; @@ -1261,6 +1230,14 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); + + ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); + FILE *PhiFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + PhiFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PhiFILE); + fclose(PhiFILE); + // ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); // FILE *POROS_FILE; // sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); From 4a1059ca27d1849e25a5de6a55eb40a0f92d10ae Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 24 Apr 2020 16:53:47 -0400 Subject: [PATCH 121/270] fix miscellaneous bugs and clean up the code --- models/GreyscaleModel.cpp | 51 +++++++-------------------------------- models/GreyscaleModel.h | 1 - 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 11d92c80..09e52601 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -199,9 +199,9 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm for (int idx=0; idxLastExterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - int n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n"); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); // initialize phi based on PhaseLabel (include solid component labels) double *Poros, *Perm; Poros = new double[Np]; - Perm = new double[Np]; + Perm = new double[Np]; AssignComponentLabels(Poros,Perm); ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); + delete [] Poros; + delete [] Perm; } @@ -595,11 +567,6 @@ void ScaLBL_GreyscaleModel::Run(){ } } } - //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - vax = Mask->Comm.sumReduce( vax_loc ); vay = Mask->Comm.sumReduce( vay_loc ); vaz = Mask->Comm.sumReduce( vaz_loc ); diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index a99925b1..3e883b16 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -62,7 +62,6 @@ public: signed char *id; int *NeighborList; - int *dvcMap; double *fq; double *Permeability;//grey voxel permeability double *Porosity; From 4c84ab8eb9ae86812b3da7d925fc9d6744d6a850 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 25 Apr 2020 17:01:01 -0400 Subject: [PATCH 122/270] GPU ONLY; clean up the two-phase greyscale code; rename the old greyscaleColor to greyscaleFE --- common/ScaLBL.h | 28 +-- gpu/GreyscaleColor.cu | 114 ++++++------ models/GreyscaleColorModel.cpp | 235 ++++++++++++------------ models/GreyscaleColorModel.h | 8 +- tests/lbpm_greyscaleColor_simulator.cpp | 18 +- 5 files changed, 198 insertions(+), 205 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 849b8252..7ebeffa5 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -74,46 +74,46 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis // GREYSCALE COLOR MODEL -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, double *Bq, double *Den, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Phi, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, double *Pressure, double rhoA,double rhoB, int Np); -extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, +extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); // MRT MODEL diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 4841e212..c6c35654 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -3,7 +3,7 @@ #define NBLOCKS 1024 #define NTHREADS 256 -__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Poros,double *Velocity, +__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Poros,double *Velocity, double *Pressure, double rhoA,double rhoB, int N){ int n; @@ -66,7 +66,7 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *D } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, double *Bq, double *Den, +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure){ @@ -675,7 +675,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, } } -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure){ @@ -1329,7 +1329,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double } } -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ @@ -1969,7 +1969,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, dou } } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ @@ -2561,7 +2561,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double } } -__global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ +__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ int n; int S = Np/NBLOCKS/NTHREADS + 1; double phi; @@ -2599,7 +2599,7 @@ __global__ void dvc_ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, d } } -__global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, +__global__ void dvc_ScaLBL_D3Q7_GreyscaleFE_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ int idx; //double nA,nB; @@ -2627,7 +2627,7 @@ __global__ void dvc_ScaLBL_D3Q7_GreyColorIMRT_Init(double *Phi, double *Cq, doub } } -__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *neighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *neighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ int n,nread; double fq,nA,nB; @@ -2694,7 +2694,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *neighborList, } } -__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ int n; double fq,nA,nB; @@ -2755,7 +2755,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double } } -__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *neighborList, double *Cq, double *Phi, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *neighborList, double *Cq, double *Phi, int start, int finish, int Np){ int n,nread; double fq,phi; @@ -2805,7 +2805,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *neighborList, doub } } -__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Phi, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np){ int n; double fq,phi; @@ -2849,7 +2849,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Ph } } -__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ int n,nn; // distributions @@ -2948,7 +2948,7 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, doub } } -__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ +__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ int n,nn; // distributions @@ -3042,9 +3042,9 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, dou } } -__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, +__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ - //**GreyscaleColor model related parameters: + //**GreyscaleFE model related parameters: //kappaA, kappaB: characterize interfacial tension //lambdaA, lambdaB: characterize bulk free energy //nA: concentration of liquid 1; @@ -3190,149 +3190,149 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborLis } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(double *dist, double *Aq, double *Bq, double *Den, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleFE<<>>(dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColor: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleFE: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleFE<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColor: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleFE: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColorChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColorChem<<>>(dist, Cq, Phi, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleFEChem<<>>(dist, Cq, Phi, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleFEChem: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColorChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, double Gx, double Gy, double Gz, double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColorChem<<>>(neighborList, dist, Cq, Phi, SolidForce, start, finish, Np, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleFEChem<<>>(neighborList, dist, Cq, Phi, SolidForce, start, finish, Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColorChem: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleFEChem: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q7_GreyColorIMRT_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_GreyColorIMRT_Init<<>>(Phi, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); +extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ + dvc_ScaLBL_D3Q7_GreyscaleFE_Init<<>>(Phi, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_GreyscaleFE_Init: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_GreyColorIMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ - dvc_ScaLBL_D3Q19_GreyColorIMRT_Init<<>>(dist,Den,rhoA,rhoB,Np); +extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ + dvc_ScaLBL_D3Q19_GreyscaleFE_IMRT_Init<<>>(dist,Den,rhoA,rhoB,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyColorIMRT_Init: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_IMRT_Init: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorDensity<<>>(NeighborList, Aq, Bq, Den, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEDensity<<>>(NeighborList, Aq, Bq, Den, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleColorDensity: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleFEDensity: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorDensity<<>>(Aq, Bq, Den, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEDensity<<>>(Aq, Bq, Den, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorDensity: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleFEDensity: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAodd_GreyscaleColorPhi<<>>(NeighborList, Cq, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEPhi<<>>(NeighborList, Cq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleColorPhi: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleFEPhi: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(double *Cq, double *Phi, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_AAeven_GreyscaleColorPhi<<>>(Cq, Phi, start, finish, Np); + dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEPhi<<>>(Cq, Phi, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleColorPhi: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleFEPhi: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleColor_Gradient<<>>(neighborList, Den, DenGrad, start, finish, Np); + dvc_ScaLBL_D3Q19_GreyscaleFE_Gradient<<>>(neighborList, Den, DenGrad, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Gradient: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Gradient: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleColor_Laplacian<<>>(neighborList, Den, DenLap, start, finish, Np); +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ + dvc_ScaLBL_D3Q19_GreyscaleFE_Laplacian<<>>(neighborList, Den, DenLap, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Laplacian: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Laplacian: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_GreyscaleColor_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, double *Pressure, double rhoA,double rhoB, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleColor_Pressure<<>>(dist, Den, Porosity, Velocity, Pressure, rhoA, rhoB, Np); + dvc_ScaLBL_D3Q19_GreyscaleFE_Pressure<<>>(dist, Den, Porosity, Velocity, Pressure, rhoA, rhoB, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Pressure: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Pressure: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_GreyscaleColor_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, +extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleColor_PressureTensor<<>>(neighborList,Phi,Pressure,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); + dvc_ScaLBL_D3Q19_GreyscaleFE_PressureTensor<<>>(neighborList,Phi,Pressure,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_PressureTensor: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_PressureTensor: %s \n",cudaGetErrorString(err)); } } diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 1827b961..0cce81ec 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -13,7 +13,7 @@ void DeleteArray( const TYPE *p ) delete [] p; } -ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): +ScaLBL_GreyscaleFEModel::ScaLBL_GreyscaleFEModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0), rhoA(0),rhoB(0),gamma(0),kappaA(0),kappaB(0),lambdaA(0),lambdaB(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), @@ -23,15 +23,15 @@ Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0), SignDist.fill(0); } -ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ +ScaLBL_GreyscaleFEModel::~ScaLBL_GreyscaleFEModel(){ } -void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ +void ScaLBL_GreyscaleFEModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); - greyscaleColor_db = db->getDatabase( "GreyscaleColor" ); + greyscaleFE_db = db->getDatabase( "GreyscaleFE" ); analysis_db = db->getDatabase( "Analysis" ); vis_db = db->getDatabase( "Visualization" ); @@ -57,62 +57,62 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ lambdaB = 1.0e-3; // ---------------------- Greyscale Model parameters -----------------------// - if (greyscaleColor_db->keyExists( "timestepMax" )){ - timestepMax = greyscaleColor_db->getScalar( "timestepMax" ); + if (greyscaleFE_db->keyExists( "timestepMax" )){ + timestepMax = greyscaleFE_db->getScalar( "timestepMax" ); } - if (greyscaleColor_db->keyExists( "tauA" )){ - tauA = greyscaleColor_db->getScalar( "tauA" ); + if (greyscaleFE_db->keyExists( "tauA" )){ + tauA = greyscaleFE_db->getScalar( "tauA" ); } - if (greyscaleColor_db->keyExists( "tauB" )){ - tauB = greyscaleColor_db->getScalar( "tauB" ); + if (greyscaleFE_db->keyExists( "tauB" )){ + tauB = greyscaleFE_db->getScalar( "tauB" ); } - tauA_eff = greyscaleColor_db->getWithDefault( "tauA_eff", tauA); - tauB_eff = greyscaleColor_db->getWithDefault( "tauB_eff", tauB); - if (greyscaleColor_db->keyExists( "rhoA" )){ - rhoA = greyscaleColor_db->getScalar( "rhoA" ); + tauA_eff = greyscaleFE_db->getWithDefault( "tauA_eff", tauA); + tauB_eff = greyscaleFE_db->getWithDefault( "tauB_eff", tauB); + if (greyscaleFE_db->keyExists( "rhoA" )){ + rhoA = greyscaleFE_db->getScalar( "rhoA" ); } - if (greyscaleColor_db->keyExists( "rhoB" )){ - rhoB = greyscaleColor_db->getScalar( "rhoB" ); + if (greyscaleFE_db->keyExists( "rhoB" )){ + rhoB = greyscaleFE_db->getScalar( "rhoB" ); } - if (greyscaleColor_db->keyExists( "gamma" )){ - gamma = greyscaleColor_db->getScalar( "gamma" ); + if (greyscaleFE_db->keyExists( "gamma" )){ + gamma = greyscaleFE_db->getScalar( "gamma" ); } - if (greyscaleColor_db->keyExists( "kappaA" )){ - kappaA = greyscaleColor_db->getScalar( "kappaA" ); + if (greyscaleFE_db->keyExists( "kappaA" )){ + kappaA = greyscaleFE_db->getScalar( "kappaA" ); } - if (greyscaleColor_db->keyExists( "kappaB" )){ - kappaB = greyscaleColor_db->getScalar( "kappaB" ); + if (greyscaleFE_db->keyExists( "kappaB" )){ + kappaB = greyscaleFE_db->getScalar( "kappaB" ); } - if (greyscaleColor_db->keyExists( "lambdaA" )){ - lambdaA = greyscaleColor_db->getScalar( "lambdaA" ); + if (greyscaleFE_db->keyExists( "lambdaA" )){ + lambdaA = greyscaleFE_db->getScalar( "lambdaA" ); } - if (greyscaleColor_db->keyExists( "lambdaB" )){ - lambdaB = greyscaleColor_db->getScalar( "lambdaB" ); + if (greyscaleFE_db->keyExists( "lambdaB" )){ + lambdaB = greyscaleFE_db->getScalar( "lambdaB" ); } - if (greyscaleColor_db->keyExists( "dp" )){ - dp = greyscaleColor_db->getScalar( "dp" ); + if (greyscaleFE_db->keyExists( "dp" )){ + dp = greyscaleFE_db->getScalar( "dp" ); } - if (greyscaleColor_db->keyExists( "F" )){ - Fx = greyscaleColor_db->getVector( "F" )[0]; - Fy = greyscaleColor_db->getVector( "F" )[1]; - Fz = greyscaleColor_db->getVector( "F" )[2]; + if (greyscaleFE_db->keyExists( "F" )){ + Fx = greyscaleFE_db->getVector( "F" )[0]; + Fy = greyscaleFE_db->getVector( "F" )[1]; + Fz = greyscaleFE_db->getVector( "F" )[2]; } - if (greyscaleColor_db->keyExists( "Restart" )){ - Restart = greyscaleColor_db->getScalar( "Restart" ); + if (greyscaleFE_db->keyExists( "Restart" )){ + Restart = greyscaleFE_db->getScalar( "Restart" ); } - if (greyscaleColor_db->keyExists( "din" )){ - din = greyscaleColor_db->getScalar( "din" ); + if (greyscaleFE_db->keyExists( "din" )){ + din = greyscaleFE_db->getScalar( "din" ); } - if (greyscaleColor_db->keyExists( "dout" )){ - dout = greyscaleColor_db->getScalar( "dout" ); + if (greyscaleFE_db->keyExists( "dout" )){ + dout = greyscaleFE_db->getScalar( "dout" ); } - if (greyscaleColor_db->keyExists( "flux" )){ - flux = greyscaleColor_db->getScalar( "flux" ); + if (greyscaleFE_db->keyExists( "flux" )){ + flux = greyscaleFE_db->getScalar( "flux" ); } - if (greyscaleColor_db->keyExists( "tolerance" )){ - tolerance = greyscaleColor_db->getScalar( "tolerance" ); + if (greyscaleFE_db->keyExists( "tolerance" )){ + tolerance = greyscaleFE_db->getScalar( "tolerance" ); } - //auto collision = greyscaleColor_db->getWithDefault( "collision", "IMRT" ); + //auto collision = greyscaleFE_db->getWithDefault( "collision", "IMRT" ); //if (collision == "BGK"){ // CollisionType=2; //} @@ -126,7 +126,7 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ // ------------------------------------------------------------------------// } -void ScaLBL_GreyscaleColorModel::SetDomain(){ +void ScaLBL_GreyscaleFEModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases // domain parameters @@ -157,7 +157,7 @@ void ScaLBL_GreyscaleColorModel::SetDomain(){ nprocz = Dm->nprocz(); } -void ScaLBL_GreyscaleColorModel::ReadInput(){ +void ScaLBL_GreyscaleFEModel::ReadInput(){ sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); @@ -206,7 +206,7 @@ void ScaLBL_GreyscaleColorModel::ReadInput(){ if (rank == 0) cout << "Domain set." << endl; } -void ScaLBL_GreyscaleColorModel::AssignSolidForce(double *SolidPotential, double *SolidForce){ +void ScaLBL_GreyscaleFEModel::AssignSolidForce(double *SolidPotential, double *SolidForce){ double *Dst; Dst = new double [3*3*3]; @@ -303,7 +303,7 @@ void ScaLBL_GreyscaleColorModel::AssignSolidForce(double *SolidPotential, double } -void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double *Permeability, double *SolidPotential) +void ScaLBL_GreyscaleFEModel::AssignComponentLabels(double *Porosity, double *Permeability, double *SolidPotential) { size_t NLABELS=0; signed char VALUE=0; @@ -311,10 +311,10 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double double PERMEABILITY=0.f; double AFFINITY=0.f; - auto LabelList = greyscaleColor_db->getVector( "ComponentLabels" ); - auto AffinityList = greyscaleColor_db->getVector( "ComponentAffinity" ); - auto PorosityList = greyscaleColor_db->getVector( "PorosityList" ); - auto PermeabilityList = greyscaleColor_db->getVector( "PermeabilityList" ); + auto LabelList = greyscaleFE_db->getVector( "ComponentLabels" ); + auto AffinityList = greyscaleFE_db->getVector( "ComponentAffinity" ); + auto PorosityList = greyscaleFE_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscaleFE_db->getVector( "PermeabilityList" ); //1. Requirement for "ComponentLabels": // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 @@ -452,7 +452,7 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels(double *Porosity, double } } -void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ +void ScaLBL_GreyscaleFEModel::Density_and_Phase_Init(){ size_t NLABELS=0; signed char VALUE=0; @@ -460,13 +460,13 @@ void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ vector LabelList{1,2}; vector SwList{0.0,1.0}; - if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ + if (greyscaleFE_db->keyExists( "GreyNodeLabels" )){ LabelList.clear(); - LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); + LabelList = greyscaleFE_db->getVector( "GreyNodeLabels" ); } - if (greyscaleColor_db->keyExists( "GreyNodeSw" )){ + if (greyscaleFE_db->keyExists( "GreyNodeSw" )){ SwList.clear(); - SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); + SwList = greyscaleFE_db->getVector( "GreyNodeSw" ); } NLABELS=LabelList.size(); @@ -525,7 +525,7 @@ void ScaLBL_GreyscaleColorModel::Density_and_Phase_Init(){ delete [] Phi_temp; } -void ScaLBL_GreyscaleColorModel::Create(){ +void ScaLBL_GreyscaleFEModel::Create(){ /* * This function creates the variables needed to run a LBM */ @@ -608,7 +608,7 @@ void ScaLBL_GreyscaleColorModel::Create(){ } -void ScaLBL_GreyscaleColorModel::Initialize(){ +void ScaLBL_GreyscaleFEModel::Initialize(){ if (Restart == true){ //TODO: Restart funtion is currently not working; need updates if (rank==0){ @@ -631,12 +631,7 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); - //ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //TODO need proper initialization ! //TODO need to initialize velocity field ! //this is required for calculating the pressure_dvc @@ -645,12 +640,10 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ else{ if (rank==0) printf ("Initializing density field \n"); Density_and_Phase_Init();//initialize density field - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - //ScaLBL_D3Q7_GreyColorIMRT_Init(Den, Aq, Bq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q19_GreyscaleColor_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyColorIMRT_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyColorIMRT_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_GreyscaleFE_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components + ScaLBL_D3Q7_GreyscaleFE_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); if (rank==0) printf ("Initializing distributions \n"); //ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); @@ -667,7 +660,7 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ } } -void ScaLBL_GreyscaleColorModel::Run(){ +void ScaLBL_GreyscaleFEModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -683,8 +676,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ if (analysis_db->keyExists( "restart_interval" )){ restart_interval = analysis_db->getScalar( "restart_interval" ); } - if (greyscaleColor_db->keyExists( "timestep" )){ - timestep = greyscaleColor_db->getScalar( "timestep" ); + if (greyscaleFE_db->keyExists( "timestep" )){ + timestep = greyscaleFE_db->getScalar( "timestep" ); } if (rank==0){ @@ -714,64 +707,64 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(NeighborList, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAodd_GreyscaleColorPhi(NeighborList, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(NeighborList, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure - //ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + //ScaLBL_D3Q19_GreyscaleFE_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); // Compute pressure gradient - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor //NOTE send and recv halo causes problems - it errorneously changes Phi //ScaLBL_Comm->SendHalo(Phi); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); //ScaLBL_Comm->RecvHalo(Phi); //ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); /* Compute gradient of the pressure tensor */ // call the recv Grad function once per tensor element // 1st tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); // 2nd tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); // 3rd tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); // 4th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); // 5th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); // 6th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleFEChem(NeighborList, fq, Cq, Phi, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -783,7 +776,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } - ScaLBL_D3Q19_AAodd_GreyscaleColorChem(NeighborList, fq, Cq, Phi, SolidForce, + ScaLBL_D3Q19_AAodd_GreyscaleFEChem(NeighborList, fq, Cq, Phi, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -796,63 +789,63 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAeven_GreyscaleColorPhi(Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Update local pressure - //ScaLBL_D3Q19_GreyscaleColor_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); + //ScaLBL_D3Q19_GreyscaleFE_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); // Compute pressure gradient - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(Pressure_dvc); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); ScaLBL_DeviceBarrier(); // Compute Pressure Tensor //ScaLBL_Comm->SendHalo(Phi); //NOTE send and recv halo causes problems - it errorneously changes Phi - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); + ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); //ScaLBL_Comm->RecvHalo(Phi); //ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleColor_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); + ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); /* Compute gradient of the pressure tensor */ // call the recv Grad function once per tensor element // 1st tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); // 2nd tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); // 3rd tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); // 4th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); // 5th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); // 6th tensor element - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleFEChem(fq, Cq, Phi, SolidForce, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -864,7 +857,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); // } - ScaLBL_D3Q19_AAeven_GreyscaleColorChem(fq, Cq, Phi, SolidForce, + ScaLBL_D3Q19_AAeven_GreyscaleFEChem(fq, Cq, Phi, SolidForce, 0, ScaLBL_Comm->LastExterior(), Np, tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); @@ -1012,9 +1005,9 @@ void ScaLBL_GreyscaleColorModel::Run(){ if (timestep%restart_interval==0){ //Use rank=0 write out Restart.db if (rank==0) { - greyscaleColor_db->putScalar("timestep",timestep); - greyscaleColor_db->putScalar( "Restart", true ); - current_db->putDatabase("GreyscaleColor", greyscaleColor_db); + greyscaleFE_db->putScalar("timestep",timestep); + greyscaleFE_db->putScalar( "Restart", true ); + current_db->putDatabase("GreyscaleFE", greyscaleFE_db); std::ofstream OutStream("Restart.db"); current_db->print(OutStream, ""); OutStream.close(); @@ -1038,7 +1031,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ } PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_greyscaleColor_simulator",1); + PROFILE_SAVE("lbpm_greyscaleFE_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -1059,7 +1052,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ************************************************************************ } -void ScaLBL_GreyscaleColorModel::VelocityField(){ +void ScaLBL_GreyscaleFEModel::VelocityField(){ /* Minkowski Morphology(Mask); int SIZE=Np*sizeof(double); @@ -1175,7 +1168,7 @@ void ScaLBL_GreyscaleColorModel::VelocityField(){ } -void ScaLBL_GreyscaleColorModel::WriteDebug(){ +void ScaLBL_GreyscaleFEModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 8b0cd85e..f98d1b9b 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -16,10 +16,10 @@ Implementation of multicomponent greyscale lattice boltzmann model #include "ProfilerApp.h" #include "threadpool/thread_pool.h" -class ScaLBL_GreyscaleColorModel{ +class ScaLBL_GreyscaleFEModel{ public: - ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleColorModel(); + ScaLBL_GreyscaleFEModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleFEModel(); // functions in they should be run void ReadParams(string filename); @@ -59,7 +59,7 @@ public: // input database std::shared_ptr db; std::shared_ptr domain_db; - std::shared_ptr greyscaleColor_db; + std::shared_ptr greyscaleFE_db; std::shared_ptr analysis_db; std::shared_ptr vis_db; diff --git a/tests/lbpm_greyscaleColor_simulator.cpp b/tests/lbpm_greyscaleColor_simulator.cpp index b836e03c..41c08774 100644 --- a/tests/lbpm_greyscaleColor_simulator.cpp +++ b/tests/lbpm_greyscaleColor_simulator.cpp @@ -41,16 +41,16 @@ int main(int argc, char **argv) ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm); + ScaLBL_GreyscaleFEModel GreyscaleFE(rank,nprocs,comm); auto filename = argv[1]; - GreyscaleColor.ReadParams(filename); - GreyscaleColor.SetDomain(); // this reads in the domain - GreyscaleColor.ReadInput(); - GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables - GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables - GreyscaleColor.Run(); - //GreyscaleColor.VelocityField(); - GreyscaleColor.WriteDebug(); + GreyscaleFE.ReadParams(filename); + GreyscaleFE.SetDomain(); // this reads in the domain + GreyscaleFE.ReadInput(); + GreyscaleFE.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleFE.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleFE.Run(); + //GreyscaleFE.VelocityField(); + GreyscaleFE.WriteDebug(); } // **************************************************** MPI_Barrier(comm); From d5ccfb628d9ed9f3d5f9c7999a905130cbd3df9c Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 25 Apr 2020 17:01:59 -0400 Subject: [PATCH 123/270] rename the two-phase FE-based greyscale model --- models/{GreyscaleColorModel.cpp => GreyscaleFEModel.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/{GreyscaleColorModel.cpp => GreyscaleFEModel.cpp} (100%) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleFEModel.cpp similarity index 100% rename from models/GreyscaleColorModel.cpp rename to models/GreyscaleFEModel.cpp From 0d3436047a9bb750b4138e178d71a37a650fb1e8 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 25 Apr 2020 17:02:43 -0400 Subject: [PATCH 124/270] rename the two-phase FE-based greyscale mode; step 2 --- models/{GreyscaleColorModel.h => GreyscaleFEModel.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/{GreyscaleColorModel.h => GreyscaleFEModel.h} (100%) diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleFEModel.h similarity index 100% rename from models/GreyscaleColorModel.h rename to models/GreyscaleFEModel.h From 86c3217a0fddf24e1a536e91260e64a53d887003 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 25 Apr 2020 17:03:17 -0400 Subject: [PATCH 125/270] rename the two-phase FE-based greyscale model; step 3 --- ...reyscaleColor_simulator.cpp => lbpm_greyscaleFE_simulator.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{lbpm_greyscaleColor_simulator.cpp => lbpm_greyscaleFE_simulator.cpp} (100%) diff --git a/tests/lbpm_greyscaleColor_simulator.cpp b/tests/lbpm_greyscaleFE_simulator.cpp similarity index 100% rename from tests/lbpm_greyscaleColor_simulator.cpp rename to tests/lbpm_greyscaleFE_simulator.cpp From 5a8f4f60fcfb101919923b378b0254e9bda36aed Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 25 Apr 2020 17:12:43 -0400 Subject: [PATCH 126/270] GPU only;complete renaming everything about the old greyscaleColor model to greyscaleFE --- models/GreyscaleFEModel.cpp | 2 +- models/GreyscaleFEModel.h | 2 +- tests/CMakeLists.txt | 2 +- tests/lbpm_greyscaleFE_simulator.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/GreyscaleFEModel.cpp b/models/GreyscaleFEModel.cpp index 0cce81ec..97ed0f98 100644 --- a/models/GreyscaleFEModel.cpp +++ b/models/GreyscaleFEModel.cpp @@ -1,7 +1,7 @@ /* Greyscale lattice boltzmann model */ -#include "models/GreyscaleColorModel.h" +#include "models/GreyscaleFEModel.h" #include "analysis/distance.h" #include "analysis/morphology.h" #include diff --git a/models/GreyscaleFEModel.h b/models/GreyscaleFEModel.h index f98d1b9b..34a6f999 100644 --- a/models/GreyscaleFEModel.h +++ b/models/GreyscaleFEModel.h @@ -1,5 +1,5 @@ /* -Implementation of multicomponent greyscale lattice boltzmann model +Implementation of multicomponent greyscale free-energy based lattice boltzmann model */ #include #include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 56eda802..ca88274a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,7 +4,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_greyscaleFE_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_greyscaleFE_simulator.cpp b/tests/lbpm_greyscaleFE_simulator.cpp index 41c08774..22157bee 100644 --- a/tests/lbpm_greyscaleFE_simulator.cpp +++ b/tests/lbpm_greyscaleFE_simulator.cpp @@ -9,7 +9,7 @@ #include "common/ScaLBL.h" #include "common/Communication.h" #include "common/MPI.h" -#include "models/GreyscaleColorModel.h" +#include "models/GreyscaleFEModel.h" //#define WRITE_SURFACES using namespace std; From 6a37aa9e590b5bf8d5c02bd7f1628fdcc88087af Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 27 Apr 2020 11:03:40 -0400 Subject: [PATCH 127/270] rename greyscale FE model GPU code --- gpu/{GreyscaleColor.cu => GreyscaleFE.cu} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gpu/{GreyscaleColor.cu => GreyscaleFE.cu} (100%) diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleFE.cu similarity index 100% rename from gpu/GreyscaleColor.cu rename to gpu/GreyscaleFE.cu From 85158fab52a01336dda2c961823de78031eb2c6d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 27 Apr 2020 11:12:12 -0400 Subject: [PATCH 128/270] Initialize greyscale SC model --- gpu/GreyscaleSC.cu | 1671 +++++++++++++++++++++++++++++++++++ models/GreyscaleSCModel.cpp | 864 ++++++++++++++++++ models/GreyscaleSCModel.h | 91 ++ 3 files changed, 2626 insertions(+) create mode 100644 gpu/GreyscaleSC.cu create mode 100644 models/GreyscaleSCModel.cpp create mode 100644 models/GreyscaleSCModel.h diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu new file mode 100644 index 00000000..a3fba989 --- /dev/null +++ b/gpu/GreyscaleSC.cu @@ -0,0 +1,1671 @@ +#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity, double *Pressure){ + int n; + // conserved momemnts + double rho,vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure; + //double uu; + // non-conserved moments + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + // q=7 + nr7 = neighborList[n+6*Np]; + f7 = dist[nr7]; + + // q = 8 + nr8 = neighborList[n+7*Np]; + f8 = dist[nr8]; + + // q=9 + nr9 = neighborList[n+8*Np]; + f9 = dist[nr9]; + + // q = 10 + nr10 = neighborList[n+9*Np]; + f10 = dist[nr10]; + + // q=11 + nr11 = neighborList[n+10*Np]; + f11 = dist[nr11]; + + // q=12 + nr12 = neighborList[n+11*Np]; + f12 = dist[nr12]; + + // q=13 + nr13 = neighborList[n+12*Np]; + f13 = dist[nr13]; + + // q=14 + nr14 = neighborList[n+13*Np]; + f14 = dist[nr14]; + + // q=15 + nr15 = neighborList[n+14*Np]; + f15 = dist[nr15]; + + // q=16 + nr16 = neighborList[n+15*Np]; + f16 = dist[nr16]; + + // q=17 + //fq = dist[18*Np+n]; + nr17 = neighborList[n+16*Np]; + f17 = dist[nr17]; + + // q=18 + nr18 = neighborList[n+17*Np]; + f18 = dist[nr18]; + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + pressure = rho/porosity/3.0; + vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; + vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; + vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the body force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = -porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx; + Fy = -porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy; + Fz = -porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz; + if (porosity==1.0){ + Fx=Gx; + Fy=Gy; + Fz=Gz; + } + + //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// +// // q=0 +// dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 1 +// dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q=2 +// dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 3 +// dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 4 +// dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); +// +// // q = 5 +// dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); +// +// // q = 6 +// dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) +// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); +// +// // q = 7 +// dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 8 +// dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 9 +// dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 10 +// dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + +// Fz*(0. - (3.*uz)/porosity)); +// +// // q = 11 +// dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); +// +// // q = 12 +// dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + +// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 13 +// dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + +// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); +// +// // q= 14 +// dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); +// +// // q = 15 +// dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); +// +// // q = 16 +// dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + +// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 17 +// dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + +// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); +// +// // q = 18 +// dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) +// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + +// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + //----------------------------------------------------------------------------------------------------------------------------------------// + + + //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// + // q=0 + dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) + +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); + + // q = 7 + dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); + + // q = 8 + dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); + + // q = 9 + dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); + + // q = 10 + dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); + + // q = 11 + dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); + + // q = 12 + dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); + + // q = 13 + dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); + + // q= 14 + dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); + + // q = 15 + dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); + + // q = 16 + dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); + + // q = 17 + dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); + + // q = 18 + dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); + //-------------------------------------------------------------------------------------------------------------------------------------------// + + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity, double Den, double *Pressure){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + fq = dist[nread]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nread]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nread = neighborList[n+5*Np]; + fq = dist[nread]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/Den+0.5*porosity*Gx; + vy = jy/Den+0.5*porosity*Gy; + vz = jz/Den+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = Den*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = Den*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = Den*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=Den*Gx; + Fy=Den*Gy; + Fz=Den*Gz; + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + nread = neighborList[n+Np]; + dist[nread] = fq; + + // q=2 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + nread = neighborList[n]; + dist[nread] = fq; + + // q = 3 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + nread = neighborList[n+3*Np]; + dist[nread] = fq; + + // q = 4 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + nread = neighborList[n+2*Np]; + dist[nread] = fq; + + // q = 5 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + nread = neighborList[n+5*Np]; + dist[nread] = fq; + + // q = 6 + fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + nread = neighborList[n+4*Np]; + dist[nread] = fq; + + // q = 7 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den) +{ + int n; + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAodd_Greyscale<<>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT<<>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ + dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); + } +} diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp new file mode 100644 index 00000000..08cf09b5 --- /dev/null +++ b/models/GreyscaleSCModel.cpp @@ -0,0 +1,864 @@ +/* +Greyscale lattice boltzmann model + */ +#include "models/GreyscaleModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include +#include + +template +void DeleteArray( const TYPE *p ) +{ + delete [] p; +} + +ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + SignDist.resize(Nx,Ny,Nz); + SignDist.fill(0); + +} +ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ + +} + +void ScaLBL_GreyscaleModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + greyscale_db = db->getDatabase( "Greyscale" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tau = 1.0; + tau_eff = tau; + Den = 1.0;//constant density + tolerance = 0.01; + Fx = Fy = Fz = 0.0; + Restart=false; + din=dout=1.0; + flux=0.0; + dp = 10.0; //unit of 'dp': voxel + CollisionType = 1; //1: IMRT; 2: BGK + + // ---------------------- Greyscale Model parameters -----------------------// + if (greyscale_db->keyExists( "timestepMax" )){ + timestepMax = greyscale_db->getScalar( "timestepMax" ); + } + if (greyscale_db->keyExists( "tau" )){ + tau = greyscale_db->getScalar( "tau" ); + } + tau_eff = greyscale_db->getWithDefault( "tau_eff", tau ); + if (greyscale_db->keyExists( "Den" )){ + Den = greyscale_db->getScalar( "Den" ); + } + if (greyscale_db->keyExists( "dp" )){ + dp = greyscale_db->getScalar( "dp" ); + } + if (greyscale_db->keyExists( "F" )){ + Fx = greyscale_db->getVector( "F" )[0]; + Fy = greyscale_db->getVector( "F" )[1]; + Fz = greyscale_db->getVector( "F" )[2]; + } + if (greyscale_db->keyExists( "Restart" )){ + Restart = greyscale_db->getScalar( "Restart" ); + } + if (greyscale_db->keyExists( "din" )){ + din = greyscale_db->getScalar( "din" ); + } + if (greyscale_db->keyExists( "dout" )){ + dout = greyscale_db->getScalar( "dout" ); + } + if (greyscale_db->keyExists( "flux" )){ + flux = greyscale_db->getScalar( "flux" ); + } + if (greyscale_db->keyExists( "tolerance" )){ + tolerance = greyscale_db->getScalar( "tolerance" ); + } + auto collision = greyscale_db->getWithDefault( "collision", "IMRT" ); + if (collision == "BGK"){ + CollisionType=2; + } + // ------------------------------------------------------------------------// + + //------------------------ Other Domain parameters ------------------------// + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + // ------------------------------------------------------------------------// +} + +void ScaLBL_GreyscaleModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + + SignDist.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + PorosityMap.resize(Nx,Ny,Nz); + Pressure.resize(Nx,Ny,Nz); + + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_GreyscaleModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + int count = 0; + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kgetVector( "ComponentLabels" ); + auto PorosityList = greyscale_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + + NLABELS=LabelList.size(); + if (NLABELS != PorosityList.size()){ + ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Porosity[idx] = POROSITY; + } + } + } + } + } + + if (NLABELS != PermeabilityList.size()){ + ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); + } + for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + } + } + } + } + } + + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm.sumReduce(label_count[idx]); + + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); + } +} + + +void ScaLBL_GreyscaleModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + //......................................................... + // don't perform computations at the eight corners + //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device neighbor list \n"); + fflush(stdout); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + // initialize phi based on PhaseLabel (include solid component labels) + double *Poros, *Perm; + Poros = new double[Np]; + Perm = new double[Np]; + AssignComponentLabels(Poros,Perm); + ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); + delete [] Poros; + delete [] Perm; +} + + +void ScaLBL_GreyscaleModel::Initialize(){ + if (rank==0) printf ("Initializing distributions \n"); + //TODO: for BGK, you need to consider voxel porosity + // for IMRT, the whole set of feq is different + // if in the future you have different collison mode, need to write two set of initialization functions + if (CollisionType==1){ + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + if (rank==0) printf("Collision model: Incompressible MRT.\n"); + } + else if (CollisionType==2){ + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf("Collision model: BGK.\n"); + } + else{ + if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); + ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); + CollisionType=1; + greyscale_db->putScalar( "collision", "IMRT" ); + } + + if (Restart == true){ + if (rank==0){ + printf("Initializing distributions from Restart! \n"); + } + // Read in the restart file to CPU buffers + std::shared_ptr cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + FILE *File; + File=fopen(LocalRestartFile,"rb"); + fread(cfq.get(),sizeof(double),19*Np,File); + fclose(File); + + // Copy the restart data to the GPU + ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + + MPI_Barrier(comm); + } +} + +void ScaLBL_GreyscaleModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int visualization_interval = 1000; + int restart_interval = 10000; // number of timesteps in between in saving distributions for restart + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "visualization_interval" )){ + visualization_interval = analysis_db->getScalar( "visualization_interval" ); + } + if (analysis_db->keyExists( "restart_interval" )){ + restart_interval = analysis_db->getScalar( "restart_interval" ); + } + if (greyscale_db->keyExists( "timestep" )){ + timestep = greyscale_db->getScalar( "timestep" ); + } + + if (rank==0){ + printf("********************************************************\n"); + printf("No. of timesteps: %i \n", timestepMax); + fflush(stdout); + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); + //......................................... + + Minkowski Morphology(Mask); + + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + auto current_db = db->cloneDatabase(); + double rlx = 1.0/tau; + double rlx_eff = 1.0/tau_eff; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + // *************ODD TIMESTEP*************// + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + switch (CollisionType){ + case 1: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + case 2: + ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); + break; + default: + ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; + } + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + + if (timestep%analysis_interval==0){ + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); + //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + //double px_loc,py_loc,pz_loc; + //double px,py,pz; + //double mass_loc,mass_glb; + + //parameters for domain average + int64_t i,j,k,n,imin,jmin,kmin,kmax; + // If external boundary conditions are set, do not average over the inlet and outlet + kmin=1; kmax=Nz-1; + //In case user forgets to specify the inlet/outlet buffer layers for BC>0 + if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; + if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; + + imin=jmin=1; + // If inlet/outlet layers exist use these as default + //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; + //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; + if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer + if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; + +// px_loc = py_loc = pz_loc = 0.f; +// mass_loc = 0.f; +// for (int k=kmin; k 0){ +// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +// mass_loc += Den*PorosityMap(i,j,k); +// } +// } +// } +// } +// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// +// vax = px/mass_glb; +// vay = py/mass_glb; +// vaz = pz/mass_glb; + + vax_loc = vay_loc = vaz_loc = 0.f; + for (int k=kmin; k 0){ + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; + } + } + } + } + vax = Mask->Comm.sumReduce( vax_loc ); + vay = Mask->Comm.sumReduce( vay_loc ); + vaz = Mask->Comm.sumReduce( vaz_loc ); + count = Mask->Comm.sumReduce( count_loc ); + + vax /= count; + vay /= count; + vaz /= count; + + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + + error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); + flow_rate_previous = flow_rate; + + //if (rank==0) printf("Computing Minkowski functionals \n"); + Morphology.ComputeScalar(SignDist,0.f); + //Morphology.PrintAll(); + double mu = (tau-0.5)/3.f; + double Vs = Morphology.V(); + double As = Morphology.A(); + double Hs = Morphology.H(); + double Xs = Morphology.X(); + Vs = Dm->Comm.sumReduce( Vs); + As = Dm->Comm.sumReduce( As); + Hs = Dm->Comm.sumReduce( Hs); + Xs = Dm->Comm.sumReduce( Xs); + + double h = Dm->voxel_length; + //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; + + if (rank==0){ + printf(" AbsPerm = %.5g [micron^2]\n",absperm); + bool WriteHeader=false; + FILE * log_file = fopen("Permeability.csv","r"); + if (log_file != NULL) + fclose(log_file); + else + WriteHeader=true; + log_file = fopen("Permeability.csv","a"); + if (WriteHeader) + fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", + timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); + + fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, + h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); + fclose(log_file); + } + } + + if (timestep%visualization_interval==0){ + VelocityField(); + } + + if (timestep%restart_interval==0){ + //Use rank=0 write out Restart.db + if (rank==0) { + greyscale_db->putScalar("timestep",timestep); + greyscale_db->putScalar( "Restart", true ); + current_db->putDatabase("Greyscale", greyscale_db); + std::ofstream OutStream("Restart.db"); + current_db->print(OutStream, ""); + OutStream.close(); + + } + //Write out Restart data. + std::shared_ptr cfq; + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU + + FILE *RESTARTFILE; + RESTARTFILE=fopen(LocalRestartFile,"wb"); + fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); + fclose(RESTARTFILE); + MPI_Barrier(comm); + } + } + + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + +void ScaLBL_GreyscaleModel::VelocityField(){ + +/* Minkowski Morphology(Mask); + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); + + memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); + Morphology.Initialize(); + Morphology.UpdateMeshValues(); + Morphology.ComputeLocal(); + Morphology.Reduce(); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int n=0; nLastExterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + + for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double mu = (tau-0.5)/3.f; + if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); + if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, + Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); + */ + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + auto PressureVar = std::make_shared(); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + PressureVar->name = "Pressure"; + PressureVar->type = IO::VariableType::VolumeVariable; + PressureVar->dim = 1; + PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PressureVar); + + Array& SignData = visData[0].vars[0]->data; + Array& VelxData = visData[0].vars[1]->data; + Array& VelyData = visData[0].vars[2]->data; + Array& VelzData = visData[0].vars[3]->data; + Array& PressureData = visData[0].vars[4]->data; + + ASSERT(visData[0].vars[0]->name=="SignDist"); + ASSERT(visData[0].vars[1]->name=="Velocity_x"); + ASSERT(visData[0].vars[2]->name=="Velocity_y"); + ASSERT(visData[0].vars[3]->name=="Velocity_z"); + ASSERT(visData[0].vars[4]->name=="Pressure"); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + + fillData.copy(SignDist,SignData); + fillData.copy(Velocity_x,VelxData); + fillData.copy(Velocity_y,VelyData); + fillData.copy(Velocity_z,VelzData); + fillData.copy(Pressure,PressureData); + + IO::writeData( timestep, visData, Dm->Comm ); + +} + +void ScaLBL_GreyscaleModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + + //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); + +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); +// +// ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); +// FILE *PFILE; +// sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); +// PFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PFILE); +// fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); + FILE *POROS_FILE; + sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); + POROS_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,POROS_FILE); + fclose(POROS_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); + FILE *PERM_FILE; + sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); + PERM_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PERM_FILE); + fclose(PERM_FILE); +} diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h new file mode 100644 index 00000000..3e883b16 --- /dev/null +++ b/models/GreyscaleSCModel.h @@ -0,0 +1,91 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "common/MPI.h" +#include "common/Database.h" +#include "common/ScaLBL.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_GreyscaleModel{ +public: + ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + void VelocityField(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + int CollisionType; + double tau; + double tau_eff; + double Den;//constant density + double tolerance; + double Fx,Fy,Fz,flux; + double din,dout; + double dp;//solid particle diameter, unit in voxel + double GreyPorosity; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr greyscale_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + signed char *id; + int *NeighborList; + double *fq; + double *Permeability;//grey voxel permeability + double *Porosity; + double *Velocity; + double *Pressure_dvc; + IntArray Map; + DoubleArray SignDist; + DoubleArray Velocity_x; + DoubleArray Velocity_y; + DoubleArray Velocity_z; + DoubleArray PorosityMap; + DoubleArray Pressure; + +private: + MPI_Comm comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + void AssignComponentLabels(double *Porosity, double *Permeablity); + +}; + From 8e2efa8f0595fe4ebb27a8baa101943f839db813 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 30 Apr 2020 23:42:17 -0400 Subject: [PATCH 129/270] GreyscaleSC model;GPU only;save the work;model does not function correctly --- common/ScaLBL.cpp | 434 ++- common/ScaLBL.h | 25 +- gpu/GreyscaleSC.cu | 3851 ++++++++++++++++---------- models/GreyscaleSCModel.cpp | 1034 ++++--- models/GreyscaleSCModel.h | 26 +- tests/CMakeLists.txt | 1 + tests/lbpm_greyscaleSC_simulator.cpp | 59 + 7 files changed, 3568 insertions(+), 1862 deletions(-) create mode 100644 tests/lbpm_greyscaleSC_simulator.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 0feb5558..8d868218 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -78,43 +78,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ BoundaryCondition = Dm->BoundaryCondition; //...................................................................................... - ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 5*sendCount_X*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 5*sendCount_y*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 5*sendCount_z*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 2*5*sendCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 2*5*sendCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 2*5*sendCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 2*5*sendCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 2*5*sendCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 2*5*sendCount_Z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, 2*sendCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, 2*sendCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, 2*sendCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, 2*sendCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, 2*sendCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, 2*sendCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, 2*sendCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, 2*sendCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, 2*sendCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, 2*sendCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, 2*sendCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, 2*sendCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... - ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 5*recvCount_x*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 5*recvCount_X*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 5*recvCount_y*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 5*recvCount_z*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory - ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 2*5*recvCount_x*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 2*5*recvCount_X*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 2*5*recvCount_y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 2*5*recvCount_Y*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 2*5*recvCount_z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 2*5*recvCount_Z*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, 2*recvCount_xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, 2*recvCount_xY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, 2*recvCount_Xy*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, 2*recvCount_XY*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, 2*recvCount_xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, 2*recvCount_xZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, 2*recvCount_Xz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, 2*recvCount_XZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, 2*recvCount_yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, 2*recvCount_yZ*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, 2*recvCount_Yz*sizeof(double)); // Allocate device memory + ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, 2*recvCount_YZ*sizeof(double)); // Allocate device memory //...................................................................................... ScaLBL_AllocateZeroCopy((void **) &dvcSendList_x, sendCount_x*sizeof(int)); // Allocate device memory ScaLBL_AllocateZeroCopy((void **) &dvcSendList_X, sendCount_X*sizeof(int)); // Allocate device memory @@ -1052,6 +1052,368 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ } +void ScaLBL_Communicator::BiSendD3Q19AA(double *Aq, double *Bq){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + if (Lock==true){ + ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + } + else{ + Lock=true; + } + // assign tag of 19 to D3Q19 communication + sendtag = recvtag = 38; + ScaLBL_DeviceBarrier(); + // Pack the distributions + //...Packing for x face(2,8,10,12,14)................................ + ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,0*sendCount_x,sendCount_x,sendbuf_x,Aq,N); + ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,1*sendCount_x,sendCount_x,sendbuf_x,Aq,N); + ScaLBL_D3Q19_Pack(10,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,Aq,N); + ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,Aq,N); + ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,Aq,N); + ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,5*sendCount_x,sendCount_x,sendbuf_x,Bq,N); + ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,6*sendCount_x,sendCount_x,sendbuf_x,Bq,N); + ScaLBL_D3Q19_Pack(10,dvcSendList_x,7*sendCount_x,sendCount_x,sendbuf_x,Bq,N); + ScaLBL_D3Q19_Pack(12,dvcSendList_x,8*sendCount_x,sendCount_x,sendbuf_x,Bq,N); + ScaLBL_D3Q19_Pack(14,dvcSendList_x,9*sendCount_x,sendCount_x,sendbuf_x,Bq,N); + + req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 10*sendCount_x,rank_x,sendtag); + req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 10*recvCount_X,rank_X,recvtag); + + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,0*sendCount_X,sendCount_X,sendbuf_X,Aq,N); + ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,1*sendCount_X,sendCount_X,sendbuf_X,Aq,N); + ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,Aq,N); + ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,Aq,N); + ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,Aq,N); + ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,5*sendCount_X,sendCount_X,sendbuf_X,Bq,N); + ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,6*sendCount_X,sendCount_X,sendbuf_X,Bq,N); + ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,7*sendCount_X,sendCount_X,sendbuf_X,Bq,N); + ScaLBL_D3Q19_Pack(11,dvcSendList_X,8*sendCount_X,sendCount_X,sendbuf_X,Bq,N); + ScaLBL_D3Q19_Pack(13,dvcSendList_X,9*sendCount_X,sendCount_X,sendbuf_X,Bq,N); + + req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 10*sendCount_X,rank_X,sendtag); + req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 10*recvCount_x,rank_x,recvtag); + + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,0*sendCount_y,sendCount_y,sendbuf_y,Aq,N); + ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,1*sendCount_y,sendCount_y,sendbuf_y,Aq,N); + ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,Aq,N); + ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,Aq,N); + ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,Aq,N); + ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,5*sendCount_y,sendCount_y,sendbuf_y,Bq,N); + ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,6*sendCount_y,sendCount_y,sendbuf_y,Bq,N); + ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,7*sendCount_y,sendCount_y,sendbuf_y,Bq,N); + ScaLBL_D3Q19_Pack(16,dvcSendList_y,8*sendCount_y,sendCount_y,sendbuf_y,Bq,N); + ScaLBL_D3Q19_Pack(18,dvcSendList_y,9*sendCount_y,sendCount_y,sendbuf_y,Bq,N); + + req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 10*sendCount_y,rank_y,sendtag); + req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 10*recvCount_Y,rank_Y,recvtag); + + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,0*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); + ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,1*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); + ScaLBL_D3Q19_Pack(10,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); + ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); + ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); + ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,5*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,6*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + ScaLBL_D3Q19_Pack(10,dvcSendList_Y,7*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + ScaLBL_D3Q19_Pack(15,dvcSendList_Y,8*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + ScaLBL_D3Q19_Pack(17,dvcSendList_Y,9*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + + req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 10*sendCount_Y,rank_Y,sendtag); + req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 10*recvCount_y,rank_y,recvtag); + + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,0*sendCount_z,sendCount_z,sendbuf_z,Aq,N); + ScaLBL_D3Q19_Pack(12,dvcSendList_z,1*sendCount_z,sendCount_z,sendbuf_z,Aq,N); + ScaLBL_D3Q19_Pack(13,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,Aq,N); + ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,Aq,N); + ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,Aq,N); + ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,5*sendCount_z,sendCount_z,sendbuf_z,Bq,N); + ScaLBL_D3Q19_Pack(12,dvcSendList_z,6*sendCount_z,sendCount_z,sendbuf_z,Bq,N); + ScaLBL_D3Q19_Pack(13,dvcSendList_z,7*sendCount_z,sendCount_z,sendbuf_z,Bq,N); + ScaLBL_D3Q19_Pack(16,dvcSendList_z,8*sendCount_z,sendCount_z,sendbuf_z,Bq,N); + ScaLBL_D3Q19_Pack(17,dvcSendList_z,9*sendCount_z,sendCount_z,sendbuf_z,Bq,N); + + req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 10*sendCount_z,rank_z,sendtag); + req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 10*recvCount_Z,rank_Z,recvtag); + + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,0*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); + ScaLBL_D3Q19_Pack(11,dvcSendList_Z,1*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); + ScaLBL_D3Q19_Pack(14,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); + ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); + ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); + ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,5*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); + ScaLBL_D3Q19_Pack(11,dvcSendList_Z,6*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); + ScaLBL_D3Q19_Pack(14,dvcSendList_Z,7*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); + ScaLBL_D3Q19_Pack(15,dvcSendList_Z,8*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); + ScaLBL_D3Q19_Pack(18,dvcSendList_Z,9*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); + + req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 10*sendCount_Z,rank_Z,sendtag); + req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 10*recvCount_z,rank_z,recvtag); + + //...Pack the xy edge (8)................................ + ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0*sendCount_xy,sendCount_xy,sendbuf_xy,Aq,N); + ScaLBL_D3Q19_Pack(8,dvcSendList_xy,1*sendCount_xy,sendCount_xy,sendbuf_xy,Bq,N); + req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, 2*sendCount_xy,rank_xy,sendtag); + req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, 2*recvCount_XY,rank_XY,recvtag); + + //...Pack the Xy edge (9)................................ + ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Aq,N); + ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,1*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Bq,N); + req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, 2*sendCount_Xy,rank_Xy,sendtag); + req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, 2*recvCount_xY,rank_xY,recvtag); + + //...Pack the xY edge (10)................................ + ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0*sendCount_xY,sendCount_xY,sendbuf_xY,Aq,N); + ScaLBL_D3Q19_Pack(10,dvcSendList_xY,1*sendCount_xY,sendCount_xY,sendbuf_xY,Bq,N); + req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, 2*sendCount_xY,rank_xY,sendtag); + req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, 2*recvCount_Xy,rank_Xy,recvtag); + + //...Pack the XY edge (7)................................ + ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0*sendCount_XY,sendCount_XY,sendbuf_XY,Aq,N); + ScaLBL_D3Q19_Pack(7,dvcSendList_XY,1*sendCount_XY,sendCount_XY,sendbuf_XY,Bq,N); + req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, 2*sendCount_XY,rank_XY,sendtag); + req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, 2*recvCount_xy,rank_xy,recvtag); + + //...Pack the xz edge (12)................................ + ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0*sendCount_xz,sendCount_xz,sendbuf_xz,Aq,N); + ScaLBL_D3Q19_Pack(12,dvcSendList_xz,1*sendCount_xz,sendCount_xz,sendbuf_xz,Bq,N); + req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, 2*sendCount_xz,rank_xz,sendtag); + req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, 2*recvCount_XZ,rank_XZ,recvtag); + + //...Pack the xZ edge (14)................................ + ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Aq,N); + ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,1*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Bq,N); + req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, 2*sendCount_xZ,rank_xZ,sendtag); + req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, 2*recvCount_Xz,rank_Xz,recvtag); + + //...Pack the Xz edge (13)................................ + ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Aq,N); + ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,1*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Bq,N); + req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, 2*sendCount_Xz,rank_Xz,sendtag); + req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, 2*recvCount_xZ,rank_xZ,recvtag); + + //...Pack the XZ edge (11)................................ + ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Aq,N); + ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,1*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Bq,N); + req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, 2*sendCount_XZ,rank_XZ,sendtag); + req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, 2*recvCount_xz,rank_xz,recvtag); + + //...Pack the yz edge (16)................................ + ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0*sendCount_yz,sendCount_yz,sendbuf_yz,Aq,N); + ScaLBL_D3Q19_Pack(16,dvcSendList_yz,1*sendCount_yz,sendCount_yz,sendbuf_yz,Bq,N); + req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, 2*sendCount_yz,rank_yz,sendtag); + req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, 2*recvCount_YZ,rank_YZ,recvtag); + + //...Pack the yZ edge (18)................................ + ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Aq,N); + ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,1*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Bq,N); + req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, 2*sendCount_yZ,rank_yZ,sendtag); + req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, 2*recvCount_Yz,rank_Yz,recvtag); + + //...Pack the Yz edge (17)................................ + ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Aq,N); + ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,1*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Bq,N); + req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, 2*sendCount_Yz,rank_Yz,sendtag); + req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, 2*recvCount_yZ,rank_yZ,recvtag); + + //...Pack the YZ edge (15)................................ + ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Aq,N); + ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,1*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Bq,N); + req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, 2*sendCount_YZ,rank_YZ,sendtag); + req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, 2*recvCount_yz,rank_yz,recvtag); + //................................................................................... +} + +void ScaLBL_Communicator::BiRecvD3Q19AA(double *Aq, double *Bq){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + //................................................................................... + // Wait for completion of D3Q19 communication + MPI_COMM_SCALBL.waitAll(18,req1); + MPI_COMM_SCALBL.waitAll(18,req2); + ScaLBL_DeviceBarrier(); + + //................................................................................... + // NOTE: AA Routine writes to opposite + // Unpack the distributions on the device + //................................................................................... + //...Unpacking for x face(2,8,10,12,14)................................ + ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,recvbuf_x,Aq,N); + ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,recvbuf_x,Aq,N); + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,Aq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,Aq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,Aq,N); + ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); + ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); + //................................................................................... + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,recvbuf_X,Aq,N); + ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,recvbuf_X,Aq,N); + ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,recvbuf_X,Aq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,recvbuf_X,Aq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,recvbuf_X,Aq,N); + ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); + ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); + ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); + //................................................................................... + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,recvbuf_y,Aq,N); + ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,recvbuf_y,Aq,N); + ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,recvbuf_y,Aq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,recvbuf_y,Aq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,recvbuf_y,Aq,N); + ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); + ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); + ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); + //................................................................................... + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); + ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); + ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); + ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); + //................................................................................... + + //...Pack the xy edge (8)................................ + ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,Aq,N); + ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,&recvbuf_xy[recvCount_xy],Bq,N); + + //...Pack the Xy edge (9)................................ + ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,recvbuf_Xy,Aq,N); + ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,&recvbuf_Xy[recvCount_Xy],Bq,N); + + //...Pack the xY edge (10)................................ + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,Aq,N); + ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,&recvbuf_xY[recvCount_xY],Bq,N); + + //...Pack the XY edge (7)................................ + ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,Aq,N); + ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,&recvbuf_XY[recvCount_XY],Bq,N); + + if (BoundaryCondition > 0 && kproc == 0){ + // don't unpack little z + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + //...Pack the xZ edge (14)................................ + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N); + //...Pack the XZ edge (11)................................ + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N); + //...Pack the yZ edge (18)................................ + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N); + //...Pack the YZ edge (15)................................ + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N); + } + else if (BoundaryCondition > 0 && kproc == nprocz-1){ + // don't unpack big Z + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + //...Pack the xz edge (12)................................ + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N); + //...Pack the Xz edge (13)................................ + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N); + //...Pack the yz edge (16)................................ + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N); + //...Pack the Yz edge (17)................................ + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N); + } + else { + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N); + ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); + ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); + //...Pack the xZ edge (14)................................ + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N); + ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N); + //...Pack the XZ edge (11)................................ + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N); + ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N); + //...Pack the yZ edge (18)................................ + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N); + ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N); + //...Pack the YZ edge (15)................................ + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N); + ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N); + //...Pack the xz edge (12)................................ + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N); + ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N); + //...Pack the Xz edge (13)................................ + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N); + ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N); + //...Pack the yz edge (16)................................ + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N); + ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N); + //...Pack the Yz edge (17)................................ + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N); + ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N); + } + + //................................................................................... + Lock=false; // unlock the communicator after communications complete + //................................................................................... + +} + void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ // Recieves halo and incorporates into D3Q19 based stencil gradient computation diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 7ebeffa5..5220bed2 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -56,7 +56,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); -// GREYSCALE MODEL +// GREYSCALE MODEL (Single-component) extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den); @@ -72,7 +72,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); -// GREYSCALE COLOR MODEL +// GREYSCALE FREE-ENERGY MODEL (Two-component) extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, @@ -116,6 +116,25 @@ extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, dou extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); +// GREYSCALE SHAN-CHEN MODEL (Two-component) + +extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(double *distA, double *distB, double *Den, int Np); + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double *distA, double *distB, double *Den, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *distB, double *Den, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np); + + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); @@ -225,6 +244,8 @@ public: // void RecvD3Q19AA(double *f_even, double *f_odd); void SendD3Q19AA(double *dist); void RecvD3Q19AA(double *dist); + void BiSendD3Q19AA(double *Aq, double *Bq); + void BiRecvD3Q19AA(double *Aq, double *Bq); // void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); // void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); void SendD3Q7AA(double *Aq); diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index a3fba989..6219d42f 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -3,1083 +3,41 @@ #define NBLOCKS 1024 #define NTHREADS 256 -__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity, double *Pressure){ - int n; - // conserved momemnts - double rho,vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure; - //double uu; - // non-conserved moments - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; - double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s 10Np => odd part of dist) - f1 = dist[nr1]; // reading the f1 data into register fq - - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - f2 = dist[nr2]; // reading the f2 data into register fq - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - f3 = dist[nr3]; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - f4 = dist[nr4]; - - // q=5 - nr5 = neighborList[n+4*Np]; - f5 = dist[nr5]; - - // q = 6 - nr6 = neighborList[n+5*Np]; - f6 = dist[nr6]; - - // q=7 - nr7 = neighborList[n+6*Np]; - f7 = dist[nr7]; - - // q = 8 - nr8 = neighborList[n+7*Np]; - f8 = dist[nr8]; - - // q=9 - nr9 = neighborList[n+8*Np]; - f9 = dist[nr9]; - - // q = 10 - nr10 = neighborList[n+9*Np]; - f10 = dist[nr10]; - - // q=11 - nr11 = neighborList[n+10*Np]; - f11 = dist[nr11]; - - // q=12 - nr12 = neighborList[n+11*Np]; - f12 = dist[nr12]; - - // q=13 - nr13 = neighborList[n+12*Np]; - f13 = dist[nr13]; - - // q=14 - nr14 = neighborList[n+13*Np]; - f14 = dist[nr14]; - - // q=15 - nr15 = neighborList[n+14*Np]; - f15 = dist[nr15]; - - // q=16 - nr16 = neighborList[n+15*Np]; - f16 = dist[nr16]; - - // q=17 - //fq = dist[18*Np+n]; - nr17 = neighborList[n+16*Np]; - f17 = dist[nr17]; - - // q=18 - nr18 = neighborList[n+17*Np]; - f18 = dist[nr18]; - - porosity = Poros[n]; - perm = Perm[n]; - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - pressure = rho/porosity/3.0; - vx = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14)/rho+0.5*porosity*Gx; - vy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18)/rho+0.5*porosity*Gy; - vz = (f5-f6+f11-f12-f13+f14+f15-f16-f17+f18)/rho+0.5*porosity*Gz; - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the body force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = -porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx; - Fy = -porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy; - Fz = -porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz; - if (porosity==1.0){ - Fx=Gx; - Fy=Gy; - Fz=Gz; - } - - //------------------------ BGK collison where body force has higher-order terms ----------------------------------------------------------// -// // q=0 -// dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// + 0.3333333333333333*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); -// -// // q = 1 -// dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); -// -// // q=2 -// dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3. + (6.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); -// -// // q = 3 -// dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); -// -// // q = 4 -// dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. + (6.*uy)/porosity) + Fz*(0. - (3.*uz)/porosity)); -// -// // q = 5 -// dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(3. + (6.*uz)/porosity)); -// -// // q = 6 -// dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) -// +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(0. - (3.*uy)/porosity) + Fz*(-3. + (6.*uz)/porosity)); -// -// // q = 7 -// dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + -// Fz*(0. - (3.*uz)/porosity)); -// -// // q = 8 -// dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + Fy*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + -// Fz*(0. - (3.*uz)/porosity)); -// -// // q = 9 -// dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + Fy*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + -// Fz*(0. - (3.*uz)/porosity)); -// -// // q = 10 -// dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + -// Fz*(0. - (3.*uz)/porosity)); -// -// // q = 11 -// dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + -// Fz*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); -// -// // q = 12 -// dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + -// Fz*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); -// -// // q = 13 -// dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + -// Fz*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); -// -// // q= 14 -// dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(0. - (3.*uy)/porosity) + Fx*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + -// Fz*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); -// -// // q = 15 -// dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + -// Fz*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); -// -// // q = 16 -// dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + -// Fz*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); -// -// // q = 17 -// dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + -// Fz*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); -// -// // q = 18 -// dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) -// +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(0. - (3.*ux)/porosity) + Fy*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + -// Fz*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - //----------------------------------------------------------------------------------------------------------------------------------------// - - - //------------------------ BGK collison where body force has NO higher-order terms ----------------------------------------------------------// - // q=0 - dist[n] = f0*(1.0-rlx) + rlx*0.3333333333333333*rho*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity); - - // q = 1 - dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(3.)); - - // q=2 - dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fx*(-3.)); - - // q = 3 - dist[nr4] = f3*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(3.)); - - // q = 4 - dist[nr3] = f4*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fy*(-3.)); - - // q = 5 - dist[nr6] = f5*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(3.)); - - // q = 6 - dist[nr5] = f6*(1.0-rlx) + rlx*0.05555555555555555*rho*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*rho*(1. - 0.5*rlx)*(Fz*(-3.)); - - // q = 7 - dist[nr8] = f7*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(3.)); - - // q = 8 - dist[nr7] = f8*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(-3.)); - - // q = 9 - dist[nr10] = f9*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fy*(-3.)); - - // q = 10 - dist[nr9] = f10*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fy*(3.)); - - // q = 11 - dist[nr12] = f11*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(3.)); - - // q = 12 - dist[nr11] = f12*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(-3.)); - - // q = 13 - dist[nr14] = f13*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(3.) + Fz*(-3.)); - - // q= 14 - dist[nr13] = f14*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fx*(-3.) + Fz*(3.)); - - // q = 15 - dist[nr16] = f15*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(3.)); - - // q = 16 - dist[nr15] = f16*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(-3.)); - - // q = 17 - dist[nr18] = f17*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(3.) + Fz*(-3.)); - - // q = 18 - dist[nr17] = f18*(1.0-rlx) + rlx*0.027777777777777776*rho*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*rho*(1. - 0.5*rlx)*(Fy*(-3.) + Fz*(3.)); - //-------------------------------------------------------------------------------------------------------------------------------------------// - - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity, double Den, double *Pressure){ - - int n; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; - double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double rlx_setA = rlx; - double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s 10Np => odd part of dist) - fq = dist[nread]; // reading the f1 data into register fq - pressure = fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; + // q=1 + nread = neighborList[n]; // neighbor 2 + fq = distA[nread]; // reading the f1 data into register fq + rhoA_next += fq; + m1A -= 11.0*fq; + m2A -= 4.0*fq; + jxA = fq; + m4A = -4.0*fq; + m9A = 2.0*fq; + m10A = -4.0*fq; - // q=2 - nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = dist[nread]; // reading the f2 data into register fq - pressure += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); + // q=2 + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = distA[nread]; // reading the f2 data into register fq + rhoA_next += fq; + m1A -= 11.0*(fq); + m2A -= 4.0*(fq); + jxA -= fq; + m4A += 4.0*(fq); + m9A += 2.0*(fq); + m10A -= 4.0*(fq); - // q=3 - nread = neighborList[n+2*Np]; // neighbor 4 - fq = dist[nread]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = distA[nread]; + rhoA_next += fq; + m1A -= 11.0*fq; + m2A -= 4.0*fq; + jyA = fq; + m6A = -4.0*fq; + m9A -= fq; + m10A += 2.0*fq; + m11A = fq; + m12A = -2.0*fq; - // q = 4 - nread = neighborList[n+3*Np]; // neighbor 3 - fq = dist[nread]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = distA[nread]; + rhoA_next += fq; + m1A -= 11.0*fq; + m2A -= 4.0*fq; + jyA -= fq; + m6A += 4.0*fq; + m9A -= fq; + m10A += 2.0*fq; + m11A += fq; + m12A -= 2.0*fq; - // q=5 - nread = neighborList[n+4*Np]; - fq = dist[nread]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; + // q=5 + nread = neighborList[n+4*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A -= 11.0*fq; + m2A -= 4.0*fq; + jzA = fq; + m8A = -4.0*fq; + m9A -= fq; + m10A += 2.0*fq; + m11A -= fq; + m12A += 2.0*fq; - // q = 6 - nread = neighborList[n+5*Np]; - fq = dist[nread]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - // q=7 - nread = neighborList[n+6*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; + // q = 6 + nread = neighborList[n+5*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A -= 11.0*fq; + m2A -= 4.0*fq; + jzA -= fq; + m8A += 4.0*fq; + m9A -= fq; + m10A += 2.0*fq; + m11A -= fq; + m12A += 2.0*fq; - // q = 8 - nread = neighborList[n+7*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; + // q=7 + nread = neighborList[n+6*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA += fq; + m4A += fq; + jyA += fq; + m6A += fq; + m9A += fq; + m10A += fq; + m11A += fq; + m12A += fq; + m13A = fq; + m16A = fq; + m17A = -fq; - // q=9 - nread = neighborList[n+8*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; + // q = 8 + nread = neighborList[n+7*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA -= fq; + m4A -= fq; + jyA -= fq; + m6A -= fq; + m9A += fq; + m10A += fq; + m11A += fq; + m12A += fq; + m13A += fq; + m16A -= fq; + m17A += fq; - // q = 10 - nread = neighborList[n+9*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; + // q=9 + nread = neighborList[n+8*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA += fq; + m4A += fq; + jyA -= fq; + m6A -= fq; + m9A += fq; + m10A += fq; + m11A += fq; + m12A += fq; + m13A -= fq; + m16A += fq; + m17A += fq; - // q=11 - nread = neighborList[n+10*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; + // q = 10 + nread = neighborList[n+9*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA -= fq; + m4A -= fq; + jyA += fq; + m6A += fq; + m9A += fq; + m10A += fq; + m11A += fq; + m12A += fq; + m13A -= fq; + m16A -= fq; + m17A -= fq; - // q=12 - nread = neighborList[n+11*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; + // q=11 + nread = neighborList[n+10*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA += fq; + m4A += fq; + jzA += fq; + m8A += fq; + m9A += fq; + m10A += fq; + m11A -= fq; + m12A -= fq; + m15A = fq; + m16A -= fq; + m18A = fq; - // q=13 - nread = neighborList[n+12*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; + // q=12 + nread = neighborList[n+11*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA -= fq; + m4A -= fq; + jzA -= fq; + m8A -= fq; + m9A += fq; + m10A += fq; + m11A -= fq; + m12A -= fq; + m15A += fq; + m16A += fq; + m18A -= fq; - // q=14 - nread = neighborList[n+13*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; + // q=13 + nread = neighborList[n+12*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA += fq; + m4A += fq; + jzA -= fq; + m8A -= fq; + m9A += fq; + m10A += fq; + m11A -= fq; + m12A -= fq; + m15A -= fq; + m16A -= fq; + m18A -= fq; - // q=15 - nread = neighborList[n+14*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; + // q=14 + nread = neighborList[n+13*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jxA -= fq; + m4A -= fq; + jzA += fq; + m8A += fq; + m9A += fq; + m10A += fq; + m11A -= fq; + m12A -= fq; + m15A -= fq; + m16A += fq; + m18A += fq; - // q=16 - nread = neighborList[n+15*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; + // q=15 + nread = neighborList[n+14*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jyA += fq; + m6A += fq; + jzA += fq; + m8A += fq; + m9A -= 2.0*fq; + m10A -= 2.0*fq; + m14A = fq; + m17A += fq; + m18A -= fq; - // q=17 - nread = neighborList[n+16*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; + // q=16 + nread = neighborList[n+15*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jyA -= fq; + m6A -= fq; + jzA -= fq; + m8A -= fq; + m9A -= 2.0*fq; + m10A -= 2.0*fq; + m14A += fq; + m17A -= fq; + m18A += fq; - // q=18 - nread = neighborList[n+17*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; + // q=17 + nread = neighborList[n+16*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jyA += fq; + m6A += fq; + jzA -= fq; + m8A -= fq; + m9A -= 2.0*fq; + m10A -= 2.0*fq; + m14A -= fq; + m17A += fq; + m18A += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = distA[nread]; + rhoA_next += fq; + m1A += 8.0*fq; + m2A += fq; + jyA -= fq; + m6A -= fq; + jzA += fq; + m8A += fq; + m9A -= 2.0*fq; + m10A -= 2.0*fq; + m14A -= fq; + m17A -= fq; + m18A -= fq; //---------------------------------------------------------------------// - porosity = Poros[n]; - perm = Perm[n]; + // ------------------- Fluid component B ---------------------------------// + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + // q=0 + fq = distB[n]; + rhoB_next = fq; + m1B = -30.0*fq; + m2B = 12.0*fq; - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + // q=1 + nread = neighborList[n]; // neighbor 2 + fq = distB[nread]; // reading the f1 data into register fq + rhoB_next += fq; + m1B -= 11.0*fq; + m2B -= 4.0*fq; + jxB = fq; + m4B = -4.0*fq; + m9B = 2.0*fq; + m10B = -4.0*fq; + + // q=2 + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = distB[nread]; // reading the f2 data into register fq + rhoB_next += fq; + m1B -= 11.0*(fq); + m2B -= 4.0*(fq); + jxB -= fq; + m4B += 4.0*(fq); + m9B += 2.0*(fq); + m10B -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = distB[nread]; + rhoB_next += fq; + m1B -= 11.0*fq; + m2B -= 4.0*fq; + jyB = fq; + m6B = -4.0*fq; + m9B -= fq; + m10B += 2.0*fq; + m11B = fq; + m12B = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = distB[nread]; + rhoB_next += fq; + m1B -= 11.0*fq; + m2B -= 4.0*fq; + jyB -= fq; + m6B += 4.0*fq; + m9B -= fq; + m10B += 2.0*fq; + m11B += fq; + m12B -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B -= 11.0*fq; + m2B -= 4.0*fq; + jzB = fq; + m8B = -4.0*fq; + m9B -= fq; + m10B += 2.0*fq; + m11B -= fq; + m12B += 2.0*fq; + + + // q = 6 + nread = neighborList[n+5*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B -= 11.0*fq; + m2B -= 4.0*fq; + jzB -= fq; + m8B += 4.0*fq; + m9B -= fq; + m10B += 2.0*fq; + m11B -= fq; + m12B += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB += fq; + m4B += fq; + jyB += fq; + m6B += fq; + m9B += fq; + m10B += fq; + m11B += fq; + m12B += fq; + m13B = fq; + m16B = fq; + m17B = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB -= fq; + m4B -= fq; + jyB -= fq; + m6B -= fq; + m9B += fq; + m10B += fq; + m11B += fq; + m12B += fq; + m13B += fq; + m16B -= fq; + m17B += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB += fq; + m4B += fq; + jyB -= fq; + m6B -= fq; + m9B += fq; + m10B += fq; + m11B += fq; + m12B += fq; + m13B -= fq; + m16B += fq; + m17B += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB -= fq; + m4B -= fq; + jyB += fq; + m6B += fq; + m9B += fq; + m10B += fq; + m11B += fq; + m12B += fq; + m13B -= fq; + m16B -= fq; + m17B -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB += fq; + m4B += fq; + jzB += fq; + m8B += fq; + m9B += fq; + m10B += fq; + m11B -= fq; + m12B -= fq; + m15B = fq; + m16B -= fq; + m18B = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB -= fq; + m4B -= fq; + jzB -= fq; + m8B -= fq; + m9B += fq; + m10B += fq; + m11B -= fq; + m12B -= fq; + m15B += fq; + m16B += fq; + m18B -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB += fq; + m4B += fq; + jzB -= fq; + m8B -= fq; + m9B += fq; + m10B += fq; + m11B -= fq; + m12B -= fq; + m15B -= fq; + m16B -= fq; + m18B -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jxB -= fq; + m4B -= fq; + jzB += fq; + m8B += fq; + m9B += fq; + m10B += fq; + m11B -= fq; + m12B -= fq; + m15B -= fq; + m16B += fq; + m18B += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jyB += fq; + m6B += fq; + jzB += fq; + m8B += fq; + m9B -= 2.0*fq; + m10B -= 2.0*fq; + m14B = fq; + m17B += fq; + m18B -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jyB -= fq; + m6B -= fq; + jzB -= fq; + m8B -= fq; + m9B -= 2.0*fq; + m10B -= 2.0*fq; + m14B += fq; + m17B -= fq; + m18B += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jyB += fq; + m6B += fq; + jzB -= fq; + m8B -= fq; + m9B -= 2.0*fq; + m10B -= 2.0*fq; + m14B -= fq; + m17B += fq; + m18B += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = distB[nread]; + rhoB_next += fq; + m1B += 8.0*fq; + m2B += fq; + jyB -= fq; + m6B -= fq; + jzB += fq; + m8B += fq; + m9B -= 2.0*fq; + m10B -= 2.0*fq; + m14B -= fq; + m17B -= fq; + m18B -= fq; + //---------------------------------------------------------------------// + + + // Compute SC fluid-fluid interaction force + GffA_x = -Gsc*rhoB_gradx; + GffA_y = -Gsc*rhoB_grady; + GffA_z = -Gsc*rhoB_gradz; + GffB_x = -Gsc*rhoA_gradx; + GffB_y = -Gsc*rhoA_grady; + GffB_z = -Gsc*rhoA_gradz; + // Compute SC fluid-solid force + GfsA_x = SolidForceA[n+0*Np]; + GfsA_y = SolidForceA[n+1*Np]; + GfsA_z = SolidForceA[n+2*Np]; + GfsB_x = SolidForceB[n+0*Np]; + GfsB_y = SolidForceB[n+1*Np]; + GfsB_z = SolidForceB[n+2*Np]; + + // Compute greyscale related parameters + // ------------------- Fluid Component A -----------------------// + c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permA); if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - vx = jx/Den+0.5*porosity*Gx; - vy = jy/Den+0.5*porosity*Gy; - vz = jz/Den+0.5*porosity*Gz; + vx = jxA/rhoA_next+0.5*(porosity*Gx+GffA_x+GfsA_x); + vy = jyA/rhoA_next+0.5*(porosity*Gy+GffA_y+GfsA_y); + vz = jzA/rhoA_next+0.5*(porosity*Gz+GffA_z+GfsA_z); v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); + ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = Den*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); - Fy = Den*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); - Fz = Den*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); + FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); + FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); if (porosity==1.0){ - Fx=Den*Gx; - Fy=Den*Gy; - Fz=Den*Gz; + FxA=rhoA*(Gx + GffA_x + GfsA_x); + FyA=rhoA*(Gy + GffA_y + GfsA_y); + FzA=rhoA*(Gz + GffA_z + GfsA_z); + } + // ------------------- Fluid Component B -----------------------// + // Compute greyscale related parameters + c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permB); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jxB/rhoB_next+0.5*(porosity*Gx+GffB_x+GfsB_x); + vy = jyB/rhoB_next+0.5*(porosity*Gy+GffB_y+GfsB_y); + vz = jzB/rhoB_next+0.5*(porosity*Gz+GffB_z+GfsB_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); + FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); + FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); + if (porosity==1.0){ + FxB=rhoB*(Gx + GffB_x + GfsB_x); + FyB=rhoB*(Gy + GffB_y + GfsB_y); + FzB=rhoB*(Gz + GffB_z + GfsB_z); } - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); + // Calculate barycentric velocity of the fluid mixture + ux = (rhoA_next*ux_A+rhoB_next*ux_B)/(rhoA_next+rhoB_next); + uy = (rhoA_next*uy_A+rhoB_next*uy_B)/(rhoA_next+rhoB_next); + uz = (rhoA_next*uz_A+rhoB_next*uz_B)/(rhoA_next+rhoB_next); // //..............carry out relaxation process............................................... // m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) @@ -1457,140 +770,1428 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double // m18 = m18 + rlx_setB*( - m18); // //....................................................................................................... - //-------------------- IMRT collison where body force has NO higher-order terms -------------// + // ------------------- Fluid Component A -----------------------// + rlx_setA = 1.0/tauA; + rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); + //TODO need to incoporate porosity + m1A = m1A + rlx_setA*((19*rhoA_next*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); + m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA_next*(ux*ux+uy*uy+uz*uz))- m2A) + + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); + jxA = jxA + FxA; + m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA_next)- m4A) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); + jyA = jyA + FyA; + m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA_next)- m6A) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); + jzA = jzA + FzA; + m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA_next)- m8A) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); + m9A = m9A + rlx_setA*((rhoA_next*(2*ux*ux-uy*uy-uz*uz)) - m9A) + + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); + m10A = m10A + rlx_setA*( - m10A) + + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); + m11A = m11A + rlx_setA*((rhoA_next*(uy*uy-uz*uz)) - m11A) + + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); + m12A = m12A + rlx_setA*( - m12A); + + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); + m13A = m13A + rlx_setA*( rhoA_next*(ux*uy) - m13A) + + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); + m14A = m14A + rlx_setA*( rhoA_next*(uy*uz) - m14A); + + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); + m15A = m15A + rlx_setA*( rhoA_next*(ux*uz) - m15A) + + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); + m16A = m16A + rlx_setB*( - m16A); + m17A = m17A + rlx_setB*( - m17A); + m18A = m18A + rlx_setB*( - m18A); //....................................................................................................... + // ------------------- Fluid Component A -----------------------// //.................inverse transformation...................................................... // q=0 - fq = mrt_V1*Den-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; + //fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + distA[n] = f0; // q = 1 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); nread = neighborList[n+Np]; - dist[nread] = fq; + distA[nread] = f1; // q=2 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); nread = neighborList[n]; - dist[nread] = fq; + distA[nread] = f2; // q = 3 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+3*Np]; - dist[nread] = fq; + distA[nread] = f3; // q = 4 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+2*Np]; - dist[nread] = fq; + distA[nread] = f4; // q = 5 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+5*Np]; - dist[nread] = fq; + distA[nread] = f5; // q = 6 - fq = mrt_V1*Den-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+4*Np]; - dist[nread] = fq; + distA[nread] = f6; // q = 7 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); nread = neighborList[n+7*Np]; - dist[nread] = fq; + distA[nread] = f7; // q = 8 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); nread = neighborList[n+6*Np]; - dist[nread] = fq; + distA[nread] = f8; // q = 9 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); nread = neighborList[n+9*Np]; - dist[nread] = fq; + distA[nread] = f9; // q = 10 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); nread = neighborList[n+8*Np]; - dist[nread] = fq; + distA[nread] = f10; // q = 11 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); nread = neighborList[n+11*Np]; - dist[nread] = fq; + distA[nread] = f11; // q = 12 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); nread = neighborList[n+10*Np]; - dist[nread]= fq; + distA[nread]= f12; // q = 13 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); nread = neighborList[n+13*Np]; - dist[nread] = fq; + distA[nread] = f13; // q= 14 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); nread = neighborList[n+12*Np]; - dist[nread] = fq; + distA[nread] = f14; // q = 15 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); nread = neighborList[n+15*Np]; - dist[nread] = fq; + distA[nread] = f15; // q = 16 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); nread = neighborList[n+14*Np]; - dist[nread] = fq; + distA[nread] = f16; // q = 17 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); nread = neighborList[n+17*Np]; - dist[nread] = fq; + distA[nread] = f17; // q = 18 - fq = mrt_V1*Den+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); nread = neighborList[n+16*Np]; - dist[nread] = fq; + distA[nread] = f18; //........................................................................ + + Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + // ------------------- Fluid Component B -----------------------// + rlx_setA = 1.0/tauB; + rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + //-------------------- MRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + //TODO need to incoporate porosity + m1B = m1B + rlx_setA*((19*rhoB_next*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); + m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB_next*(ux*ux+uy*uy+uz*uz))- m2B) + + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); + jxB = jxB + FxB; + m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB_next)- m4B) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); + jyB = jyB + FyB; + m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB_next)- m6B) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); + jzB = jzB + FzB; + m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB_next)- m8B) + + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); + m9B = m9B + rlx_setA*((rhoB_next*(2*ux*ux-uy*uy-uz*uz)) - m9B) + + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); + m10B = m10B + rlx_setA*( - m10B) + + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); + m11B = m11B + rlx_setA*((rhoB_next*(uy*uy-uz*uz)) - m11B) + + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); + m12B = m12B + rlx_setA*( - m12B) + + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); + m13B = m13B + rlx_setA*( rhoB_next*(ux*uy) - m13B) + + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); + m14B = m14B + rlx_setA*( rhoB_next*(uy*uz) - m14B) + + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); + m15B = m15B + rlx_setA*( rhoB_next*(ux*uz) - m15B) + + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); + m16B = m16B + rlx_setB*( - m16B); + m17B = m17B + rlx_setB*( - m17B); + m18B = m18B + rlx_setB*( - m18B); + //....................................................................................................... + + + // ------------------- Fluid Component B -----------------------// + //.................inverse transformation...................................................... + // q=0 + //fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + distB[n] = f0; + + // q = 1 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + nread = neighborList[n+Np]; + distB[nread] = f1; + + // q=2 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + nread = neighborList[n]; + distB[nread] = f2; + + // q = 3 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + nread = neighborList[n+3*Np]; + distB[nread] = f3; + + // q = 4 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + nread = neighborList[n+2*Np]; + distB[nread] = f4; + + // q = 5 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + nread = neighborList[n+5*Np]; + distB[nread] = f5; + + // q = 6 + //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + nread = neighborList[n+4*Np]; + distB[nread] = f6; + + // q = 7 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + nread = neighborList[n+7*Np]; + distB[nread] = f7; + + // q = 8 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + nread = neighborList[n+6*Np]; + distB[nread] = f8; + + // q = 9 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + nread = neighborList[n+9*Np]; + distB[nread] = f9; + + // q = 10 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + nread = neighborList[n+8*Np]; + distB[nread] = f10; + + // q = 11 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + nread = neighborList[n+11*Np]; + distB[nread] = f11; + + // q = 12 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + nread = neighborList[n+10*Np]; + distB[nread]= f12; + + // q = 13 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + nread = neighborList[n+13*Np]; + distB[nread] = f13; + + // q= 14 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + nread = neighborList[n+12*Np]; + distB[nread] = f14; + + // q = 15 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + nread = neighborList[n+15*Np]; + distB[nread] = f15; + + // q = 16 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + nread = neighborList[n+14*Np]; + distB[nread] = f16; + + // q = 17 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + nread = neighborList[n+17*Np]; + distB[nread] = f17; + + // q = 18 + //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + nread = neighborList[n+16*Np]; + distB[nread] = f18; + //........................................................................ + + Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + //Update velocity on device Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; //Update pressure on device - Pressure[n] = pressure; + Pressure[n] = (rhoA_next+rhoB_next+Gsc*rhoA_next*rhoB_next)/3.0; + //Update density + //Den[n] = rhoA_next; + //Den[n+Np] = rhoB_next; } } } -__global__ void dvc_ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den) +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + int n; + double vx,vy,vz,v_mag; + double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; + double ux,uy,uz; + // conserved momemnts + double jxA,jyA,jzA; + double jxB,jyB,jzB; + double rhoA,rhoB; + double rhoA_next,rhoB_next; + // non-conserved moments + double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; + double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; + double fq; + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double permA,permB;//effective relative perm + double c0, c1; //Guo's model parameters + double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity + double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity + double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double FxB, FyB, FzB; + double rlx_setA,rlx_setB; + double rhoA_gradx,rhoA_grady,rhoA_gradz; + double rhoB_gradx,rhoB_grady,rhoB_gradz; + double GffA_x,GffA_y,GffA_z; + double GfsA_x,GfsA_y,GfsA_z; + double GffB_x,GffB_y,GffB_z; + double GfsB_x,GfsB_y,GfsB_z; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Pressure); + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double Den,double *Pressure){ - - dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT<<>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,Den,Pressure); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale_IMRT: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ - dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); +extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(double *distA,double *distB, double *Den, int Np){ + dvc_ScaLBL_D3Q19_GreyscaleSC_Init<<>>(distA,distB,Den,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double *distA, double *distB, double *Den, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_Density<<>>(NeighborList, distA, distB, Den, start, finish, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_Density: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *distB, double *Den, int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_Density<<>>(distA, distB, Den, start, finish, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_Density: %s \n",cudaGetErrorString(err)); + } +} + + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC<<>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC: %s \n",cudaGetErrorString(err)); } } diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 08cf09b5..da5d08e3 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -1,7 +1,7 @@ /* Greyscale lattice boltzmann model */ -#include "models/GreyscaleModel.h" +#include "models/GreyscaleSCModel.h" #include "analysis/distance.h" #include "analysis/morphology.h" #include @@ -13,76 +13,84 @@ void DeleteArray( const TYPE *p ) delete [] p; } -ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +ScaLBL_GreyscaleSCModel::ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), +rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); SignDist.fill(0); } -ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){ +ScaLBL_GreyscaleSCModel::~ScaLBL_GreyscaleSCModel(){ } -void ScaLBL_GreyscaleModel::ReadParams(string filename){ +void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); - greyscale_db = db->getDatabase( "Greyscale" ); + greyscaleSC_db = db->getDatabase( "GreyscaleSC" ); analysis_db = db->getDatabase( "Analysis" ); vis_db = db->getDatabase( "Visualization" ); // set defaults timestepMax = 100000; - tau = 1.0; - tau_eff = tau; - Den = 1.0;//constant density + tauA = 1.0; + tauB = 1.0; + tauA_eff = tauA; + tauB_eff = tauB; + rhoA = rhoB = 1.0; + rhoA_minor = rhoB_minor = 0.01;//dissolved density + Gsc = 2.0;//SC fluid-fluid interaction coefficient tolerance = 0.01; Fx = Fy = Fz = 0.0; Restart=false; din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel - CollisionType = 1; //1: IMRT; 2: BGK // ---------------------- Greyscale Model parameters -----------------------// - if (greyscale_db->keyExists( "timestepMax" )){ - timestepMax = greyscale_db->getScalar( "timestepMax" ); + if (greyscaleSC_db->keyExists( "timestepMax" )){ + timestepMax = greyscaleSC_db->getScalar( "timestepMax" ); } - if (greyscale_db->keyExists( "tau" )){ - tau = greyscale_db->getScalar( "tau" ); + if (greyscaleSC_db->keyExists( "tauA" )){ + tauA = greyscaleSC_db->getScalar( "tauA" ); } - tau_eff = greyscale_db->getWithDefault( "tau_eff", tau ); - if (greyscale_db->keyExists( "Den" )){ - Den = greyscale_db->getScalar( "Den" ); + if (greyscaleSC_db->keyExists( "tauB" )){ + tauB = greyscaleSC_db->getScalar( "tauB" ); } - if (greyscale_db->keyExists( "dp" )){ - dp = greyscale_db->getScalar( "dp" ); + tauA_eff = greyscaleSC_db->getWithDefault( "tauA_eff", tauA ); + tauB_eff = greyscaleSC_db->getWithDefault( "tauB_eff", tauB ); + rhoA = greyscaleSC_db->getWithDefault( "rhoA", rhoA ); + rhoB = greyscaleSC_db->getWithDefault( "rhoB", rhoB ); + rhoA_minor = greyscaleSC_db->getWithDefault( "rhoA_minor", rhoA_minor ); + rhoB_minor = greyscaleSC_db->getWithDefault( "rhoB_minor", rhoB_minor ); + if (greyscaleSC_db->keyExists( "Gsc" )){ + Gsc = greyscaleSC_db->getScalar( "Gsc" ); } - if (greyscale_db->keyExists( "F" )){ - Fx = greyscale_db->getVector( "F" )[0]; - Fy = greyscale_db->getVector( "F" )[1]; - Fz = greyscale_db->getVector( "F" )[2]; + if (greyscaleSC_db->keyExists( "dp" )){ + dp = greyscaleSC_db->getScalar( "dp" ); } - if (greyscale_db->keyExists( "Restart" )){ - Restart = greyscale_db->getScalar( "Restart" ); + if (greyscaleSC_db->keyExists( "F" )){ + Fx = greyscaleSC_db->getVector( "F" )[0]; + Fy = greyscaleSC_db->getVector( "F" )[1]; + Fz = greyscaleSC_db->getVector( "F" )[2]; } - if (greyscale_db->keyExists( "din" )){ - din = greyscale_db->getScalar( "din" ); + if (greyscaleSC_db->keyExists( "Restart" )){ + Restart = greyscaleSC_db->getScalar( "Restart" ); } - if (greyscale_db->keyExists( "dout" )){ - dout = greyscale_db->getScalar( "dout" ); + if (greyscaleSC_db->keyExists( "din" )){ + din = greyscaleSC_db->getScalar( "din" ); } - if (greyscale_db->keyExists( "flux" )){ - flux = greyscale_db->getScalar( "flux" ); + if (greyscaleSC_db->keyExists( "dout" )){ + dout = greyscaleSC_db->getScalar( "dout" ); } - if (greyscale_db->keyExists( "tolerance" )){ - tolerance = greyscale_db->getScalar( "tolerance" ); + if (greyscaleSC_db->keyExists( "flux" )){ + flux = greyscaleSC_db->getScalar( "flux" ); } - auto collision = greyscale_db->getWithDefault( "collision", "IMRT" ); - if (collision == "BGK"){ - CollisionType=2; + if (greyscaleSC_db->keyExists( "tolerance" )){ + tolerance = greyscaleSC_db->getScalar( "tolerance" ); } // ------------------------------------------------------------------------// @@ -94,7 +102,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ // ------------------------------------------------------------------------// } -void ScaLBL_GreyscaleModel::SetDomain(){ +void ScaLBL_GreyscaleSCModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases // domain parameters @@ -125,7 +133,7 @@ void ScaLBL_GreyscaleModel::SetDomain(){ nprocz = Dm->nprocz(); } -void ScaLBL_GreyscaleModel::ReadInput(){ +void ScaLBL_GreyscaleSCModel::ReadInput(){ sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); @@ -174,44 +182,65 @@ void ScaLBL_GreyscaleModel::ReadInput(){ if (rank == 0) cout << "Domain set." << endl; } -/******************************************************** - * AssignComponentLabels * - ********************************************************/ -void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Permeability) +void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() { + + double *Poros, *Perm; + Poros = new double[Np]; + Perm = new double[Np]; + double *SolidPotentialA_host = new double [Nx*Ny*Nz]; + double *SolidPotentialB_host = new double [Nx*Ny*Nz]; + double *SolidForceA_host = new double[3*Np]; + double *SolidForceB_host = new double[3*Np]; + size_t NLABELS=0; signed char VALUE=0; double POROSITY=0.f; double PERMEABILITY=0.f; + double AFFINITY_A=0.f; + double AFFINITY_B=0.f; - auto LabelList = greyscale_db->getVector( "ComponentLabels" ); - auto PorosityList = greyscale_db->getVector( "PorosityList" ); - auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + auto PorosityList = greyscaleSC_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscaleSC_db->getVector( "PermeabilityList" ); + auto LabelList = greyscaleSC_db->getVector( "ComponentLabels" ); + auto AffinityListA = greyscaleSC_db->getVector( "ComponentAffinityA" ); + auto AffinityListB = greyscaleSC_db->getVector( "ComponentAffinityB" ); + + //1. Requirement for "ComponentLabels": + // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 + // *label = 1 and 2 are reserved for NW and W phase respectively. + //2. Requirement for "ComponentAffinity": + // *should be in the same length as "ComponentLabels" + // *could leave ComponentAffinityA and B=0.0 for label=1 and 2 + //3. Requirement for "PorosityList": + // *for ComponentLables <=0, put porosity value = 0.0; + // *for ComponentLabels >=3, put the corresponding sub-resolution porosity + // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) + //4. Requirement for "PermeabilityList": + // *for ComponentLabels <=2, does not matter, can leave it as 1.0 NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size()){ - ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ + ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList and PermeabilityList must all be the same length! \n"); } double label_count[NLABELS]; double label_count_global[NLABELS]; - // Assign the labels for (int idx=0; idx 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability for (int k=0;k0) && (VALUE == LabelList[idx])){ POROSITY=PorosityList[idx]; label_count[idx] += 1.0; idx = NLABELS; - //Mask->id[n] = 0; // set mask to zero since this is an immobile component } } int idx = Map(i,j,k); @@ -220,16 +249,15 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); } else{ - Porosity[idx] = POROSITY; + Poros[idx] = POROSITY; } } } } } - if (NLABELS != PermeabilityList.size()){ - ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); - } + //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability for (int k=0;k0) && (VALUE == LabelList[idx])){ PERMEABILITY=PermeabilityList[idx]; idx = NLABELS; //Mask->id[n] = 0; // set mask to zero since this is an immobile component @@ -250,13 +278,150 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); } else{ - Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + Perm[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; } } } } } + //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase + for (int k=0;k=3){ + AFFINITY_A=AffinityListA[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 + AFFINITY_B=AffinityListB[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 + } + else{//i.e. label = 1 or 2 + AFFINITY_A=0.0; + AFFINITY_B=0.0; + } + idx = NLABELS; + } + } + //NOTE: node_ID = 1 and 2 are reserved + if ((VALUE == 1)||(VALUE == 2)){ + AFFINITY_A=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists + AFFINITY_B=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists + } + SolidPotentialA_host[n] = AFFINITY_A; + SolidPotentialB_host[n] = AFFINITY_B; + } + } + } + + // Calculate Shan-Chen fluid-solid forces + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f/18.f; + double w_edge = 1.f/36.f; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kid[nn] <= 0)||(Mask->id[nn]>=3)){ + double vec_x = double(ii-1); + double vec_y = double(jj-1); + double vec_z = double(kk-1); + double GWNS_A=SolidPotentialA_host[nn]; + double GWNS_B=SolidPotentialB_host[nn]; + phi_x_A += GWNS_A*weight*vec_x; + phi_y_A += GWNS_A*weight*vec_y; + phi_z_A += GWNS_A*weight*vec_z; + phi_x_B += GWNS_B*weight*vec_x; + phi_y_B += GWNS_B*weight*vec_y; + phi_z_B += GWNS_B*weight*vec_z; + } + } + } + } + SolidForceA_host[idx+0*Np] = phi_x_A; + SolidForceA_host[idx+1*Np] = phi_y_A; + SolidForceA_host[idx+2*Np] = phi_z_A; + SolidForceB_host[idx+0*Np] = phi_x_B; + SolidForceB_host[idx+1*Np] = phi_y_B; + SolidForceB_host[idx+2*Np] = phi_z_B; + } + } + } + } // Set Dm to match Mask for (int i=0; iid[i] = Mask->id[i]; @@ -284,10 +449,98 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm } printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); } + + //Copy all data to device + ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); + ScaLBL_CopyToDevice(SolidForceA, SolidForceA_host, 3*Np*sizeof(double)); + ScaLBL_CopyToDevice(SolidForceB, SolidForceB_host, 3*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] SolidPotentialA_host; + delete [] SolidPotentialB_host; + delete [] SolidForceA_host; + delete [] SolidForceB_host; + delete [] Poros; + delete [] Perm; + delete [] Dst; } +void ScaLBL_GreyscaleSCModel::Density_Init(){ -void ScaLBL_GreyscaleModel::Create(){ + size_t NLABELS=0; + signed char VALUE=0; + + vector LabelList{1,2}; + vector SwList{0.0,1.0}; + + if (greyscaleSC_db->keyExists( "GreyNodeLabels" )){ + LabelList.clear(); + LabelList = greyscaleSC_db->getVector( "GreyNodeLabels" ); + } + if (greyscaleSC_db->keyExists( "GreyNodeSw" )){ + SwList.clear(); + SwList = greyscaleSC_db->getVector( "GreyNodeSw" ); + } + + NLABELS=LabelList.size(); + if (NLABELS != SwList.size()){ + ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); + } + + double *Den_temp; + Den_temp=new double [2*Np]; + double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. + double nB=0.5; + + //double *Phi_temp; + //Phi_temp=new double [Np]; + //double phi = 0.0; + + for (int k=0; kid[n]; + if (VALUE>0){ + for (unsigned int idx=0; idx < NLABELS; idx++){ + if (VALUE == LabelList[idx]){ + double Sw = SwList[idx]; + if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); + nB=Sw; + nA=1.0-Sw; + //phi = nA-nB; + idx = NLABELS; + } + } + if (VALUE==1){//label=1 reserved for NW phase + //TODO; maybe need rho_major and rho_minor initialization + nA=rhoA; + nB=rhoB_minor; + //phi = nA-nB; + } + else if(VALUE==2){//label=2 reserved for W phase + //TODO; maybe need rho_major and rho_minor initialization + nA=rhoA_minor; + nB=rhoB; + //phi = nA-nB; + } + int idx = Map(i,j,k); + Den_temp[idx+0*Np] = nA; + Den_temp[idx+1*Np] = nB; + //Phi_temp[idx] = phi; + } + } + } + } + //copy to device + ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); + //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] Den_temp; + //delete [] Phi_temp; +} + +void ScaLBL_GreyscaleSCModel::Create(){ /* * This function creates the variables needed to run a LBM */ @@ -324,70 +577,90 @@ void ScaLBL_GreyscaleModel::Create(){ neighborSize=18*(Np*sizeof(int)); //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &fqA, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &fqB, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidForceA, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidForceB, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device neighbor list \n"); fflush(stdout); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - // initialize phi based on PhaseLabel (include solid component labels) - double *Poros, *Perm; - Poros = new double[Np]; - Perm = new double[Np]; - AssignComponentLabels(Poros,Perm); - ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); - ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); - delete [] Poros; - delete [] Perm; } - -void ScaLBL_GreyscaleModel::Initialize(){ - if (rank==0) printf ("Initializing distributions \n"); - //TODO: for BGK, you need to consider voxel porosity - // for IMRT, the whole set of feq is different - // if in the future you have different collison mode, need to write two set of initialization functions - if (CollisionType==1){ - ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); - if (rank==0) printf("Collision model: Incompressible MRT.\n"); - } - else if (CollisionType==2){ - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf("Collision model: BGK.\n"); - } - else{ - if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); - ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); - CollisionType=1; - greyscale_db->putScalar( "collision", "IMRT" ); - } - +void ScaLBL_GreyscaleSCModel::Initialize(){ if (Restart == true){ - if (rank==0){ - printf("Initializing distributions from Restart! \n"); - } - // Read in the restart file to CPU buffers - std::shared_ptr cfq; - cfq = std::shared_ptr(new double[19*Np],DeleteArray); - FILE *File; - File=fopen(LocalRestartFile,"rb"); - fread(cfq.get(),sizeof(double),19*Np,File); - fclose(File); - - // Copy the restart data to the GPU - ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - - MPI_Barrier(comm); +// //TODO: Restart funtion is currently not working; need updates +// if (rank==0){ +// printf("Initializing density field and distributions from Restart! \n"); +// } +// // Read in the restart file to CPU buffers +// std::shared_ptr cfq; +// cfq = std::shared_ptr(new double[19*Np],DeleteArray); +// std::shared_ptr cDen; +// cDen = std::shared_ptr(new double[2*Np],DeleteArray); +// FILE *File; +// File=fopen(LocalRestartFile,"rb"); +// fread(cfq.get(),sizeof(double),19*Np,File); +// fread(cDen.get(),sizeof(double),2*Np,File); +// fclose(File); +// +// // Copy the restart data to the GPU +// ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); +// ScaLBL_DeviceBarrier(); +// MPI_Barrier(comm); +// +// //TODO need proper initialization ! +// +// //TODO need to initialize velocity field ! +// //this is required for calculating the pressure_dvc +// //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity } + else{ + if (rank==0) printf ("Initializing solid affinities \n"); + AssignGreyscaleAndSolidLabels(); + if (rank==0) printf ("Initializing density field \n"); + Density_Init();//initialize density field + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_GreyscaleSC_Init(fqA, fqB, Den, Np); + + //debug + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A_init.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B_init.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + //Velocity also needs initialization (for old incompressible momentum transport) + //if (rank==0) printf ("Initializing velocity field \n"); + //double *vel_init; + //vel_init = new double [3*Np]; + //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; + //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); + //ScaLBL_DeviceBarrier(); + //delete [] vel_init; + } } -void ScaLBL_GreyscaleModel::Run(){ +void ScaLBL_GreyscaleSCModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -403,8 +676,8 @@ void ScaLBL_GreyscaleModel::Run(){ if (analysis_db->keyExists( "restart_interval" )){ restart_interval = analysis_db->getScalar( "restart_interval" ); } - if (greyscale_db->keyExists( "timestep" )){ - timestep = greyscale_db->getScalar( "timestep" ); + if (greyscaleSC_db->keyExists( "timestep" )){ + timestep = greyscaleSC_db->getScalar( "timestep" ); } if (rank==0){ @@ -425,235 +698,310 @@ void ScaLBL_GreyscaleModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); auto current_db = db->cloneDatabase(); - double rlx = 1.0/tau; - double rlx_eff = 1.0/tau_eff; double error = 1.0; double flow_rate_previous = 0.0; while (timestep < timestepMax && error > tolerance) { //************************************************************************/ // *************ODD TIMESTEP*************// timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + //ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + //ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + //ScaLBL_DeviceBarrier(); + //ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); + + // Compute density gradient + // fluid component A + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); + ScaLBL_DeviceBarrier(); + // fluid component B + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[Np]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); + ScaLBL_DeviceBarrier(); + + //debug + //DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + //FILE *AFILE; + //sprintf(LocalRankFilename,"A_time_%i_prior.%05i.raw",timestep,rank); + //AFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,AFILE); + //fclose(AFILE); + + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + //FILE *BFILE; + //sprintf(LocalRankFilename,"B_time_%i_prior.%05i.raw",timestep,rank); + //BFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,BFILE); + //fclose(BFILE); + + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } + //if (BoundaryCondition == 3){ + // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + //} + ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + //debug + ////DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ////FILE *AFILE; + //sprintf(LocalRankFilename,"A_time_%i_after.%05i.raw",timestep,rank); + //AFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,AFILE); + //fclose(AFILE); + + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ////FILE *BFILE; + //sprintf(LocalRankFilename,"B_time_%i_after.%05i.raw",timestep,rank); + //BFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,BFILE); + //fclose(BFILE); // *************EVEN TIMESTEP*************// timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + //ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + //ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + //ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + //ScaLBL_DeviceBarrier(); + //ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); + + // Compute density gradient + // fluid component A + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[0]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); + ScaLBL_DeviceBarrier(); + // fluid component B + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(&Den[Np]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); + ScaLBL_DeviceBarrier(); + + //debug + ////DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ////FILE *AFILE; + //sprintf(LocalRankFilename,"A_time_%i_prior.%05i.raw",timestep,rank); + //AFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,AFILE); + //fclose(AFILE); + + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ////FILE *BFILE; + //sprintf(LocalRankFilename,"B_time_%i_prior.%05i.raw",timestep,rank); + //BFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,BFILE); + //fclose(BFILE); + + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - switch (CollisionType){ - case 1: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - case 2: - ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); - break; - default: - ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); - break; - } - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //if (BoundaryCondition == 3){ + // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + //} + ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + //debug + ////DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ////FILE *AFILE; + //sprintf(LocalRankFilename,"A_time_%i_after.%05i.raw",timestep,rank); + //AFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,AFILE); + //fclose(AFILE); + + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ////FILE *BFILE; + //sprintf(LocalRankFilename,"B_time_%i_after.%05i.raw",timestep,rank); + //BFILE = fopen(LocalRankFilename,"wb"); + //fwrite(PhaseField.data(),8,N,BFILE); + //fclose(BFILE); + //************************************************************************/ - if (timestep%analysis_interval==0){ - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); - //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); - - double count_loc=0; - double count; - double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - //double px_loc,py_loc,pz_loc; - //double px,py,pz; - //double mass_loc,mass_glb; - - //parameters for domain average - int64_t i,j,k,n,imin,jmin,kmin,kmax; - // If external boundary conditions are set, do not average over the inlet and outlet - kmin=1; kmax=Nz-1; - //In case user forgets to specify the inlet/outlet buffer layers for BC>0 - if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; - if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; - - imin=jmin=1; - // If inlet/outlet layers exist use these as default - //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; - //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; - if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer - if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; - -// px_loc = py_loc = pz_loc = 0.f; -// mass_loc = 0.f; +// if (timestep%analysis_interval==0){ +// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); +// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); +// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); +// +// double count_loc=0; +// double count; +// double vax,vay,vaz; +// double vax_loc,vay_loc,vaz_loc; +// //double px_loc,py_loc,pz_loc; +// //double px,py,pz; +// //double mass_loc,mass_glb; +// +// //parameters for domain average +// int64_t i,j,k,n,imin,jmin,kmin,kmax; +// // If external boundary conditions are set, do not average over the inlet and outlet +// kmin=1; kmax=Nz-1; +// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 +// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; +// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; +// +// imin=jmin=1; +// // If inlet/outlet layers exist use these as default +// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; +// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; +// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer +// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; +// +//// px_loc = py_loc = pz_loc = 0.f; +//// mass_loc = 0.f; +//// for (int k=kmin; k 0){ +//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +//// mass_loc += Den*PorosityMap(i,j,k); +//// } +//// } +//// } +//// } +//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// +//// vax = px/mass_glb; +//// vay = py/mass_glb; +//// vaz = pz/mass_glb; +// +// vax_loc = vay_loc = vaz_loc = 0.f; // for (int k=kmin; k 0){ -// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); -// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); -// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); -// mass_loc += Den*PorosityMap(i,j,k); +// vax_loc += Velocity_x(i,j,k); +// vay_loc += Velocity_y(i,j,k); +// vaz_loc += Velocity_z(i,j,k); +// count_loc+=1.0; // } // } // } // } -// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +// vax = Mask->Comm.sumReduce( vax_loc ); +// vay = Mask->Comm.sumReduce( vay_loc ); +// vaz = Mask->Comm.sumReduce( vaz_loc ); +// count = Mask->Comm.sumReduce( count_loc ); +// +// vax /= count; +// vay /= count; +// vaz /= count; +// +// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); +// double dir_x = Fx/force_mag; +// double dir_y = Fy/force_mag; +// double dir_z = Fz/force_mag; +// if (force_mag == 0.0){ +// // default to z direction +// dir_x = 0.0; +// dir_y = 0.0; +// dir_z = 1.0; +// force_mag = 1.0; +// } +// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; +// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); // -// vax = px/mass_glb; -// vay = py/mass_glb; -// vaz = pz/mass_glb; - - vax_loc = vay_loc = vaz_loc = 0.f; - for (int k=kmin; k 0){ - vax_loc += Velocity_x(i,j,k); - vay_loc += Velocity_y(i,j,k); - vaz_loc += Velocity_z(i,j,k); - count_loc+=1.0; - } - } - } - } - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); - - vax /= count; - vay /= count; - vaz /= count; - - double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - double dir_x = Fx/force_mag; - double dir_y = Fy/force_mag; - double dir_z = Fz/force_mag; - if (force_mag == 0.0){ - // default to z direction - dir_x = 0.0; - dir_y = 0.0; - dir_z = 1.0; - force_mag = 1.0; - } - //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; - double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); - - error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); - flow_rate_previous = flow_rate; - - //if (rank==0) printf("Computing Minkowski functionals \n"); - Morphology.ComputeScalar(SignDist,0.f); - //Morphology.PrintAll(); - double mu = (tau-0.5)/3.f; - double Vs = Morphology.V(); - double As = Morphology.A(); - double Hs = Morphology.H(); - double Xs = Morphology.X(); - Vs = Dm->Comm.sumReduce( Vs); - As = Dm->Comm.sumReduce( As); - Hs = Dm->Comm.sumReduce( Hs); - Xs = Dm->Comm.sumReduce( Xs); - - double h = Dm->voxel_length; - //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; - double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; - - if (rank==0){ - printf(" AbsPerm = %.5g [micron^2]\n",absperm); - bool WriteHeader=false; - FILE * log_file = fopen("Permeability.csv","r"); - if (log_file != NULL) - fclose(log_file); - else - WriteHeader=true; - log_file = fopen("Permeability.csv","a"); - if (WriteHeader) - fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", - timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); - - fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, - h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); - fclose(log_file); - } - } +// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); +// flow_rate_previous = flow_rate; +// +// //if (rank==0) printf("Computing Minkowski functionals \n"); +// Morphology.ComputeScalar(SignDist,0.f); +// //Morphology.PrintAll(); +// double mu = (tau-0.5)/3.f; +// double Vs = Morphology.V(); +// double As = Morphology.A(); +// double Hs = Morphology.H(); +// double Xs = Morphology.X(); +// Vs = Dm->Comm.sumReduce( Vs); +// As = Dm->Comm.sumReduce( As); +// Hs = Dm->Comm.sumReduce( Hs); +// Xs = Dm->Comm.sumReduce( Xs); +// +// double h = Dm->voxel_length; +// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; +// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; +// +// if (rank==0){ +// printf(" AbsPerm = %.5g [micron^2]\n",absperm); +// bool WriteHeader=false; +// FILE * log_file = fopen("Permeability.csv","r"); +// if (log_file != NULL) +// fclose(log_file); +// else +// WriteHeader=true; +// log_file = fopen("Permeability.csv","a"); +// if (WriteHeader) +// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", +// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); +// +// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, +// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); +// fclose(log_file); +// } +// } if (timestep%visualization_interval==0){ VelocityField(); } - if (timestep%restart_interval==0){ - //Use rank=0 write out Restart.db - if (rank==0) { - greyscale_db->putScalar("timestep",timestep); - greyscale_db->putScalar( "Restart", true ); - current_db->putDatabase("Greyscale", greyscale_db); - std::ofstream OutStream("Restart.db"); - current_db->print(OutStream, ""); - OutStream.close(); - - } - //Write out Restart data. - std::shared_ptr cfq; - cfq = std::shared_ptr(new double[19*Np],DeleteArray); - ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU - - FILE *RESTARTFILE; - RESTARTFILE=fopen(LocalRestartFile,"wb"); - fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); - fclose(RESTARTFILE); - MPI_Barrier(comm); - } +// if (timestep%restart_interval==0){ +// //Use rank=0 write out Restart.db +// if (rank==0) { +// greyscaleSC_db->putScalar("timestep",timestep); +// greyscaleSC_db->putScalar( "Restart", true ); +// current_db->putDatabase("GreyscaleSC", greyscaleSC_db); +// std::ofstream OutStream("Restart.db"); +// current_db->print(OutStream, ""); +// OutStream.close(); +// +// } +// //Write out Restart data. +// std::shared_ptr cfq; +// cfq = std::shared_ptr(new double[19*Np],DeleteArray); +// ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU +// +// FILE *RESTARTFILE; +// RESTARTFILE=fopen(LocalRestartFile,"wb"); +// fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); +// fclose(RESTARTFILE); +// MPI_Barrier(comm); +// } } PROFILE_STOP("Loop"); @@ -678,7 +1026,7 @@ void ScaLBL_GreyscaleModel::Run(){ // ************************************************************************ } -void ScaLBL_GreyscaleModel::VelocityField(){ +void ScaLBL_GreyscaleSCModel::VelocityField(){ /* Minkowski Morphology(Mask); int SIZE=Np*sizeof(double); @@ -794,7 +1142,7 @@ void ScaLBL_GreyscaleModel::VelocityField(){ } -void ScaLBL_GreyscaleModel::WriteDebug(){ +void ScaLBL_GreyscaleSCModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); @@ -806,26 +1154,26 @@ void ScaLBL_GreyscaleModel::WriteDebug(){ // fwrite(PhaseField.data(),8,N,OUTFILE); // fclose(OUTFILE); // -// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A.%05i.raw",rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B.%05i.raw",rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); -// -// ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); -// FILE *PFILE; -// sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); -// PFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,PFILE); -// fclose(PFILE); + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); FILE *VELX_FILE; @@ -848,17 +1196,17 @@ void ScaLBL_GreyscaleModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); - ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); - FILE *POROS_FILE; - sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); - POROS_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,POROS_FILE); - fclose(POROS_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); - FILE *PERM_FILE; - sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); - PERM_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PERM_FILE); - fclose(PERM_FILE); +// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); +// FILE *POROS_FILE; +// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); +// POROS_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,POROS_FILE); +// fclose(POROS_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); +// FILE *PERM_FILE; +// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); +// PERM_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PERM_FILE); +// fclose(PERM_FILE); } diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index 3e883b16..348490c4 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -16,10 +16,10 @@ Implementation of color lattice boltzmann model #include "ProfilerApp.h" #include "threadpool/thread_pool.h" -class ScaLBL_GreyscaleModel{ +class ScaLBL_GreyscaleSCModel{ public: - ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleModel(); + ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleSCModel(); // functions in they should be run void ReadParams(string filename); @@ -36,9 +36,11 @@ public: int timestep,timestepMax; int BoundaryCondition; int CollisionType; - double tau; - double tau_eff; - double Den;//constant density + double tauA,tauB; + double tauA_eff,tauB_eff; + double Gsc; + double rhoA,rhoB; + double rhoA_minor,rhoB_minor;//dissolved density double tolerance; double Fx,Fy,Fz,flux; double din,dout; @@ -56,17 +58,21 @@ public: // input database std::shared_ptr db; std::shared_ptr domain_db; - std::shared_ptr greyscale_db; + std::shared_ptr greyscaleSC_db; std::shared_ptr analysis_db; std::shared_ptr vis_db; signed char *id; int *NeighborList; - double *fq; + double *fqA, *fqB; double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; double *Pressure_dvc; + double *Den; + double *DenGradA,*DenGradB; + double *SolidForceA,*SolidForceB; + IntArray Map; DoubleArray SignDist; DoubleArray Velocity_x; @@ -85,7 +91,7 @@ private: char LocalRankFilename[40]; char LocalRestartFile[40]; - void AssignComponentLabels(double *Porosity, double *Permeablity); - + void AssignGreyscaleAndSolidLabels(); + void Density_Init(); }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ca88274a..362d020e 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,6 +5,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscaleFE_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_greyscaleSC_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_greyscaleSC_simulator.cpp b/tests/lbpm_greyscaleSC_simulator.cpp new file mode 100644 index 00000000..b95fff04 --- /dev/null +++ b/tests/lbpm_greyscaleSC_simulator.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI.h" +#include "models/GreyscaleSCModel.h" +//#define WRITE_SURFACES + +using namespace std; + + +int main(int argc, char **argv) +{ + //***************************************** + // ***** MPI STUFF **************** + //***************************************** + // Initialize MPI + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { + // parallel domain size (# of sub-domains) + int nprocx,nprocy,nprocz; + int iproc,jproc,kproc; + + if (rank == 0){ + printf("****************************************\n"); + printf("Running Greyscale Two-Phase Calculation \n"); + printf("****************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + ScaLBL_GreyscaleSCModel GreyscaleSC(rank,nprocs,comm); + auto filename = argv[1]; + GreyscaleSC.ReadParams(filename); + GreyscaleSC.SetDomain(); // this reads in the domain + GreyscaleSC.ReadInput(); + GreyscaleSC.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleSC.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleSC.Run(); + //GreyscaleSC.VelocityField(); + GreyscaleSC.WriteDebug(); + } + // **************************************************** + MPI_Barrier(comm); + MPI_Finalize(); + // **************************************************** +} From c423d14e7418123db10e70411015c53e7d1c6431 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 1 May 2020 17:44:48 -0400 Subject: [PATCH 130/270] GreyscaleSC debugging: save the work; yet to function correctly --- gpu/GreyscaleSC.cu | 592 ++++++++++++++++++------------------ models/GreyscaleSCModel.cpp | 183 +++++------ 2 files changed, 367 insertions(+), 408 deletions(-) diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index 6219d42f..951b00e4 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -21,7 +21,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; double fq; - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) double porosity; double perm;//voxel permeability @@ -687,9 +687,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di c1 = porosity*0.5*GeoFun/sqrt(permA); if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - vx = jxA/rhoA_next+0.5*(porosity*Gx+GffA_x+GfsA_x); - vy = jyA/rhoA_next+0.5*(porosity*Gy+GffA_y+GfsA_y); - vz = jzA/rhoA_next+0.5*(porosity*Gz+GffA_z+GfsA_z); + vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); + vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); + vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); v_mag=sqrt(vx*vx+vy*vy+vz*vz); ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); @@ -713,9 +713,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di c1 = porosity*0.5*GeoFun/sqrt(permB); if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - vx = jxB/rhoB_next+0.5*(porosity*Gx+GffB_x+GfsB_x); - vy = jyB/rhoB_next+0.5*(porosity*Gy+GffB_y+GfsB_y); - vz = jzB/rhoB_next+0.5*(porosity*Gz+GffB_z+GfsB_z); + vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); + vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); + vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); v_mag=sqrt(vx*vx+vy*vy+vz*vz); ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); @@ -733,9 +733,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di } // Calculate barycentric velocity of the fluid mixture - ux = (rhoA_next*ux_A+rhoB_next*ux_B)/(rhoA_next+rhoB_next); - uy = (rhoA_next*uy_A+rhoB_next*uy_B)/(rhoA_next+rhoB_next); - uz = (rhoA_next*uz_A+rhoB_next*uz_B)/(rhoA_next+rhoB_next); + ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); + uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); + uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); // //..............carry out relaxation process............................................... // m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) @@ -776,32 +776,32 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1A = m1A + rlx_setA*((19*rhoA_next*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); - m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA_next*(ux*ux+uy*uy+uz*uz))- m2A) + m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); jxA = jxA + FxA; - m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA_next)- m4A) + m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); jyA = jyA + FyA; - m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA_next)- m6A) + m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA)- m6A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); jzA = jzA + FzA; - m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA_next)- m8A) + m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA)- m8A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); - m9A = m9A + rlx_setA*((rhoA_next*(2*ux*ux-uy*uy-uz*uz)) - m9A) + m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); m10A = m10A + rlx_setA*( - m10A) + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); - m11A = m11A + rlx_setA*((rhoA_next*(uy*uy-uz*uz)) - m11A) + m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); - m12A = m12A + rlx_setA*( - m12A); + m12A = m12A + rlx_setA*( - m12A) + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); - m13A = m13A + rlx_setA*( rhoA_next*(ux*uy) - m13A) + m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); - m14A = m14A + rlx_setA*( rhoA_next*(uy*uz) - m14A); + m14A = m14A + rlx_setA*( rhoA*(uy*uz) - m14A) + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); - m15A = m15A + rlx_setA*( rhoA_next*(ux*uz) - m15A) + m15A = m15A + rlx_setA*( rhoA*(ux*uz) - m15A) + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); m16A = m16A + rlx_setB*( - m16A); m17A = m17A + rlx_setB*( - m17A); @@ -812,121 +812,121 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di // ------------------- Fluid Component A -----------------------// //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; - f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; - distA[n] = f0; + fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + //f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + distA[n] = fq; // q = 1 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); - f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + //f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); nread = neighborList[n+Np]; - distA[nread] = f1; + distA[nread] = fq; // q=2 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); - f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + //f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); nread = neighborList[n]; - distA[nread] = f2; + distA[nread] = fq; // q = 3 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + //f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+3*Np]; - distA[nread] = f3; + distA[nread] = fq; // q = 4 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + //f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+2*Np]; - distA[nread] = f4; + distA[nread] = fq; // q = 5 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + //f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+5*Np]; - distA[nread] = f5; + distA[nread] = fq; // q = 6 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + //f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+4*Np]; - distA[nread] = f6; + distA[nread] = fq; // q = 7 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); - f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + //f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); nread = neighborList[n+7*Np]; - distA[nread] = f7; + distA[nread] = fq; // q = 8 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); - f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + //f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); nread = neighborList[n+6*Np]; - distA[nread] = f8; + distA[nread] = fq; // q = 9 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); - f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + //f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); nread = neighborList[n+9*Np]; - distA[nread] = f9; + distA[nread] = fq; // q = 10 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); - f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + //f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); nread = neighborList[n+8*Np]; - distA[nread] = f10; + distA[nread] = fq; // q = 11 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); - f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + //f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); nread = neighborList[n+11*Np]; - distA[nread] = f11; + distA[nread] = fq; // q = 12 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); - f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + //f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); nread = neighborList[n+10*Np]; - distA[nread]= f12; + distA[nread]= fq; // q = 13 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); - f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + //f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); nread = neighborList[n+13*Np]; - distA[nread] = f13; + distA[nread] = fq; // q= 14 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); - f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + //f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); nread = neighborList[n+12*Np]; - distA[nread] = f14; + distA[nread] = fq; // q = 15 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); - f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + //f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); nread = neighborList[n+15*Np]; - distA[nread] = f15; + distA[nread] = fq; // q = 16 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); - f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + //f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); nread = neighborList[n+14*Np]; - distA[nread] = f16; + distA[nread] = fq; // q = 17 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); - f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + //f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); nread = neighborList[n+17*Np]; - distA[nread] = f17; + distA[nread] = fq; // q = 18 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); - f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + //f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); nread = neighborList[n+16*Np]; - distA[nread] = f18; + distA[nread] = fq; //........................................................................ - Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + //Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; // //..............carry out relaxation process............................................... @@ -968,32 +968,32 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1B = m1B + rlx_setA*((19*rhoB_next*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); - m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB_next*(ux*ux+uy*uy+uz*uz))- m2B) + m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); jxB = jxB + FxB; - m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB_next)- m4B) + m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); jyB = jyB + FyB; - m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB_next)- m6B) + m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB)- m6B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); jzB = jzB + FzB; - m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB_next)- m8B) + m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB)- m8B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); - m9B = m9B + rlx_setA*((rhoB_next*(2*ux*ux-uy*uy-uz*uz)) - m9B) + m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); m10B = m10B + rlx_setA*( - m10B) + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); - m11B = m11B + rlx_setA*((rhoB_next*(uy*uy-uz*uz)) - m11B) + m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); m12B = m12B + rlx_setA*( - m12B) + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); - m13B = m13B + rlx_setA*( rhoB_next*(ux*uy) - m13B) + m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); - m14B = m14B + rlx_setA*( rhoB_next*(uy*uz) - m14B) + m14B = m14B + rlx_setA*( rhoB*(uy*uz) - m14B) + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); - m15B = m15B + rlx_setA*( rhoB_next*(ux*uz) - m15B) + m15B = m15B + rlx_setA*( rhoB*(ux*uz) - m15B) + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); m16B = m16B + rlx_setB*( - m16B); m17B = m17B + rlx_setB*( - m17B); @@ -1004,127 +1004,127 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di // ------------------- Fluid Component B -----------------------// //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; - f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; - distB[n] = f0; + fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + //f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + distB[n] = fq; // q = 1 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); - f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + //f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); nread = neighborList[n+Np]; - distB[nread] = f1; + distB[nread] = fq; // q=2 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); - f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + //f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); nread = neighborList[n]; - distB[nread] = f2; + distB[nread] = fq; // q = 3 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + //f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); nread = neighborList[n+3*Np]; - distB[nread] = f3; + distB[nread] = fq; // q = 4 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + //f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); nread = neighborList[n+2*Np]; - distB[nread] = f4; + distB[nread] = fq; // q = 5 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + //f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); nread = neighborList[n+5*Np]; - distB[nread] = f5; + distB[nread] = fq; // q = 6 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + //f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); nread = neighborList[n+4*Np]; - distB[nread] = f6; + distB[nread] = fq; // q = 7 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); - f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + //f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); nread = neighborList[n+7*Np]; - distB[nread] = f7; + distB[nread] = fq; // q = 8 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); - f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + //f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); nread = neighborList[n+6*Np]; - distB[nread] = f8; + distB[nread] = fq; // q = 9 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); - f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + //f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); nread = neighborList[n+9*Np]; - distB[nread] = f9; + distB[nread] = fq; // q = 10 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); - f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + //f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); nread = neighborList[n+8*Np]; - distB[nread] = f10; + distB[nread] = fq; // q = 11 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); - f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + //f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); nread = neighborList[n+11*Np]; - distB[nread] = f11; + distB[nread] = fq; // q = 12 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); - f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + //f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); nread = neighborList[n+10*Np]; - distB[nread]= f12; + distB[nread]= fq; // q = 13 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); - f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + //f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); nread = neighborList[n+13*Np]; - distB[nread] = f13; + distB[nread] = fq; // q= 14 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); - f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + //f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); nread = neighborList[n+12*Np]; - distB[nread] = f14; + distB[nread] = fq; // q = 15 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); - f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + //f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); nread = neighborList[n+15*Np]; - distB[nread] = f15; + distB[nread] = fq; // q = 16 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); - f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + //f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); nread = neighborList[n+14*Np]; - distB[nread] = f16; + distB[nread] = fq; // q = 17 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); - f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + //f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); nread = neighborList[n+17*Np]; - distB[nread] = f17; + distB[nread] = fq; // q = 18 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); - f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + //f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); nread = neighborList[n+16*Np]; - distB[nread] = f18; + distB[nread] = fq; //........................................................................ - Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + //Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; //Update velocity on device Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; //Update pressure on device - Pressure[n] = (rhoA_next+rhoB_next+Gsc*rhoA_next*rhoB_next)/3.0; + Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; //Update density //Den[n] = rhoA_next; //Den[n+Np] = rhoB_next; @@ -1151,7 +1151,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; double fq; - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) double porosity; double perm;//voxel permeability @@ -1781,9 +1781,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB c1 = porosity*0.5*GeoFun/sqrt(permA); if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - vx = jxA/rhoA_next+0.5*(porosity*Gx+GffA_x+GfsA_x); - vy = jyA/rhoA_next+0.5*(porosity*Gy+GffA_y+GfsA_y); - vz = jzA/rhoA_next+0.5*(porosity*Gz+GffA_z+GfsA_z); + vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); + vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); + vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); v_mag=sqrt(vx*vx+vy*vy+vz*vz); ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); @@ -1807,9 +1807,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB c1 = porosity*0.5*GeoFun/sqrt(permB); if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - vx = jxB/rhoB_next+0.5*(porosity*Gx+GffB_x+GfsB_x); - vy = jyB/rhoB_next+0.5*(porosity*Gy+GffB_y+GfsB_y); - vz = jzB/rhoB_next+0.5*(porosity*Gz+GffB_z+GfsB_z); + vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); + vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); + vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); v_mag=sqrt(vx*vx+vy*vy+vz*vz); ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); @@ -1827,9 +1827,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB } // Calculate barycentric velocity of the fluid mixture - ux = (rhoA_next*ux_A+rhoB_next*ux_B)/(rhoA_next+rhoB_next); - uy = (rhoA_next*uy_A+rhoB_next*uy_B)/(rhoA_next+rhoB_next); - uz = (rhoA_next*uz_A+rhoB_next*uz_B)/(rhoA_next+rhoB_next); + ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); + uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); + uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); // //..............carry out relaxation process............................................... @@ -1872,32 +1872,32 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1A = m1A + rlx_setA*((19*rhoA_next*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); - m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA_next*(ux*ux+uy*uy+uz*uz))- m2A) + m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); jxA = jxA + FxA; - m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA_next)- m4A) + m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); jyA = jyA + FyA; - m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA_next)- m6A) + m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA)- m6A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); jzA = jzA + FzA; - m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA_next)- m8A) + m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA)- m8A) + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); - m9A = m9A + rlx_setA*((rhoA_next*(2*ux*ux-uy*uy-uz*uz)) - m9A) + m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); m10A = m10A + rlx_setA*( - m10A) + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); - m11A = m11A + rlx_setA*((rhoA_next*(uy*uy-uz*uz)) - m11A) + m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); - m12A = m12A + rlx_setA*( - m12A); + m12A = m12A + rlx_setA*( - m12A) + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); - m13A = m13A + rlx_setA*( rhoA_next*(ux*uy) - m13A) + m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); - m14A = m14A + rlx_setA*( rhoA_next*(uy*uz) - m14A); + m14A = m14A + rlx_setA*( rhoA*(uy*uz) - m14A) + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); - m15A = m15A + rlx_setA*( rhoA_next*(ux*uz) - m15A) + m15A = m15A + rlx_setA*( rhoA*(ux*uz) - m15A) + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); m16A = m16A + rlx_setB*( - m16A); m17A = m17A + rlx_setB*( - m17A); @@ -1908,102 +1908,102 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB // ------------------- Fluid Component A -----------------------// //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; - f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; - distA[n] = f0; + fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + //f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + distA[n] = fq; // q = 1 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); - f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); - distA[1*Np+n] = f1; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + //f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + distA[1*Np+n] = fq; // q=2 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); - f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); - distA[2*Np+n] = f2; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + //f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + distA[2*Np+n] = fq; // q = 3 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - distA[3*Np+n] = f3; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + //f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + distA[3*Np+n] = fq; // q = 4 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - distA[4*Np+n] = f4; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + //f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + distA[4*Np+n] = fq; // q = 5 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - distA[5*Np+n] = f5; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + //f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + distA[5*Np+n] = fq; // q = 6 - //fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - distA[6*Np+n] = f6; + fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + //f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + distA[6*Np+n] = fq; // q = 7 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); - f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); - distA[7*Np+n] = f7; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + //f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + distA[7*Np+n] = fq; // q = 8 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); - f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); - distA[8*Np+n] = f8; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + //f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + distA[8*Np+n] = fq; // q = 9 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); - f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); - distA[9*Np+n] = f9; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + //f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + distA[9*Np+n] = fq; // q = 10 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); - f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); - distA[10*Np+n] = f10; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + //f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + distA[10*Np+n] = fq; // q = 11 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); - f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); - distA[11*Np+n] = f11; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + //f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + distA[11*Np+n] = fq; // q = 12 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); - f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); - distA[12*Np+n] = f12; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + //f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + distA[12*Np+n] = fq; // q = 13 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); - f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); - distA[13*Np+n] = f13; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + //f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + distA[13*Np+n] = fq; // q= 14 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); - f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); - distA[14*Np+n] = f14; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + //f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + distA[14*Np+n] = fq; // q = 15 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); - f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); - distA[15*Np+n] = f15; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + //f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + distA[15*Np+n] = fq; // q = 16 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); - f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); - distA[16*Np+n] = f16; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + //f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + distA[16*Np+n] = fq; // q = 17 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); - f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); - distA[17*Np+n] = f17; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + //f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + distA[17*Np+n] = fq; // q = 18 - //fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); - f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); - distA[18*Np+n] = f18; + fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + //f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + distA[18*Np+n] = fq; //........................................................................ - Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + //Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; // //..............carry out relaxation process............................................... // m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) @@ -2045,32 +2045,32 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1B = m1B + rlx_setA*((19*rhoB_next*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); - m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB_next*(ux*ux+uy*uy+uz*uz))- m2B) + m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); jxB = jxB + FxB; - m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB_next)- m4B) + m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); jyB = jyB + FyB; - m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB_next)- m6B) + m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB)- m6B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); jzB = jzB + FzB; - m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB_next)- m8B) + m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB)- m8B) + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); - m9B = m9B + rlx_setA*((rhoB_next*(2*ux*ux-uy*uy-uz*uz)) - m9B) + m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); m10B = m10B + rlx_setA*( - m10B) + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); - m11B = m11B + rlx_setA*((rhoB_next*(uy*uy-uz*uz)) - m11B) + m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); m12B = m12B + rlx_setA*( - m12B) + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); - m13B = m13B + rlx_setA*( rhoB_next*(ux*uy) - m13B) + m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); - m14B = m14B + rlx_setA*( rhoB_next*(uy*uz) - m14B) + m14B = m14B + rlx_setA*( rhoB*(uy*uz) - m14B) + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); - m15B = m15B + rlx_setA*( rhoB_next*(ux*uz) - m15B) + m15B = m15B + rlx_setA*( rhoB*(ux*uz) - m15B) + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); m16B = m16B + rlx_setB*( - m16B); m17B = m17B + rlx_setB*( - m17B); @@ -2081,108 +2081,108 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB // ------------------- Fluid Component B -----------------------// //.................inverse transformation...................................................... // q=0 - //fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; - f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; - distB[n] = f0; + fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + //f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + distB[n] = fq; // q = 1 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); - f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); - distB[1*Np+n] = f1; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + //f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + distB[1*Np+n] = fq; // q=2 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); - f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); - distB[2*Np+n] = f2; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + //f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + distB[2*Np+n] = fq; // q = 3 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - distB[3*Np+n] = f3; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + //f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + distB[3*Np+n] = fq; // q = 4 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - distB[4*Np+n] = f4; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + //f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + distB[4*Np+n] = fq; // q = 5 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - distB[5*Np+n] = f5; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + //f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + distB[5*Np+n] = fq; // q = 6 - //fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - distB[6*Np+n] = f6; + fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + //f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + distB[6*Np+n] = fq; // q = 7 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); - f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); - distB[7*Np+n] = f7; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + //f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + distB[7*Np+n] = fq; // q = 8 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); - f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); - distB[8*Np+n] = f8; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + //f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + distB[8*Np+n] = fq; // q = 9 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); - f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); - distB[9*Np+n] = f9; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + //f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + distB[9*Np+n] = fq; // q = 10 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); - f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); - distB[10*Np+n] = f10; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + //f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + distB[10*Np+n] = fq; // q = 11 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); - f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); - distB[11*Np+n] = f11; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + //f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + distB[11*Np+n] = fq; // q = 12 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); - f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); - distB[12*Np+n] = f12; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + //f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + distB[12*Np+n] = fq; // q = 13 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); - f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); - distB[13*Np+n] = f13; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + //f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + distB[13*Np+n] = fq; // q= 14 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); - f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); - distB[14*Np+n] = f14; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + //f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + distB[14*Np+n] = fq; // q = 15 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); - f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); - distB[15*Np+n] = f15; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + //f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + distB[15*Np+n] = fq; // q = 16 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); - f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); - distB[16*Np+n] = f16; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + //f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + distB[16*Np+n] = fq; // q = 17 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); - f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); - distB[17*Np+n] = f17; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + //f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + distB[17*Np+n] = fq; // q = 18 - //fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); - f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); - distB[18*Np+n] = f18; + fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + //f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + distB[18*Np+n] = fq; //........................................................................ - Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; + //Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; //Update velocity on device Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; //Update pressure on device - Pressure[n] = (rhoA_next+rhoB_next+Gsc*rhoA_next*rhoB_next)/3.0; + Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; //Update density //Den[n] = rhoA_next; //Den[n+Np] = rhoB_next; diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index da5d08e3..a20f4bc9 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -706,21 +706,73 @@ void ScaLBL_GreyscaleSCModel::Run(){ timestep++; // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) - //ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - //ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - //ScaLBL_DeviceBarrier(); - //ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); - + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); // Compute density gradient // fluid component A ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + // fluid component B + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + // Compute density gradient ScaLBL_Comm->SendHalo(&Den[0]); ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); ScaLBL_DeviceBarrier(); + ScaLBL_Comm->SendHalo(&Den[Np]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); + ScaLBL_DeviceBarrier(); + + //debug + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + + // Collsion + ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + // Collsion + ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// + timestep++; + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); + // Compute density gradient + // fluid component A + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // fluid component B ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + // Compute density gradient + ScaLBL_Comm->SendHalo(&Den[0]); + ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); + ScaLBL_DeviceBarrier(); ScaLBL_Comm->SendHalo(&Den[Np]); ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); @@ -728,124 +780,31 @@ void ScaLBL_GreyscaleSCModel::Run(){ //debug //DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); //FILE *AFILE; - //sprintf(LocalRankFilename,"A_time_%i_prior.%05i.raw",timestep,rank); - //AFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,AFILE); - //fclose(AFILE); + sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); //FILE *BFILE; - //sprintf(LocalRankFilename,"B_time_%i_prior.%05i.raw",timestep,rank); - //BFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,BFILE); - //fclose(BFILE); + sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - //if (BoundaryCondition == 3){ - // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - //} - ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - - //debug - ////DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ////FILE *AFILE; - //sprintf(LocalRankFilename,"A_time_%i_after.%05i.raw",timestep,rank); - //AFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,AFILE); - //fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ////FILE *BFILE; - //sprintf(LocalRankFilename,"B_time_%i_after.%05i.raw",timestep,rank); - //BFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,BFILE); - //fclose(BFILE); - // *************EVEN TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - //ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - //ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - //ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - //ScaLBL_DeviceBarrier(); - //ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); - - // Compute density gradient - // fluid component A - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[0]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); - ScaLBL_DeviceBarrier(); - // fluid component B - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&Den[Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); - ScaLBL_DeviceBarrier(); - - //debug - ////DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ////FILE *AFILE; - //sprintf(LocalRankFilename,"A_time_%i_prior.%05i.raw",timestep,rank); - //AFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,AFILE); - //fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ////FILE *BFILE; - //sprintf(LocalRankFilename,"B_time_%i_prior.%05i.raw",timestep,rank); - //BFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,BFILE); - //fclose(BFILE); - - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + // Collsion ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - //if (BoundaryCondition == 3){ - // ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - // ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - //} + + // Collsion ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - //debug - ////DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ////FILE *AFILE; - //sprintf(LocalRankFilename,"A_time_%i_after.%05i.raw",timestep,rank); - //AFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,AFILE); - //fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ////FILE *BFILE; - //sprintf(LocalRankFilename,"B_time_%i_after.%05i.raw",timestep,rank); - //BFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,BFILE); - //fclose(BFILE); - //************************************************************************/ // if (timestep%analysis_interval==0){ From 9f8af47d2bb35d68a299f7ed949e6a0293336073 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 3 May 2020 18:03:44 -0400 Subject: [PATCH 131/270] continue GreyscaleSC debugging;save the work --- common/ScaLBL.h | 13 +- gpu/GreyscaleSC.cu | 878 +++++++++++++++++++++++++++++++++++- models/GreyscaleSCModel.cpp | 67 +-- 3 files changed, 907 insertions(+), 51 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 5220bed2..526ab5a4 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -124,16 +124,25 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *distB, double *Den, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index 951b00e4..dea5476f 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -3,7 +3,7 @@ #define NBLOCKS 1024 #define NTHREADS 256 -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ @@ -791,11 +791,15 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); - m10A = m10A + rlx_setA*( - m10A) + //m10A = m10A + rlx_setA*( - m10A) + // + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); + m10A = m10A + rlx_setA*( -0.5*(rhoA*(2*ux*ux-uy*uy-uz*uz))- m10A) + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); - m12A = m12A + rlx_setA*( - m12A) + //m12A = m12A + rlx_setA*( - m12A) + // + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); + m12A = m12A + rlx_setA*( -0.5*(rhoA*(uy*uy-uz*uz))- m12A) + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); @@ -983,11 +987,15 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); - m10B = m10B + rlx_setA*( - m10B) + //m10B = m10B + rlx_setA*( - m10B) + // + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); + m10B = m10B + rlx_setA*( -0.5*(rhoB*(2*ux*ux-uy*uy-uz*uz))- m10B) + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); - m12B = m12B + rlx_setA*( - m12B) + //m12B = m12B + rlx_setA*( - m12B) + // + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); + m12B = m12B + rlx_setA*( -0.5*(rhoB*(uy*uy-uz*uz))- m12B) + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); @@ -1133,7 +1141,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC(int *neighborList, double *di } } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ @@ -1887,11 +1895,15 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); - m10A = m10A + rlx_setA*( - m10A) + //m10A = m10A + rlx_setA*( - m10A) + // + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); + m10A = m10A + rlx_setA*( -0.5*(rhoA*(2*ux*ux-uy*uy-uz*uz))- m10A) + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); - m12A = m12A + rlx_setA*( - m12A) + //m12A = m12A + rlx_setA*( - m12A) + // + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); + m12A = m12A + rlx_setA*( -0.5*(rhoA*(uy*uy-uz*uz))- m12A) + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); @@ -2060,11 +2072,15 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); - m10B = m10B + rlx_setA*( - m10B) + //m10B = m10B + rlx_setA*( - m10B) + // + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); + m10B = m10B + rlx_setA*( -0.5*(rhoB*(2*ux*ux-uy*uy-uz*uz))- m10B) + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); - m12B = m12B + rlx_setA*( - m12B) + //m12B = m12B + rlx_setA*( - m12B) + // + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); + m12B = m12B + rlx_setA*( -0.5*(rhoB*(uy*uy-uz*uz))- m12B) + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); @@ -2191,6 +2207,808 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB } } +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + int n; + int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18; + double vx,vy,vz,v_mag; + double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; + double ux,uy,uz; + double rhoA,rhoB; + double jxA,jyA,jzA; + double jxB,jyB,jzB; + // distribution functions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A,f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B,f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double permA,permB;//effective relative perm + double c0, c1; //Guo's model parameters + double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity + double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity + double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double FxB, FyB, FzB; + double tau,rlx; + double phi;//phase field indicator + double rhoA_gradx,rhoA_grady,rhoA_gradz; + double rhoB_gradx,rhoB_grady,rhoB_gradz; + double GffA_x,GffA_y,GffA_z; + double GfsA_x,GfsA_y,GfsA_z; + double GffB_x,GffB_y,GffB_z; + double GfsB_x,GfsB_y,GfsB_z; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + f1A = distA[nr1]; // reading the f1 data into register fq + f1B = distB[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2A = distA[nr2]; // reading the f2 data into register fq + f2B = distB[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3A = distA[nr3]; + f3B = distB[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4A = distA[nr4]; + f4B = distB[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5A = distA[nr5]; + f5B = distB[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6A = distA[nr6]; + f6B = distB[nr6]; + + // q=7 + nr7 = neighborList[n+6*Np]; + f7A = distA[nr7]; + f7B = distB[nr7]; + + // q = 8 + nr8 = neighborList[n+7*Np]; + f8A = distA[nr8]; + f8B = distB[nr8]; + + // q=9 + nr9 = neighborList[n+8*Np]; + f9A = distA[nr9]; + f9B = distB[nr9]; + + // q = 10 + nr10 = neighborList[n+9*Np]; + f10A = distA[nr10]; + f10B = distB[nr10]; + + // q=11 + nr11 = neighborList[n+10*Np]; + f11A = distA[nr11]; + f11B = distB[nr11]; + + // q=12 + nr12 = neighborList[n+11*Np]; + f12A = distA[nr12]; + f12B = distB[nr12]; + + // q=13 + nr13 = neighborList[n+12*Np]; + f13A = distA[nr13]; + f13B = distB[nr13]; + + // q=14 + nr14 = neighborList[n+13*Np]; + f14A = distA[nr14]; + f14B = distB[nr14]; + + // q=15 + nr15 = neighborList[n+14*Np]; + f15A = distA[nr15]; + f15B = distB[nr15]; + + // q=16 + nr16 = neighborList[n+15*Np]; + f16A = distA[nr16]; + f16B = distB[nr16]; + + // q=17 + //fq = dist[18*Np+n]; + nr17 = neighborList[n+16*Np]; + f17A = distA[nr17]; + f17B = distB[nr17]; + + // q=18 + nr18 = neighborList[n+17*Np]; + f18A = distA[nr18]; + f18B = distB[nr18]; + //---------------------------------------------------------------------// + + // Compute SC fluid-fluid interaction force + GffA_x = -Gsc*rhoB_gradx; + GffA_y = -Gsc*rhoB_grady; + GffA_z = -Gsc*rhoB_gradz; + GffB_x = -Gsc*rhoA_gradx; + GffB_y = -Gsc*rhoA_grady; + GffB_z = -Gsc*rhoA_gradz; + // Compute SC fluid-solid force + GfsA_x = SolidForceA[n+0*Np]; + GfsA_y = SolidForceA[n+1*Np]; + GfsA_z = SolidForceA[n+2*Np]; + GfsB_x = SolidForceB[n+0*Np]; + GfsB_y = SolidForceB[n+1*Np]; + GfsB_z = SolidForceB[n+2*Np]; + + // Compute greyscale related parameters + // ------------------- Fluid Component A -----------------------// + jxA = f1A-f2A+f7A-f8A+f9A-f10A+f11A-f12A+f13A-f14A; + jyA = f3A-f4A+f7A-f8A-f9A+f10A+f15A-f16A+f17A-f18A; + jzA = f5A-f6A+f11A-f12A-f13A+f14A+f15A-f16A-f17A+f18A; + + c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permA); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); + vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); + vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); + FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); + FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); + if (porosity==1.0){ + FxA=rhoA*(Gx + GffA_x + GfsA_x); + FyA=rhoA*(Gy + GffA_y + GfsA_y); + FzA=rhoA*(Gz + GffA_z + GfsA_z); + } + // ------------------- Fluid Component B -----------------------// + // Compute greyscale related parameters + jxB = f1B-f2B+f7B-f8B+f9B-f10B+f11B-f12B+f13B-f14B; + jyB = f3B-f4B+f7B-f8B-f9B+f10B+f15B-f16B+f17B-f18B; + jzB = f5B-f6B+f11B-f12B-f13B+f14B+f15B-f16B-f17B+f18B; + + c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permB); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); + vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); + vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); + FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); + FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); + if (porosity==1.0){ + FxB=rhoB*(Gx + GffB_x + GfsB_x); + FyB=rhoB*(Gy + GffB_y + GfsB_y); + FzB=rhoB*(Gz + GffB_z + GfsB_z); + } + + // Calculate barycentric velocity of the fluid mixture + ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); + uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); + uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); + + //..............carry out relaxation process............................................... + // ------------------- Fluid Component A -----------------------// + // q=0 + distA[n] = f0A*(1.0-rlx) + rlx*0.3333333333333333*rhoA*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + + 0.3333333333333333*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 1 + distA[nr2] = f1A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q=2 + distA[nr1] = f2A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(-3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 3 + distA[nr4] = f3A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 4 + distA[nr3] = f4A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 5 + distA[nr6] = f5A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(3. + (6.*uz)/porosity)); + + // q = 6 + distA[nr5] = f6A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(-3. + (6.*uz)/porosity)); + + // q = 7 + distA[nr8] = f7A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 8 + distA[nr7] = f8A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyA*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 9 + distA[nr10] = f9A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyA*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 10 + distA[nr9] = f10A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 11 + distA[nr12] = f11A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + + // q = 12 + distA[nr11] = f12A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + + FzA*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + + // q = 13 + distA[nr14] = f13A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + + FzA*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + + // q= 14 + distA[nr13] = f14A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + + // q = 15 + distA[nr16] = f15A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + + // q = 16 + distA[nr15] = f16A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + + FzA*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 17 + distA[nr18] = f17A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + + FzA*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 18 + distA[nr17] = f18A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + + + // ------------------- Fluid Component B -----------------------// + // q=0 + distB[n] = f0B*(1.0-rlx) + rlx*0.3333333333333333*rhoB*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + + 0.3333333333333333*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 1 + distB[nr2] = f1B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q=2 + distB[nr1] = f2B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(-3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 3 + distB[nr4] = f3B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 4 + distB[nr3] = f4B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 5 + distB[nr6] = f5B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(3. + (6.*uz)/porosity)); + + // q = 6 + distB[nr5] = f6B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(-3. + (6.*uz)/porosity)); + + // q = 7 + distB[nr8] = f7B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 8 + distB[nr7] = f8B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyB*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 9 + distB[nr10] = f9B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyB*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 10 + distB[nr9] = f10B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 11 + distB[nr12] = f11B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + + // q = 12 + distB[nr11] = f12B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + + FzB*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + + // q = 13 + distB[nr14] = f13B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + + FzB*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + + // q= 14 + distB[nr13] = f14B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + + // q = 15 + distB[nr16] = f15B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + + // q = 16 + distB[nr15] = f16B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + + FzB*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 17 + distB[nr18] = f17B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + + FzB*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 18 + distB[nr17] = f18B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + int n; + double vx,vy,vz,v_mag; + double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; + double ux,uy,uz; + double rhoA,rhoB; + double jxA,jyA,jzA; + double jxB,jyB,jzB; + // distribution functions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A,f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B,f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double permA,permB;//effective relative perm + double c0, c1; //Guo's model parameters + double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity + double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity + double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double FxB, FyB, FzB; + double tau,rlx; + double phi;//phase field indicator + double rhoA_gradx,rhoA_grady,rhoA_gradz; + double rhoB_gradx,rhoB_grady,rhoB_gradz; + double GffA_x,GffA_y,GffA_z; + double GfsA_x,GfsA_y,GfsA_z; + double GffB_x,GffB_y,GffB_z; + double GfsB_x,GfsB_y,GfsB_z; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT<<>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK<<>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK: %s \n",cudaGetErrorString(err)); } } diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index a20f4bc9..7e437b7b 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -727,33 +727,34 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_DeviceBarrier(); //debug - DoubleArray PhaseField(Nx,Ny,Nz); - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - FILE *AFILE; - sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - FILE *BFILE; - sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); +// DoubleArray PhaseField(Nx,Ny,Nz); +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + // *************EVEN TIMESTEP*************// timestep++; // Compute the density field @@ -778,29 +779,29 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); ScaLBL_DeviceBarrier(); - //debug - //DoubleArray PhaseField(Nx,Ny,Nz); - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - //FILE *AFILE; - sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - //FILE *BFILE; - sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); +// //debug +// //DoubleArray PhaseField(Nx,Ny,Nz); +// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// //FILE *AFILE; +// sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// //FILE *BFILE; +// sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); From 16e187e1dc0d637a28d1efce7f2c1716a3bc1017 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 4 May 2020 14:50:23 -0400 Subject: [PATCH 132/270] add pseudo-reflection --- common/Domain.cpp | 220 ++++++++++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 93 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 33d6117a..32e13501 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -256,6 +256,7 @@ void Domain::Decomp( const std::string& Filename ) int64_t i,j,k,n; int64_t xStart,yStart,zStart; int checkerSize; + bool USE_CHECKER = false; //int inlet_layers_x, inlet_layers_y, inlet_layers_z; //int outlet_layers_x, outlet_layers_y, outlet_layers_z; xStart=yStart=zStart=0; @@ -295,6 +296,7 @@ void Domain::Decomp( const std::string& Filename ) } if (database->keyExists( "checkerSize" )){ checkerSize = database->getScalar( "checkerSize" ); + USE_CHECKER = true; } else { checkerSize = SIZE[0]; @@ -367,7 +369,7 @@ void Domain::Decomp( const std::string& Filename ) } } printf("Read segmented data from %s \n",Filename.c_str()); - + // relabel the data std::vector LabelCount(ReadValues.size(),0); for (int k = 0; k 0){ - // use checkerboard pattern - printf("Checkerboard pattern at x inlet for %i layers \n",inlet_layers_x); - for (int k = 0; k 0){ + // use checkerboard pattern + printf("Checkerboard pattern at x inlet for %i layers \n",inlet_layers_x); + for (int k = 0; k 0){ + printf("Checkerboard pattern at y inlet for %i layers \n",inlet_layers_y); + // use checkerboard pattern + for (int k = 0; k 0){ + printf("Checkerboard pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); + // use checkerboard pattern + for (int k = zStart; k < zStart+inlet_layers_z; k++){ + for (int j = 0; j 0){ + // use checkerboard pattern + printf("Checkerboard pattern at x outlet for %i layers \n",outlet_layers_x); + for (int k = 0; k 0){ + printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y); + // use checkerboard pattern + for (int k = 0; k 0){ + printf("Checkerboard pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); + // use checkerboard pattern + for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ + for (int j = 0; j 0){ - printf("Checkerboard pattern at y inlet for %i layers \n",inlet_layers_y); - // use checkerboard pattern - for (int k = 0; k 0){ - printf("Checkerboard pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); - // use checkerboard pattern + printf("Mixed reflection pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); for (int k = zStart; k < zStart+inlet_layers_z; k++){ for (int j = 0; j 0){ + SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; } } } } } - - if (outlet_layers_x > 0){ - // use checkerboard pattern - printf("Checkerboard pattern at x outlet for %i layers \n",outlet_layers_x); - for (int k = 0; k 0){ - printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y); - // use checkerboard pattern - for (int k = 0; k 0){ - printf("Checkerboard pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); - // use checkerboard pattern + printf("Mixed reflection pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ for (int j = 0; j 0){ + SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; } } } From d424771849c46bb15ff0831c29e7ef77afb21ff7 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 4 May 2020 15:12:50 -0400 Subject: [PATCH 133/270] fix scope in Domain inlet/outlet --- common/Domain.cpp | 53 ++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 32e13501..e355310f 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -390,14 +390,11 @@ void Domain::Decomp( const std::string& Filename ) } } } - if (RANK==0){ - for (size_t idx=0; idx 0){ // use checkerboard pattern @@ -516,31 +513,31 @@ void Domain::Decomp( const std::string& Filename ) } } } - } - else { - if (inlet_layers_z > 0){ - printf("Mixed reflection pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); - for (int k = zStart; k < zStart+inlet_layers_z; k++){ - for (int j = 0; j 0){ - SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; + else { + if (inlet_layers_z > 0){ + printf("Mixed reflection pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase); + for (int k = zStart; k < zStart+inlet_layers_z; k++){ + for (int j = 0; j 0){ + SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; + } } } } } - } - if (outlet_layers_z > 0){ - printf("Mixed reflection pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); - for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ - for (int j = 0; j 0){ - SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; + if (outlet_layers_z > 0){ + printf("Mixed reflection pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase); + for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){ + for (int j = 0; j 0){ + SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; + } } } } From 7b731e327be8989fe1983e21d813627272ea05eb Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 4 May 2020 15:26:41 -0400 Subject: [PATCH 134/270] update pseudo-reflection --- common/Domain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index e355310f..3dec0128 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -520,7 +520,7 @@ void Domain::Decomp( const std::string& Filename ) for (int j = 0; j 0){ SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; } @@ -534,7 +534,7 @@ void Domain::Decomp( const std::string& Filename ) for (int j = 0; j 0){ SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id; } From e83e7940218aa20e56e3f41fabe56781f0873fc2 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 4 May 2020 23:36:27 -0400 Subject: [PATCH 135/270] GreyscaleSC model;BGK works but MRT doesnt;save the work --- common/ScaLBL.h | 15 +- gpu/GreyscaleSC.cu | 930 ++++++++++++++++-------------------- models/GreyscaleSCModel.cpp | 240 +++++++--- models/GreyscaleSCModel.h | 4 +- 4 files changed, 579 insertions(+), 610 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 526ab5a4..bf2bb9dc 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -118,32 +118,33 @@ extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, doubl // GREYSCALE SHAN-CHEN MODEL (Two-component) -extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(double *distA, double *distB, double *Den, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double *distA, double *distB, double *Den, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *distB, double *Den, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index dea5476f..d0768f8c 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -3,11 +3,12 @@ #define NBLOCKS 1024 #define NTHREADS 256 -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList,int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ + int ijk; int n, nread; double vx,vy,vz,v_mag; double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; @@ -16,7 +17,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double double jxA,jyA,jzA; double jxB,jyB,jzB; double rhoA,rhoB; - double rhoA_next,rhoB_next; + double nA,nB; // non-conserved moments double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; @@ -32,8 +33,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) double FxB, FyB, FzB; double rlx_setA,rlx_setB; - double rhoA_gradx,rhoA_grady,rhoA_gradz; - double rhoB_gradx,rhoB_grady,rhoB_gradz; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; double GffA_x,GffA_y,GffA_z; double GfsA_x,GfsA_y,GfsA_z; double GffB_x,GffB_y,GffB_z; @@ -60,18 +61,19 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double if ( n even part of dist) fq = distA[nread]; // reading the f2 data into register fq - rhoA_next += fq; + rhoA += fq; m1A -= 11.0*(fq); m2A -= 4.0*(fq); jxA -= fq; @@ -109,7 +111,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=3 nread = neighborList[n+2*Np]; // neighbor 4 fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A -= 11.0*fq; m2A -= 4.0*fq; jyA = fq; @@ -122,7 +124,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 4 nread = neighborList[n+3*Np]; // neighbor 3 fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A -= 11.0*fq; m2A -= 4.0*fq; jyA -= fq; @@ -135,7 +137,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=5 nread = neighborList[n+4*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A -= 11.0*fq; m2A -= 4.0*fq; jzA = fq; @@ -149,7 +151,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 6 nread = neighborList[n+5*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A -= 11.0*fq; m2A -= 4.0*fq; jzA -= fq; @@ -162,7 +164,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=7 nread = neighborList[n+6*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA += fq; @@ -180,7 +182,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 8 nread = neighborList[n+7*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA -= fq; @@ -198,7 +200,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=9 nread = neighborList[n+8*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA += fq; @@ -216,7 +218,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 10 nread = neighborList[n+9*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA -= fq; @@ -234,7 +236,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=11 nread = neighborList[n+10*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA += fq; @@ -252,7 +254,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=12 nread = neighborList[n+11*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA -= fq; @@ -270,7 +272,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=13 nread = neighborList[n+12*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA += fq; @@ -288,7 +290,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=14 nread = neighborList[n+13*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jxA -= fq; @@ -306,7 +308,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=15 nread = neighborList[n+14*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jyA += fq; @@ -322,7 +324,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=16 nread = neighborList[n+15*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jyA -= fq; @@ -338,7 +340,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=17 nread = neighborList[n+16*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jyA += fq; @@ -354,7 +356,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=18 nread = neighborList[n+17*Np]; fq = distA[nread]; - rhoA_next += fq; + rhoA += fq; m1A += 8.0*fq; m2A += fq; jyA -= fq; @@ -375,14 +377,14 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double //........................................................................ // q=0 fq = distB[n]; - rhoB_next = fq; + rhoB = fq; m1B = -30.0*fq; m2B = 12.0*fq; // q=1 nread = neighborList[n]; // neighbor 2 fq = distB[nread]; // reading the f1 data into register fq - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*fq; m2B -= 4.0*fq; jxB = fq; @@ -393,7 +395,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=2 nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) fq = distB[nread]; // reading the f2 data into register fq - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*(fq); m2B -= 4.0*(fq); jxB -= fq; @@ -404,7 +406,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=3 nread = neighborList[n+2*Np]; // neighbor 4 fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*fq; m2B -= 4.0*fq; jyB = fq; @@ -417,7 +419,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 4 nread = neighborList[n+3*Np]; // neighbor 3 fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*fq; m2B -= 4.0*fq; jyB -= fq; @@ -430,7 +432,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=5 nread = neighborList[n+4*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*fq; m2B -= 4.0*fq; jzB = fq; @@ -444,7 +446,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 6 nread = neighborList[n+5*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B -= 11.0*fq; m2B -= 4.0*fq; jzB -= fq; @@ -457,7 +459,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=7 nread = neighborList[n+6*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB += fq; @@ -475,7 +477,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 8 nread = neighborList[n+7*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB -= fq; @@ -493,7 +495,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=9 nread = neighborList[n+8*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB += fq; @@ -511,7 +513,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q = 10 nread = neighborList[n+9*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB -= fq; @@ -529,7 +531,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=11 nread = neighborList[n+10*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB += fq; @@ -547,7 +549,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=12 nread = neighborList[n+11*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB -= fq; @@ -565,7 +567,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=13 nread = neighborList[n+12*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB += fq; @@ -583,7 +585,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=14 nread = neighborList[n+13*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jxB -= fq; @@ -601,7 +603,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=15 nread = neighborList[n+14*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jyB += fq; @@ -617,7 +619,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=16 nread = neighborList[n+15*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jyB -= fq; @@ -633,7 +635,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=17 nread = neighborList[n+16*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jyB += fq; @@ -649,7 +651,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // q=18 nread = neighborList[n+17*Np]; fq = distB[nread]; - rhoB_next += fq; + rhoB += fq; m1B += 8.0*fq; m2B += fq; jyB -= fq; @@ -665,12 +667,12 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // Compute SC fluid-fluid interaction force - GffA_x = -Gsc*rhoB_gradx; - GffA_y = -Gsc*rhoB_grady; - GffA_z = -Gsc*rhoB_gradz; - GffB_x = -Gsc*rhoA_gradx; - GffB_y = -Gsc*rhoA_grady; - GffB_z = -Gsc*rhoA_gradz; + GffA_x = -Gsc*nB_gradx; + GffA_y = -Gsc*nB_grady; + GffA_z = -Gsc*nB_gradz; + GffB_x = -Gsc*nA_gradx; + GffB_y = -Gsc*nA_grady; + GffB_z = -Gsc*nA_gradz; // Compute SC fluid-solid force GfsA_x = SolidForceA[n+0*Np]; GfsA_y = SolidForceA[n+1*Np]; @@ -697,13 +699,13 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); - FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); - FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); + FxA = nA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); + FyA = nA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); + FzA = nA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); if (porosity==1.0){ - FxA=rhoA*(Gx + GffA_x + GfsA_x); - FyA=rhoA*(Gy + GffA_y + GfsA_y); - FzA=rhoA*(Gz + GffA_z + GfsA_z); + FxA=nA*(Gx + GffA_x + GfsA_x); + FyA=nA*(Gy + GffA_y + GfsA_y); + FzA=nA*(Gz + GffA_z + GfsA_z); } // ------------------- Fluid Component B -----------------------// // Compute greyscale related parameters @@ -723,62 +725,29 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); - FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); - FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); + FxB = nB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); + FyB = nB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); + FzB = nB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); if (porosity==1.0){ - FxB=rhoB*(Gx + GffB_x + GfsB_x); - FyB=rhoB*(Gy + GffB_y + GfsB_y); - FzB=rhoB*(Gz + GffB_z + GfsB_z); + FxB=nB*(Gx + GffB_x + GfsB_x); + FyB=nB*(Gy + GffB_y + GfsB_y); + FzB=nB*(Gz + GffB_z + GfsB_z); } // Calculate barycentric velocity of the fluid mixture - ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); - uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); - uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); + ux = (nA*ux_A+nB*ux_B)/(nA+nB); + uy = (nA*uy_A+nB*uy_B)/(nA+nB); + uz = (nA*uz_A+nB*uz_B)/(nA+nB); -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - // ------------------- Fluid Component A -----------------------// rlx_setA = 1.0/tauA; rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA_next) - m1A) + m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA) - m1A) + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); - m2A = m2A + rlx_setA*((3*rhoA_next - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) + m2A = m2A + rlx_setA*((3*rhoA - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); jxA = jxA + FxA; m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) @@ -816,165 +785,109 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // ------------------- Fluid Component A -----------------------// //.................inverse transformation...................................................... // q=0 - fq = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; - //f0 = mrt_V1*rhoA_next-mrt_V2*m1A+mrt_V3*m2A; + fq = mrt_V1*rhoA-mrt_V2*m1A+mrt_V3*m2A; distA[n] = fq; // q = 1 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); - //f1 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); nread = neighborList[n+Np]; distA[nread] = fq; // q=2 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); - //f2 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); nread = neighborList[n]; distA[nread] = fq; // q = 3 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - //f3 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+3*Np]; distA[nread] = fq; // q = 4 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - //f4 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); nread = neighborList[n+2*Np]; distA[nread] = fq; // q = 5 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - //f5 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+5*Np]; distA[nread] = fq; // q = 6 - fq = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - //f6 = mrt_V1*rhoA_next-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); + fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); nread = neighborList[n+4*Np]; distA[nread] = fq; // q = 7 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); - //f7 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); nread = neighborList[n+7*Np]; distA[nread] = fq; // q = 8 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); - //f8 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); nread = neighborList[n+6*Np]; distA[nread] = fq; // q = 9 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); - //f9 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); nread = neighborList[n+9*Np]; distA[nread] = fq; // q = 10 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); - //f10 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); nread = neighborList[n+8*Np]; distA[nread] = fq; // q = 11 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); - //f11 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); nread = neighborList[n+11*Np]; distA[nread] = fq; // q = 12 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); - //f12 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); nread = neighborList[n+10*Np]; distA[nread]= fq; // q = 13 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); - //f13 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); nread = neighborList[n+13*Np]; distA[nread] = fq; // q= 14 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); - //f14 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); nread = neighborList[n+12*Np]; distA[nread] = fq; // q = 15 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); - //f15 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); nread = neighborList[n+15*Np]; distA[nread] = fq; // q = 16 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); - //f16 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); nread = neighborList[n+14*Np]; distA[nread] = fq; // q = 17 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); - //f17 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); nread = neighborList[n+17*Np]; distA[nread] = fq; // q = 18 - fq = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); - //f18 = mrt_V1*rhoA_next+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); + fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); nread = neighborList[n+16*Np]; distA[nread] = fq; //........................................................................ - - //Den[n] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - - -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*Den) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*Den) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((Den*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*Den*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((Den*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(Den*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((Den*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((Den*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((Den*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - // ------------------- Fluid Component B -----------------------// rlx_setA = 1.0/tauB; rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); //-------------------- MRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... //TODO need to incoporate porosity - m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB_next) - m1B) + m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB) - m1B) + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); - m2B = m2B + rlx_setA*((3*rhoB_next - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) + m2B = m2B + rlx_setA*((3*rhoB - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); jxB = jxB + FxB; m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) @@ -1012,140 +925,116 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double // ------------------- Fluid Component B -----------------------// //.................inverse transformation...................................................... // q=0 - fq = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; - //f0 = mrt_V1*rhoB_next-mrt_V2*m1B+mrt_V3*m2B; + fq = mrt_V1*rhoB-mrt_V2*m1B+mrt_V3*m2B; distB[n] = fq; // q = 1 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); - //f1 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); nread = neighborList[n+Np]; distB[nread] = fq; // q=2 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); - //f2 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); nread = neighborList[n]; distB[nread] = fq; // q = 3 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - //f3 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); nread = neighborList[n+3*Np]; distB[nread] = fq; // q = 4 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - //f4 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); nread = neighborList[n+2*Np]; distB[nread] = fq; // q = 5 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - //f5 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); nread = neighborList[n+5*Np]; distB[nread] = fq; // q = 6 - fq = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - //f6 = mrt_V1*rhoB_next-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); + fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); nread = neighborList[n+4*Np]; distB[nread] = fq; // q = 7 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); - //f7 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); nread = neighborList[n+7*Np]; distB[nread] = fq; // q = 8 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); - //f8 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); nread = neighborList[n+6*Np]; distB[nread] = fq; // q = 9 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); - //f9 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); nread = neighborList[n+9*Np]; distB[nread] = fq; // q = 10 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); - //f10 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); nread = neighborList[n+8*Np]; distB[nread] = fq; // q = 11 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); - //f11 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); nread = neighborList[n+11*Np]; distB[nread] = fq; // q = 12 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); - //f12 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); nread = neighborList[n+10*Np]; distB[nread]= fq; // q = 13 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); - //f13 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); nread = neighborList[n+13*Np]; distB[nread] = fq; // q= 14 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); - //f14 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); nread = neighborList[n+12*Np]; distB[nread] = fq; // q = 15 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); - //f15 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); nread = neighborList[n+15*Np]; distB[nread] = fq; // q = 16 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); - //f16 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); nread = neighborList[n+14*Np]; distB[nread] = fq; // q = 17 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); - //f17 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); nread = neighborList[n+17*Np]; distB[nread] = fq; // q = 18 - fq = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); - //f18 = mrt_V1*rhoB_next+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); + fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); nread = neighborList[n+16*Np]; distB[nread] = fq; //........................................................................ - //Den[n+Np] = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17; - //Update velocity on device Velocity[0*Np+n] = ux; Velocity[1*Np+n] = uy; Velocity[2*Np+n] = uz; //Update pressure on device - Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; - //Update density - //Den[n] = rhoA_next; - //Den[n+Np] = rhoB_next; - + Pressure[n] = (nA+nB+Gsc*nA*nB)/3.0; } } } -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ + int ijk; int n; double vx,vy,vz,v_mag; double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; @@ -1154,7 +1043,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *d double jxA,jyA,jzA; double jxB,jyB,jzB; double rhoA,rhoB; - double rhoA_next,rhoB_next; + double nA,nB; // non-conserved moments double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; @@ -1170,8 +1059,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *d double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) double FxB, FyB, FzB; double rlx_setA,rlx_setB; - double rhoA_gradx,rhoA_grady,rhoA_gradz; - double rhoB_gradx,rhoB_grady,rhoB_gradz; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; double GffA_x,GffA_y,GffA_z; double GfsA_x,GfsA_y,GfsA_z; double GffB_x,GffB_y,GffB_z; @@ -1198,18 +1087,19 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *d if ( n>>(distA,distB,Den,Np); +__global__ void dvc_ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np){ + + int n,nn; + int ijk; + // distributions + double m1,m2,m3,m4,m5,m6,m7,m8,m9; + double m10,m11,m12,m13,m14,m15,m16,m17,m18; + double nx,ny,nz; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(Map,distA,distB,DenA,DenB,Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Init: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double *distA, double *distB, double *Den, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_Density<<>>(NeighborList, distA, distB, Den, start, finish, Np); + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_Density<<>>(NeighborList, Map, distA, distB, DenA, DenB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -3358,9 +3233,9 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, double } } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *distB, double *Den, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_Density<<>>(distA, distB, Den, start, finish, Np); + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_Density<<>>(Map,distA, distB, DenA, DenB, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -3369,12 +3244,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(double *distA, double *d } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT<<>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT<<>>(neighborList,Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); @@ -3383,12 +3258,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, double *di } } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT<<>>(Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); @@ -3397,12 +3272,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(double *distA, double *distB } } -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK<<>>(neighborList,distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK<<>>(neighborList,Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); @@ -3411,12 +3286,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, double *di } } -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB, double *Den, double *DenGradA, double *DenGradB, +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map,double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK<<>>(distA,distB,Den,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK<<>>(Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); cudaError_t err = cudaGetLastError(); @@ -3424,3 +3299,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(double *distA, double *distB printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK: %s \n",cudaGetErrorString(err)); } } + +extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np){ + + dvc_ScaLBL_D3Q19_GreyscaleSC_Gradient<<>>(neighborList, Map, Den, DenGrad, strideY, strideZ,start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Gradient: %s \n",cudaGetErrorString(err)); + } +} diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 7e437b7b..3f50dc34 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -465,6 +465,81 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() delete [] Dst; } +//void ScaLBL_GreyscaleSCModel::Density_Init(){ +// +// size_t NLABELS=0; +// signed char VALUE=0; +// +// vector LabelList{1,2}; +// vector SwList{0.0,1.0}; +// +// if (greyscaleSC_db->keyExists( "GreyNodeLabels" )){ +// LabelList.clear(); +// LabelList = greyscaleSC_db->getVector( "GreyNodeLabels" ); +// } +// if (greyscaleSC_db->keyExists( "GreyNodeSw" )){ +// SwList.clear(); +// SwList = greyscaleSC_db->getVector( "GreyNodeSw" ); +// } +// +// NLABELS=LabelList.size(); +// if (NLABELS != SwList.size()){ +// ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); +// } +// +// double *Den_temp; +// Den_temp=new double [2*Np]; +// double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. +// double nB=0.5; +// +// //double *Phi_temp; +// //Phi_temp=new double [Np]; +// //double phi = 0.0; +// +// for (int k=0; kid[n]; +// if (VALUE>0){ +// for (unsigned int idx=0; idx < NLABELS; idx++){ +// if (VALUE == LabelList[idx]){ +// double Sw = SwList[idx]; +// if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); +// nB=Sw; +// nA=1.0-Sw; +// //phi = nA-nB; +// idx = NLABELS; +// } +// } +// if (VALUE==1){//label=1 reserved for NW phase +// //TODO; maybe need rho_major and rho_minor initialization +// nA=rhoA; +// nB=rhoB_minor; +// //phi = nA-nB; +// } +// else if(VALUE==2){//label=2 reserved for W phase +// //TODO; maybe need rho_major and rho_minor initialization +// nA=rhoA_minor; +// nB=rhoB; +// //phi = nA-nB; +// } +// int idx = Map(i,j,k); +// Den_temp[idx+0*Np] = nA; +// Den_temp[idx+1*Np] = nB; +// //Phi_temp[idx] = phi; +// } +// } +// } +// } +// //copy to device +// ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); +// //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); +// ScaLBL_DeviceBarrier(); +// delete [] Den_temp; +// //delete [] Phi_temp; +//} + void ScaLBL_GreyscaleSCModel::Density_Init(){ size_t NLABELS=0; @@ -487,10 +562,11 @@ void ScaLBL_GreyscaleSCModel::Density_Init(){ ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); } - double *Den_temp; - Den_temp=new double [2*Np]; - double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. - double nB=0.5; + double *DenA_temp,*DenB_temp; + DenA_temp=new double [Nx*Ny*Nz]; + DenB_temp=new double [Nx*Ny*Nz]; + double nA=0.0;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. + double nB=0.0; //double *Phi_temp; //Phi_temp=new double [Np]; @@ -513,30 +589,32 @@ void ScaLBL_GreyscaleSCModel::Density_Init(){ } } if (VALUE==1){//label=1 reserved for NW phase - //TODO; maybe need rho_major and rho_minor initialization nA=rhoA; nB=rhoB_minor; //phi = nA-nB; } else if(VALUE==2){//label=2 reserved for W phase - //TODO; maybe need rho_major and rho_minor initialization nA=rhoA_minor; nB=rhoB; //phi = nA-nB; } - int idx = Map(i,j,k); - Den_temp[idx+0*Np] = nA; - Den_temp[idx+1*Np] = nB; - //Phi_temp[idx] = phi; + DenA_temp[n] = nA; + DenB_temp[n] = nB; + } + else{ //for ID<=0, i.e. all sorts of solid minerals, density is zero + DenA_temp[n] = 0.0; + DenB_temp[n] = 0.0; } } } } //copy to device - ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); + ScaLBL_CopyToDevice(DenA, DenA_temp, Nx*Ny*Nz*sizeof(double)); + ScaLBL_CopyToDevice(DenB, DenB_temp, Nx*Ny*Nz*sizeof(double)); //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); ScaLBL_DeviceBarrier(); - delete [] Den_temp; + delete [] DenA_temp; + delete [] DenB_temp; //delete [] Phi_temp; } @@ -559,6 +637,7 @@ void ScaLBL_GreyscaleSCModel::Create(){ // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); int Npad=(Np/16 + 2)*16; if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); @@ -577,9 +656,11 @@ void ScaLBL_GreyscaleSCModel::Create(){ neighborSize=18*(Np*sizeof(int)); //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); ScaLBL_AllocateDeviceMemory((void **) &fqA, 19*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &fqB, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenA, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &DenB, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); @@ -592,6 +673,36 @@ void ScaLBL_GreyscaleSCModel::Create(){ // Update GPU data structures if (rank==0) printf ("Setting up device neighbor list \n"); fflush(stdout); + // Copy the Map to device + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nx*Ny*Nz ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); } @@ -631,18 +742,20 @@ void ScaLBL_GreyscaleSCModel::Initialize(){ if (rank==0) printf ("Initializing density field \n"); Density_Init();//initialize density field if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_GreyscaleSC_Init(fqA, fqB, Den, Np); + ScaLBL_D3Q19_GreyscaleSC_Init(dvcMap,fqA, fqB, DenA,DenB, Np); //debug DoubleArray PhaseField(Nx,Ny,Nz); - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); FILE *AFILE; sprintf(LocalRankFilename,"A_init.%05i.raw",rank); AFILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,AFILE); fclose(AFILE); - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); FILE *BFILE; sprintf(LocalRankFilename,"B_init.%05i.raw",rank); BFILE = fopen(LocalRankFilename,"wb"); @@ -707,49 +820,32 @@ void ScaLBL_GreyscaleSCModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + // Compute density gradient // fluid component A - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->SendHalo(DenA); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenA); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // fluid component B - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Compute density gradient - ScaLBL_Comm->SendHalo(&Den[0]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); + ScaLBL_Comm_Regular->SendHalo(DenB); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenB); ScaLBL_DeviceBarrier(); - ScaLBL_Comm->SendHalo(&Den[Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); - ScaLBL_DeviceBarrier(); - - //debug -// DoubleArray PhaseField(Nx,Ny,Nz); -// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); - + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -760,48 +856,32 @@ void ScaLBL_GreyscaleSCModel::Run(){ // Compute the density field // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(fqA, fqB, Den, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + // Compute density gradient // fluid component A - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->SendHalo(DenA); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenA); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // fluid component B - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Compute density gradient - ScaLBL_Comm->SendHalo(&Den[0]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[0], DenGradA, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[0],DenGradA); + ScaLBL_Comm_Regular->SendHalo(DenB); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenB); ScaLBL_DeviceBarrier(); - ScaLBL_Comm->SendHalo(&Den[Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &Den[Np], DenGradB, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&Den[Np],DenGradB); - ScaLBL_DeviceBarrier(); - -// //debug -// //DoubleArray PhaseField(Nx,Ny,Nz); -// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// //FILE *AFILE; -// sprintf(LocalRankFilename,"A_beforeCol_time_%i.%05i.raw",timestep,rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// //FILE *BFILE; -// sprintf(LocalRankFilename,"B_beforeCol_time_%i.%05i.raw",timestep,rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); + ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(fqA, fqB, Den, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -1114,14 +1194,16 @@ void ScaLBL_GreyscaleSCModel::WriteDebug(){ // fwrite(PhaseField.data(),8,N,OUTFILE); // fclose(OUTFILE); // - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); FILE *AFILE; sprintf(LocalRankFilename,"A.%05i.raw",rank); AFILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,AFILE); fclose(AFILE); - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); FILE *BFILE; sprintf(LocalRankFilename,"B.%05i.raw",rank); BFILE = fopen(LocalRankFilename,"wb"); diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index 348490c4..bc71dc01 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -54,6 +54,7 @@ public: std::shared_ptr Dm; // this domain is for analysis std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; + std::shared_ptr ScaLBL_Comm_Regular; // input database std::shared_ptr db; @@ -64,12 +65,13 @@ public: signed char *id; int *NeighborList; + int *dvcMap; double *fqA, *fqB; double *Permeability;//grey voxel permeability double *Porosity; double *Velocity; double *Pressure_dvc; - double *Den; + double *DenA, *DenB; double *DenGradA,*DenGradB; double *SolidForceA,*SolidForceB; From 3a21a142c4580c27895cb924bce97e73f704e063 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 6 May 2020 11:12:50 -0400 Subject: [PATCH 136/270] fix bugs: add some missing terms in the IMRT collsion terms --- cpu/Greyscale.cpp | 16 ++++++++-------- gpu/Greyscale.cu | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index b4b017c8..9b68ed61 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -894,9 +894,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int //-------------------- IMRT collison where body force has higher-order terms -------------// // //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; // jx = jx + Fx; // m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) @@ -929,8 +929,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -1401,9 +1401,9 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis //-------------------- IMRT collison where body force has higher-order terms -------------// // //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; // jx = jx + Fx; // m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) @@ -1436,8 +1436,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m1 = m1 + rlx_setA*((-30*Den+19*Den(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 0a9a63e0..f826d1e3 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -912,9 +912,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); // //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; // jx = jx + Fx; // m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) @@ -946,8 +946,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); @@ -1425,9 +1425,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double pressure=0.5/porosity*(pressure-0.5*Den*u_mag*u_mag/porosity); // //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; // jx = jx + Fx; // m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) @@ -1459,8 +1459,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*Den - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); From 214917021aaf9d89c6efa6d2f795683df9421ff3 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 6 May 2020 14:10:57 -0400 Subject: [PATCH 137/270] rescale force after user time interval --- models/ColorModel.cpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 189f0059..49fc635c 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -494,6 +494,7 @@ void ScaLBL_ColorModel::Run(){ int IMAGE_COUNT = 0; std::vector ImageList; bool SET_CAPILLARY_NUMBER = false; + bool RESCALE_FORCE = false; bool MORPH_ADAPT = false; bool USE_MORPH = false; bool USE_SEED = false; @@ -502,6 +503,7 @@ void ScaLBL_ColorModel::Run(){ int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine int MIN_STEADY_TIMESTEPS = 100000; int MAX_STEADY_TIMESTEPS = 200000; + int RESCALE_FORCE_AFTER_TIMESTEP = 0; int RAMP_TIMESTEPS = 0;//50000; // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in) int CURRENT_MORPH_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) int CURRENT_STEADY_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) @@ -563,7 +565,9 @@ void ScaLBL_ColorModel::Run(){ if (color_db->keyExists( "capillary_number" )){ capillary_number = color_db->getScalar( "capillary_number" ); SET_CAPILLARY_NUMBER=true; - //RESCALE_FORCE_MAX = 1; + } + if (color_db->keyExists( "rescale_force_after_timestep" )){ + RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar( "rescale_force_after_timestep" ); } if (color_db->keyExists( "timestep" )){ timestep = color_db->getScalar( "timestep" ); @@ -791,7 +795,20 @@ void ScaLBL_ColorModel::Run(){ isSteady = true; if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; - + if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ + RESCALE_FORCE = false; + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } if ( isSteady ){ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; @@ -952,12 +969,17 @@ void ScaLBL_ColorModel::Run(){ CURRENT_STEADY_TIMESTEPS=0; initial_volume = volA*Dm->Volume; delta_volume = 0.0; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; } else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { MORPH_ADAPT = false; CURRENT_STEADY_TIMESTEPS=0; initial_volume = volA*Dm->Volume; delta_volume = 0.0; + RESCALE_FORCE = true; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; } } morph_timesteps += analysis_interval; From 09a9a05a8780941240fd790b6825cb3f771d73c3 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 6 May 2020 15:12:50 -0400 Subject: [PATCH 138/270] enable force adaptation --- models/ColorModel.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 49fc635c..7b883657 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -568,6 +568,7 @@ void ScaLBL_ColorModel::Run(){ } if (color_db->keyExists( "rescale_force_after_timestep" )){ RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar( "rescale_force_after_timestep" ); + RESCALE_FORCE = true; } if (color_db->keyExists( "timestep" )){ timestep = color_db->getScalar( "timestep" ); From 6d59bf7effc0434498c6cf7d545535baf8c7590a Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 7 May 2020 12:26:06 -0400 Subject: [PATCH 139/270] revert to BGK collision for greyscale SC model --- models/GreyscaleSCModel.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 3f50dc34..27e66bf7 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -840,12 +840,12 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -876,12 +876,12 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); From 88b560a876a6a349a596fe59c871230f4e231245 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 7 May 2020 17:07:35 -0400 Subject: [PATCH 140/270] make the SC fluid-solid force consistent with literature --- models/GreyscaleSCModel.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 27e66bf7..0faeee7d 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -402,12 +402,12 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() double vec_z = double(kk-1); double GWNS_A=SolidPotentialA_host[nn]; double GWNS_B=SolidPotentialB_host[nn]; - phi_x_A += GWNS_A*weight*vec_x; - phi_y_A += GWNS_A*weight*vec_y; - phi_z_A += GWNS_A*weight*vec_z; - phi_x_B += GWNS_B*weight*vec_x; - phi_y_B += GWNS_B*weight*vec_y; - phi_z_B += GWNS_B*weight*vec_z; + phi_x_A += -1.0*GWNS_A*weight*vec_x; + phi_y_A += -1.0*GWNS_A*weight*vec_y; + phi_z_A += -1.0*GWNS_A*weight*vec_z; + phi_x_B += -1.0*GWNS_B*weight*vec_x; + phi_y_B += -1.0*GWNS_B*weight*vec_y; + phi_z_B += -1.0*GWNS_B*weight*vec_z; } } } From 42277520edb4fe3c2bab9dab67f320c0723e033f Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 7 May 2020 19:09:26 -0400 Subject: [PATCH 141/270] update output writing --- models/GreyscaleSCModel.cpp | 61 +++++++++++++++++++--------- models/GreyscaleSCModel.h | 4 +- tests/lbpm_greyscaleSC_simulator.cpp | 1 - 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 0faeee7d..bb73fdbb 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -120,6 +120,8 @@ void ScaLBL_GreyscaleSCModel::SetDomain(){ Velocity_z.resize(Nx,Ny,Nz); PorosityMap.resize(Nx,Ny,Nz); Pressure.resize(Nx,Ny,Nz); + DenA_data.resize(Nx,Ny,Nz); + DenB_data.resize(Nx,Ny,Nz); id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way @@ -744,23 +746,23 @@ void ScaLBL_GreyscaleSCModel::Initialize(){ if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_GreyscaleSC_Init(dvcMap,fqA, fqB, DenA,DenB, Np); - //debug - DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); - FILE *AFILE; - sprintf(LocalRankFilename,"A_init.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); - FILE *BFILE; - sprintf(LocalRankFilename,"B_init.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); +// //debug +// DoubleArray PhaseField(Nx,Ny,Nz); +// //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A_init.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B_init.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); //Velocity also needs initialization (for old incompressible momentum transport) //if (rank==0) printf ("Initializing velocity field \n"); @@ -1017,7 +1019,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ // } if (timestep%visualization_interval==0){ - VelocityField(); + WriteOutput(); } // if (timestep%restart_interval==0){ @@ -1066,7 +1068,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ // ************************************************************************ } -void ScaLBL_GreyscaleSCModel::VelocityField(){ +void ScaLBL_GreyscaleSCModel::WriteOutput(){ /* Minkowski Morphology(Mask); int SIZE=Np*sizeof(double); @@ -1121,6 +1123,8 @@ void ScaLBL_GreyscaleSCModel::VelocityField(){ auto VzVar = std::make_shared(); auto SignDistVar = std::make_shared(); auto PressureVar = std::make_shared(); + auto DenAVar = std::make_shared(); + auto DenBVar = std::make_shared(); IO::initialize("","silo","false"); // Create the MeshDataStruct @@ -1155,28 +1159,47 @@ void ScaLBL_GreyscaleSCModel::VelocityField(){ PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); visData[0].vars.push_back(PressureVar); + DenAVar->name = "DenA"; + DenAVar->type = IO::VariableType::VolumeVariable; + DenAVar->dim = 1; + DenAVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(DenAVar); + DenBVar->name = "DenB"; + DenBVar->type = IO::VariableType::VolumeVariable; + DenBVar->dim = 1; + DenBVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(DenBVar); + Array& SignData = visData[0].vars[0]->data; Array& VelxData = visData[0].vars[1]->data; Array& VelyData = visData[0].vars[2]->data; Array& VelzData = visData[0].vars[3]->data; Array& PressureData = visData[0].vars[4]->data; + Array& DenAData = visData[0].vars[5]->data; + Array& DenBData = visData[0].vars[6]->data; ASSERT(visData[0].vars[0]->name=="SignDist"); ASSERT(visData[0].vars[1]->name=="Velocity_x"); ASSERT(visData[0].vars[2]->name=="Velocity_y"); ASSERT(visData[0].vars[3]->name=="Velocity_z"); ASSERT(visData[0].vars[4]->name=="Pressure"); + ASSERT(visData[0].vars[5]->name=="DenA"); + ASSERT(visData[0].vars[6]->name=="DenB"); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + ScaLBL_CopyToHost(DenA_data.data(), DenA, sizeof(double)*N); + ScaLBL_CopyToHost(DenB_data.data(), DenB, sizeof(double)*N); fillData.copy(SignDist,SignData); fillData.copy(Velocity_x,VelxData); fillData.copy(Velocity_y,VelyData); fillData.copy(Velocity_z,VelzData); fillData.copy(Pressure,PressureData); + fillData.copy(DenA_data,DenAData); + fillData.copy(DenB_data,DenBData); IO::writeData( timestep, visData, Dm->Comm ); diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index bc71dc01..5de29c3d 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -30,7 +30,7 @@ public: void Initialize(); void Run(); void WriteDebug(); - void VelocityField(); + void WriteOutput(); bool Restart,pBC; int timestep,timestepMax; @@ -82,6 +82,8 @@ public: DoubleArray Velocity_z; DoubleArray PorosityMap; DoubleArray Pressure; + DoubleArray DenA_data; + DoubleArray DenB_data; private: MPI_Comm comm; diff --git a/tests/lbpm_greyscaleSC_simulator.cpp b/tests/lbpm_greyscaleSC_simulator.cpp index b95fff04..340be938 100644 --- a/tests/lbpm_greyscaleSC_simulator.cpp +++ b/tests/lbpm_greyscaleSC_simulator.cpp @@ -49,7 +49,6 @@ int main(int argc, char **argv) GreyscaleSC.Create(); // creating the model will create data structure to match the pore structure and allocate variables GreyscaleSC.Initialize(); // initializing the model will set initial conditions for variables GreyscaleSC.Run(); - //GreyscaleSC.VelocityField(); GreyscaleSC.WriteDebug(); } // **************************************************** From e060780e99be046da01842ec928fb6662fee3507 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 15 May 2020 20:33:48 -0400 Subject: [PATCH 142/270] Damping the force regulator --- models/ColorModel.cpp | 265 +++++++++++++++++++++++------------------- 1 file changed, 148 insertions(+), 117 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 7b883657..c47c35b6 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -798,9 +798,13 @@ void ScaLBL_ColorModel::Run(){ isSteady = true; if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ RESCALE_FORCE = false; - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); if (force_mag > 1e-3){ Fx *= 1e-3/force_mag; // impose ceiling for stability Fy *= 1e-3/force_mag; @@ -811,124 +815,151 @@ void ScaLBL_ColorModel::Run(){ color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ - MORPH_ADAPT = true; - CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - //****** ENDPOINT ADAPTATION ********/ - double krA_TMP= fabs(muA*flow_rate_A / force_mag); - double krB_TMP= fabs(muB*flow_rate_B / force_mag); - log_krA = log(krA_TMP); - if (krA_TMP < 0.0){ - // cannot do endpoint adaptation if kr is negative - log_krA = log_krA_prev; - } - else if (krA_TMP < krB_TMP && morph_delta > 0.0){ - /** morphological target based on relative permeability for A **/ - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + fabs(capillary_number - Ca) / capillary_number + if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ + // reject steady points if they don't match the Ca well enough + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + // simulate the point again with new force + CURRENT_STEADY_TIMESTEPS=0; + } + else { + MORPH_ADAPT = true; + CURRENT_MORPH_TIMESTEPS=0; + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); + log_krA = log(krA_TMP); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + } + } + log_krA_prev = log_krA; + volA_prev = volA; + //******************************** **/ + /** compute averages & write data **/ + Averages->Full(); + Averages->Write(timestep); + analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.finish(); + + if (rank==0){ + printf("** WRITE STEADY POINT *** "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + double h = Dm->voxel_length; + // pressures + double pA = Averages->gnb.p; + double pB = Averages->gwb.p; + double pAc = Averages->gnc.p; + double pBc = Averages->gwc.p; + double pAB = (pA-pB)/(h*5.796*alpha); + double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + // connected contribution + double Vol_nc = Averages->gnc.V/Dm->Volume; + double Vol_wc = Averages->gwc.V/Dm->Volume; + double Vol_nd = Averages->gnd.V/Dm->Volume; + double Vol_wd = Averages->gwd.V/Dm->Volume; + double Mass_n = Averages->gnc.M + Averages->gnd.M; + double Mass_w = Averages->gwc.M + Averages->gwd.M; + double vAc_x = Averages->gnc.Px/Mass_n; + double vAc_y = Averages->gnc.Py/Mass_n; + double vAc_z = Averages->gnc.Pz/Mass_n; + double vBc_x = Averages->gwc.Px/Mass_w; + double vBc_y = Averages->gwc.Py/Mass_w; + double vBc_z = Averages->gwc.Pz/Mass_w; + // disconnected contribution + double vAd_x = Averages->gnd.Px/Mass_n; + double vAd_y = Averages->gnd.Py/Mass_n; + double vAd_z = Averages->gnd.Pz/Mass_n; + double vBd_x = Averages->gwd.Px/Mass_w; + double vBd_y = Averages->gwd.Py/Mass_w; + double vBd_z = Averages->gwd.Pz/Mass_w; + + double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); + double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); + double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); + double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); + + double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); + double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); + + double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); + double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + + double kAeff = h*h*muA*(flow_rate_A)/(force_mag); + double kBeff = h*h*muB*(flow_rate_B)/(force_mag); + + double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; + double Mobility = muA/muB; + + bool WriteHeader=false; + FILE * kr_log_file = fopen("relperm.csv","r"); + if (kr_log_file != NULL) + fclose(kr_log_file); + else + WriteHeader=true; + kr_log_file = fopen("relperm.csv","a"); + if (WriteHeader) + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fclose(kr_log_file); + + printf(" Measured capillary number %f \n ",Ca); + } + if (SET_CAPILLARY_NUMBER ){ + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + + CURRENT_STEADY_TIMESTEPS = 0; + } + else{ if (rank==0){ - printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + printf("** Continue to simulate steady *** \n "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); } } - log_krA_prev = log_krA; - volA_prev = volA; - //******************************** **/ - /** compute averages & write data **/ - Averages->Full(); - Averages->Write(timestep); - analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - analysis.finish(); - - if (rank==0){ - printf("** WRITE STEADY POINT *** "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - double h = Dm->voxel_length; - // pressures - double pA = Averages->gnb.p; - double pB = Averages->gwb.p; - double pAc = Averages->gnc.p; - double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*5.796*alpha); - double pAB_connected = (pAc-pBc)/(h*5.796*alpha); - // connected contribution - double Vol_nc = Averages->gnc.V/Dm->Volume; - double Vol_wc = Averages->gwc.V/Dm->Volume; - double Vol_nd = Averages->gnd.V/Dm->Volume; - double Vol_wd = Averages->gwd.V/Dm->Volume; - double Mass_n = Averages->gnc.M + Averages->gnd.M; - double Mass_w = Averages->gwc.M + Averages->gwd.M; - double vAc_x = Averages->gnc.Px/Mass_n; - double vAc_y = Averages->gnc.Py/Mass_n; - double vAc_z = Averages->gnc.Pz/Mass_n; - double vBc_x = Averages->gwc.Px/Mass_w; - double vBc_y = Averages->gwc.Py/Mass_w; - double vBc_z = Averages->gwc.Pz/Mass_w; - // disconnected contribution - double vAd_x = Averages->gnd.Px/Mass_n; - double vAd_y = Averages->gnd.Py/Mass_n; - double vAd_z = Averages->gnd.Pz/Mass_n; - double vBd_x = Averages->gwd.Px/Mass_w; - double vBd_y = Averages->gwd.Py/Mass_w; - double vBd_z = Averages->gwd.Pz/Mass_w; - - double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); - double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); - double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); - double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); - - double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); - double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); - - double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); - double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); - - double kAeff = h*h*muA*(flow_rate_A)/(force_mag); - double kBeff = h*h*muB*(flow_rate_B)/(force_mag); - - double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; - double Mobility = muA/muB; - - bool WriteHeader=false; - FILE * kr_log_file = fopen("relperm.csv","r"); - if (kr_log_file != NULL) - fclose(kr_log_file); - else - WriteHeader=true; - kr_log_file = fopen("relperm.csv","a"); - if (WriteHeader) - fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); - - fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); - fclose(kr_log_file); - - printf(" Measured capillary number %f \n ",Ca); - } - if (SET_CAPILLARY_NUMBER ){ - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - - CURRENT_STEADY_TIMESTEPS = 0; + morph_timesteps=0; + Ca_previous = Ca; } - else{ - if (rank==0){ - printf("** Continue to simulate steady *** \n "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - } - } - morph_timesteps=0; - Ca_previous = Ca; } if (MORPH_ADAPT ){ From c4672a828b029498ef2f20a9b6b2cb827e8d160b Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 15 May 2020 20:40:30 -0400 Subject: [PATCH 143/270] debugging --- models/ColorModel.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index c47c35b6..55ad299b 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -815,7 +815,6 @@ void ScaLBL_ColorModel::Run(){ color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ - fabs(capillary_number - Ca) / capillary_number if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ // reject steady points if they don't match the Ca well enough double RESCALE_FORCE_FACTOR = capillary_number / Ca; From 6c1ed15cda7ed5c225993772854a130764b6d957 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 15 May 2020 20:47:40 -0400 Subject: [PATCH 144/270] debugging --- models/ColorModel.cpp | 250 +++++++++++++++++++++--------------------- 1 file changed, 124 insertions(+), 126 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 55ad299b..30146ea3 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -788,9 +788,9 @@ void ScaLBL_ColorModel::Run(){ double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); - + if ( morph_timesteps > morph_interval ){ - + bool isSteady = false; if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_STEADY_TIMESTEPS > MIN_STEADY_TIMESTEPS)) isSteady = true; @@ -815,141 +815,139 @@ void ScaLBL_ColorModel::Run(){ color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ - if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ - // reject steady points if they don't match the Ca well enough - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - // simulate the point again with new force - CURRENT_STEADY_TIMESTEPS=0; + if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ + // reject steady points if they don't match the Ca well enough + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + // simulate the point again with new force + CURRENT_STEADY_TIMESTEPS=0; + } + else { + MORPH_ADAPT = true; + CURRENT_MORPH_TIMESTEPS=0; + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); + log_krA = log(krA_TMP); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); } - else { - MORPH_ADAPT = true; - CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - //****** ENDPOINT ADAPTATION ********/ - double krA_TMP= fabs(muA*flow_rate_A / force_mag); - double krB_TMP= fabs(muB*flow_rate_B / force_mag); - log_krA = log(krA_TMP); - if (krA_TMP < 0.0){ - // cannot do endpoint adaptation if kr is negative - log_krA = log_krA_prev; - } - else if (krA_TMP < krB_TMP && morph_delta > 0.0){ - /** morphological target based on relative permeability for A **/ - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); - if (rank==0){ - printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); - } - } - log_krA_prev = log_krA; - volA_prev = volA; - //******************************** **/ - /** compute averages & write data **/ - Averages->Full(); - Averages->Write(timestep); - analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - analysis.finish(); + } + log_krA_prev = log_krA; + volA_prev = volA; + //******************************** **/ + /** compute averages & write data **/ + Averages->Full(); + Averages->Write(timestep); + analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.finish(); - if (rank==0){ - printf("** WRITE STEADY POINT *** "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - double h = Dm->voxel_length; - // pressures - double pA = Averages->gnb.p; - double pB = Averages->gwb.p; - double pAc = Averages->gnc.p; - double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*5.796*alpha); - double pAB_connected = (pAc-pBc)/(h*5.796*alpha); - // connected contribution - double Vol_nc = Averages->gnc.V/Dm->Volume; - double Vol_wc = Averages->gwc.V/Dm->Volume; - double Vol_nd = Averages->gnd.V/Dm->Volume; - double Vol_wd = Averages->gwd.V/Dm->Volume; - double Mass_n = Averages->gnc.M + Averages->gnd.M; - double Mass_w = Averages->gwc.M + Averages->gwd.M; - double vAc_x = Averages->gnc.Px/Mass_n; - double vAc_y = Averages->gnc.Py/Mass_n; - double vAc_z = Averages->gnc.Pz/Mass_n; - double vBc_x = Averages->gwc.Px/Mass_w; - double vBc_y = Averages->gwc.Py/Mass_w; - double vBc_z = Averages->gwc.Pz/Mass_w; - // disconnected contribution - double vAd_x = Averages->gnd.Px/Mass_n; - double vAd_y = Averages->gnd.Py/Mass_n; - double vAd_z = Averages->gnd.Pz/Mass_n; - double vBd_x = Averages->gwd.Px/Mass_w; - double vBd_y = Averages->gwd.Py/Mass_w; - double vBd_z = Averages->gwd.Pz/Mass_w; + if (rank==0){ + printf("** WRITE STEADY POINT *** "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + double h = Dm->voxel_length; + // pressures + double pA = Averages->gnb.p; + double pB = Averages->gwb.p; + double pAc = Averages->gnc.p; + double pBc = Averages->gwc.p; + double pAB = (pA-pB)/(h*5.796*alpha); + double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + // connected contribution + double Vol_nc = Averages->gnc.V/Dm->Volume; + double Vol_wc = Averages->gwc.V/Dm->Volume; + double Vol_nd = Averages->gnd.V/Dm->Volume; + double Vol_wd = Averages->gwd.V/Dm->Volume; + double Mass_n = Averages->gnc.M + Averages->gnd.M; + double Mass_w = Averages->gwc.M + Averages->gwd.M; + double vAc_x = Averages->gnc.Px/Mass_n; + double vAc_y = Averages->gnc.Py/Mass_n; + double vAc_z = Averages->gnc.Pz/Mass_n; + double vBc_x = Averages->gwc.Px/Mass_w; + double vBc_y = Averages->gwc.Py/Mass_w; + double vBc_z = Averages->gwc.Pz/Mass_w; + // disconnected contribution + double vAd_x = Averages->gnd.Px/Mass_n; + double vAd_y = Averages->gnd.Py/Mass_n; + double vAd_z = Averages->gnd.Pz/Mass_n; + double vBd_x = Averages->gwd.Px/Mass_w; + double vBd_y = Averages->gwd.Py/Mass_w; + double vBd_z = Averages->gwd.Pz/Mass_w; - double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); - double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); - double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); - double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); + double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); + double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); + double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); + double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); - double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); - double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); + double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); + double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); - double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); - double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); + double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); - double kAeff = h*h*muA*(flow_rate_A)/(force_mag); - double kBeff = h*h*muB*(flow_rate_B)/(force_mag); + double kAeff = h*h*muA*(flow_rate_A)/(force_mag); + double kBeff = h*h*muB*(flow_rate_B)/(force_mag); - double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; - double Mobility = muA/muB; + double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; + double Mobility = muA/muB; - bool WriteHeader=false; - FILE * kr_log_file = fopen("relperm.csv","r"); - if (kr_log_file != NULL) - fclose(kr_log_file); - else - WriteHeader=true; - kr_log_file = fopen("relperm.csv","a"); - if (WriteHeader) - fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + bool WriteHeader=false; + FILE * kr_log_file = fopen("relperm.csv","r"); + if (kr_log_file != NULL) + fclose(kr_log_file); + else + WriteHeader=true; + kr_log_file = fopen("relperm.csv","a"); + if (WriteHeader) + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); - fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); - fclose(kr_log_file); + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fclose(kr_log_file); - printf(" Measured capillary number %f \n ",Ca); - } - if (SET_CAPILLARY_NUMBER ){ - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - - CURRENT_STEADY_TIMESTEPS = 0; + printf(" Measured capillary number %f \n ",Ca); + } + if (SET_CAPILLARY_NUMBER ){ + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + } else{ if (rank==0){ printf("** Continue to simulate steady *** \n "); From 02643365f9387fdab75b7c472955f75d0d2a2571 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 15 May 2020 20:54:57 -0400 Subject: [PATCH 145/270] debugging --- models/ColorModel.cpp | 143 +++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 30146ea3..8b082d7c 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -489,7 +489,7 @@ void ScaLBL_ColorModel::Initialize(){ void ScaLBL_ColorModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - + int IMAGE_INDEX = 0; int IMAGE_COUNT = 0; std::vector ImageList; @@ -518,7 +518,7 @@ void ScaLBL_ColorModel::Run(){ double initial_volume = 0.0; double delta_volume = 0.0; double delta_volume_target = 0.0; - + /* history for morphological algoirthm */ double KRA_MORPH_FACTOR=0.5; double volA_prev = 0.0; @@ -535,7 +535,7 @@ void ScaLBL_ColorModel::Run(){ if (color_db->keyExists( "krA_morph_factor" )){ KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); } - + /* defaults for simulation protocols */ auto protocol = color_db->getWithDefault( "protocol", "none" ); if (protocol == "image sequence"){ @@ -657,7 +657,7 @@ void ScaLBL_ColorModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - //std::shared_ptr analysis_db; + //std::shared_ptr analysis_db; bool Regular = false; auto current_db = db->cloneDatabase(); runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); @@ -947,73 +947,74 @@ void ScaLBL_ColorModel::Run(){ Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); color_db->putVector("F",{Fx,Fy,Fz}); } + + else{ + if (rank==0){ + printf("** Continue to simulate steady *** \n "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + } + } + morph_timesteps=0; + Ca_previous = Ca; } - else{ - if (rank==0){ - printf("** Continue to simulate steady *** \n "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + } + + if (MORPH_ADAPT ){ + CURRENT_MORPH_TIMESTEPS += analysis_interval; + if (USE_DIRECT){ + // Use image sequence + IMAGE_INDEX++; + MORPH_ADAPT = false; + if (IMAGE_INDEX < IMAGE_COUNT){ + std::string next_image = ImageList[IMAGE_INDEX]; + if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); + color_db->putScalar("image_index",IMAGE_INDEX); + ImageInit(next_image); + } + else{ + if (rank==0) printf("Finished simulating image sequence \n"); + timestep = timestepMax; } } - morph_timesteps=0; - Ca_previous = Ca; - } - } - - if (MORPH_ADAPT ){ - CURRENT_MORPH_TIMESTEPS += analysis_interval; - if (USE_DIRECT){ - // Use image sequence - IMAGE_INDEX++; - MORPH_ADAPT = false; - if (IMAGE_INDEX < IMAGE_COUNT){ - std::string next_image = ImageList[IMAGE_INDEX]; - if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); - color_db->putScalar("image_index",IMAGE_INDEX); - ImageInit(next_image); + else if (USE_SEED){ + delta_volume = volA*Dm->Volume - initial_volume; + CURRENT_MORPH_TIMESTEPS += analysis_interval; + double massChange = SeedPhaseField(seed_water); + if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); } - else{ - if (rank==0) printf("Finished simulating image sequence \n"); - timestep = timestepMax; + else if (USE_MORPHOPEN_OIL){ + delta_volume = volA*Dm->Volume - initial_volume; + if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); + MorphOpenConnected(delta_volume_target); + } + else { + if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); + //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A + delta_volume += MorphInit(beta,delta_volume_target-delta_volume); + } + + if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; + } + else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + RESCALE_FORCE = true; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; } } - else if (USE_SEED){ - delta_volume = volA*Dm->Volume - initial_volume; - CURRENT_MORPH_TIMESTEPS += analysis_interval; - double massChange = SeedPhaseField(seed_water); - if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); - } - else if (USE_MORPHOPEN_OIL){ - delta_volume = volA*Dm->Volume - initial_volume; - if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); - MorphOpenConnected(delta_volume_target); - } - else { - if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); - //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A - delta_volume += MorphInit(beta,delta_volume_target-delta_volume); - } - - if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; - } - else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - RESCALE_FORCE = true; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; - } + morph_timesteps += analysis_interval; } - morph_timesteps += analysis_interval; + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } analysis.finish(); PROFILE_STOP("Loop"); @@ -1039,7 +1040,7 @@ void ScaLBL_ColorModel::Run(){ } double ScaLBL_ColorModel::ImageInit(std::string Filename){ - + if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); Mask->Decomp(Filename); for (int i=0; iid[i]; // save what was read @@ -1067,16 +1068,16 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ Count=sumReduce( Dm->Comm, Count); PoreCount=sumReduce( Dm->Comm, PoreCount); - + if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - + ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); double saturation = Count/PoreCount; @@ -1085,14 +1086,14 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ } double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ - + int nx = Nx; int ny = Ny; int nz = Nz; int n; int N = nx*ny*nz; double volume_change=0.0; - + if (target_volume_change < 0.0){ Array id_solid(nx,ny,nz); Array phase_label(nx,ny,nz); @@ -1158,7 +1159,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ signed char notwater=1; double SW=-(target_volume_change)/count_connected; MorphOpen(distance, id_connected, Dm, SW, water, notwater); - + for (int k=0; k Date: Mon, 18 May 2020 11:12:27 -0400 Subject: [PATCH 146/270] disable steady point rejection --- models/ColorModel.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 8b082d7c..742f14aa 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -815,7 +815,7 @@ void ScaLBL_ColorModel::Run(){ color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ - if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ + /*if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ // reject steady points if they don't match the Ca well enough double RESCALE_FORCE_FACTOR = capillary_number / Ca; if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; @@ -836,6 +836,7 @@ void ScaLBL_ColorModel::Run(){ CURRENT_STEADY_TIMESTEPS=0; } else { + */ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change @@ -956,7 +957,9 @@ void ScaLBL_ColorModel::Run(){ } morph_timesteps=0; Ca_previous = Ca; - } + /*} + THIS IS WHERE YOU DISABLE STEADY POINT REJECTION + */ } if (MORPH_ADAPT ){ From 14826cba1f16bd7c275aa971422b16f80c6c8730 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 18 May 2020 11:12:42 -0400 Subject: [PATCH 147/270] make final filter optional --- tests/lbpm_uCT_pp.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/lbpm_uCT_pp.cpp b/tests/lbpm_uCT_pp.cpp index 0285b864..057ec7d9 100644 --- a/tests/lbpm_uCT_pp.cpp +++ b/tests/lbpm_uCT_pp.cpp @@ -52,6 +52,7 @@ int main(int argc, char **argv) printf("Input data file: %s\n",filename.c_str()); } + bool FILTER_CONNECTED_COMPONENTS = false; auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); auto uct_db = db->getDatabase( "uCT" ); @@ -82,10 +83,14 @@ int main(int argc, char **argv) auto center = uct_db->getVector( "center" ); auto CylRad = uct_db->getScalar( "cylinder_radius" ); auto maxLevels = uct_db->getScalar( "max_levels" ); + std::vector offset( 3, 0 ); if ( uct_db->keyExists( "offset" ) ) offset = uct_db->getVector( "offset" ); + if ( uct_db->keyExists( "filter_connected_components" ) ) + FILTER_CONNECTED_COMPONENTS = uct_db->getScalar( "filter_connected_components" ); + // Check that the number of processors >= the number of ranks if ( rank==0 ) { printf("Number of MPI ranks required: %i \n", nprocx*nprocy*nprocz); @@ -326,11 +331,13 @@ int main(int argc, char **argv) // Perform a final filter PROFILE_START("Filtering final domains"); + if (FILTER_CONNECTED_COMPONENTS){ if (rank==0) printf("Filtering final domains\n"); Array filter_Mean, filter_Dist1, filter_Dist2; filter_final( ID[0], Dist[0], *fillFloat[0], *Dm[0], filter_Mean, filter_Dist1, filter_Dist2 ); PROFILE_STOP("Filtering final domains"); + } //removeDisconnected( ID[0], *Dm[0] ); // Write the distance function to a netcdf file From 16cc9cb5aab533125b25faee5cabd22b86498448 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 20 May 2020 22:57:42 -0400 Subject: [PATCH 148/270] GreyscaleSC: save the work; pressureBC does not work --- common/ScaLBL.cpp | 39 +++ common/ScaLBL.h | 23 +- gpu/GreyscaleSC.cu | 509 ++++++++++++++++++++++++++++++++++++ models/GreyscaleSCModel.cpp | 80 ++++-- models/GreyscaleSCModel.h | 4 +- 5 files changed, 636 insertions(+), 19 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 8d868218..c8feffbe 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2075,3 +2075,42 @@ void ScaLBL_Communicator::PrintD3Q19(){ delete [] TempBuffer; } +void ScaLBL_Communicator::GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB) +{ + if (kproc == 0) { + // Set the density field on the z inlet + ScaLBL_GreyscaleSC_BC_z(dvcSendList_z, Map, DenA, DenB, vA, vB, sendCount_z, N); + } +} + +void ScaLBL_Communicator::GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB) +{ + if (kproc == nprocz-1){ + // Set the density field on the Z outlet + ScaLBL_GreyscaleSC_BC_Z(dvcSendList_Z, Map, DenA, DenB, vA, vB, sendCount_Z, N); + } +} + +void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time) +{ + if (kproc == 0) { + if (time%2==0){ + ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N); + } + else{ + ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(neighborList, dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N); + } + } +} + +void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time) +{ + if (kproc == nprocz-1){ + if (time%2==0){ + ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N); + } + else{ + ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(neighborList, dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N); + } + } +} diff --git a/common/ScaLBL.h b/common/ScaLBL.h index bf2bb9dc..0f31224b 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -145,6 +145,19 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, dou int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np); + +extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); + +extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); + +extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); + +extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); + +extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); + +extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); @@ -211,6 +224,11 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); +extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); + +extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); + + class ScaLBL_Communicator{ public: //...................................................................................... @@ -275,7 +293,10 @@ public: void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time); void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time); double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time); - + void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB); + void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB); + void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time); + void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time); // void TestSendD3Q19(double *f_even, double *f_odd); // void TestRecvD3Q19(double *f_even, double *f_odd); diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index d0768f8c..e22b7a50 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -3215,6 +3215,460 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Ma } } +__global__ void dvc_ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np) +{ + int idx,n,nm; + // Fill the outlet with component b + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + nm = Map[n]; + DenA[nm] = vA; + DenB[nm] = vB; + } +} + +__global__ void dvc_ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np) +{ + int idx,n,nm; + // Fill the outlet with component b + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + nm = Map[n]; + DenA[nm] = vA; + DenB[nm] = vB; + } +} + +__global__ void dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int Np) +{ + int idx, n; + // distributions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A; + double f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B; + double f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double uxA,uyA,uzA,CyzA,CxzA; + double uxB,uyB,uzB,CyzB,CxzB; + uxA = uyA = 0.0; + uxB = uyB = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + if (idx < count){ + + n = list[idx]; + //........................................................................ + // Read distributions + //........................................................................ + f0A = distA[n]; + f1A = distA[2*Np+n]; + f2A = distA[1*Np+n]; + f3A = distA[4*Np+n]; + f4A = distA[3*Np+n]; + f6A = distA[5*Np+n]; + f7A = distA[8*Np+n]; + f8A = distA[7*Np+n]; + f9A = distA[10*Np+n]; + f10A = distA[9*Np+n]; + f12A = distA[11*Np+n]; + f13A = distA[14*Np+n]; + f16A = distA[15*Np+n]; + f17A = distA[18*Np+n]; + + f0B = distB[n]; + f1B = distB[2*Np+n]; + f2B = distB[1*Np+n]; + f3B = distB[4*Np+n]; + f4B = distB[3*Np+n]; + f6B = distB[5*Np+n]; + f7B = distB[8*Np+n]; + f8B = distB[7*Np+n]; + f9B = distB[10*Np+n]; + f10B = distB[9*Np+n]; + f12B = distB[11*Np+n]; + f13B = distB[14*Np+n]; + f16B = distB[15*Np+n]; + f17B = distB[18*Np+n]; + //................................................... + // Determine the inlet flow velocity + //ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14); + //uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18); + uzA = dinA - (f0A+f1A+f2A+f3A+f4A+f7A+f8A+f9A+f10A + 2*(f6A+f12A+f13A+f16A+f17A)); + uzB = dinB - (f0B+f1B+f2B+f3B+f4B+f7B+f8B+f9B+f10B + 2*(f6B+f12B+f13B+f16B+f17B)); + + CxzA = 0.5*(f1A+f7A+f9A-f2A-f10A-f8A) - 0.3333333333333333*uxA; + CyzA = 0.5*(f3A+f7A+f10A-f4A-f9A-f8A) - 0.3333333333333333*uyA; + CxzB = 0.5*(f1B+f7B+f9B-f2B-f10B-f8B) - 0.3333333333333333*uxB; + CyzB = 0.5*(f3B+f7B+f10B-f4B-f9B-f8B) - 0.3333333333333333*uyB; + + f5A = f6A + 0.33333333333333338*uzA; + f11A = f12A + 0.16666666666666678*(uzA+uxA)-CxzA; + f14A = f13A + 0.16666666666666678*(uzA-uxA)+CxzA; + f15A = f16A + 0.16666666666666678*(uyA+uzA)-CyzA; + f18A = f17A + 0.16666666666666678*(uzA-uyA)+CyzA; + + f5B = f6B + 0.33333333333333338*uzB; + f11B = f12B + 0.16666666666666678*(uzB+uxB)-CxzB; + f14B = f13B + 0.16666666666666678*(uzB-uxB)+CxzB; + f15B = f16B + 0.16666666666666678*(uyB+uzB)-CyzB; + f18B = f17B + 0.16666666666666678*(uzB-uyB)+CyzB; + //........Store in "opposite" memory location.......... + distA[6*Np+n] = f5A; + distA[12*Np+n] = f11A; + distA[13*Np+n] = f14A; + distA[16*Np+n] = f15A; + distA[17*Np+n] = f18A; + + distB[6*Np+n] = f5B; + distB[12*Np+n] = f11B; + distB[13*Np+n] = f14B; + distB[16*Np+n] = f15B; + distB[17*Np+n] = f18B; + } +} + +__global__ void dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int Np) +{ + int idx,n; + // distributions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A; + double f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B; + double f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double uxA,uyA,uzA,CyzA,CxzA; + double uxB,uyB,uzB,CyzB,CxzB; + uxA = uyA = 0.0; + uxB = uyB = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + // Loop over the boundary - threadblocks delineated by start...finish + if ( idx < count ){ + + n = list[idx]; + //........................................................................ + // Read distributions + //........................................................................ + f0A = distA[n]; + f1A = distA[2*Np+n]; + f2A = distA[1*Np+n]; + f3A = distA[4*Np+n]; + f4A = distA[3*Np+n]; + f5A = distA[6*Np+n]; + f7A = distA[8*Np+n]; + f8A = distA[7*Np+n]; + f9A = distA[10*Np+n]; + f10A = distA[9*Np+n]; + f11A = distA[12*Np+n]; + f14A = distA[13*Np+n]; + f15A = distA[16*Np+n]; + f18A = distA[17*Np+n]; + + f0B = distB[n]; + f1B = distB[2*Np+n]; + f2B = distB[1*Np+n]; + f3B = distB[4*Np+n]; + f4B = distB[3*Np+n]; + f5B = distB[6*Np+n]; + f7B = distB[8*Np+n]; + f8B = distB[7*Np+n]; + f9B = distB[10*Np+n]; + f10B = distB[9*Np+n]; + f11B = distB[12*Np+n]; + f14B = distB[13*Np+n]; + f15B = distB[16*Np+n]; + f18B = distB[17*Np+n]; + + // Determine the outlet flow velocity + //ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + //uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + uzA = -doutA + (f0A+f1A+f2A+f3A+f4A+f7A+f8A+f9A+f10A + 2*(f5A+f11A+f14A+f15A+f18A)); + uzB = -doutB + (f0B+f1B+f2B+f3B+f4B+f7B+f8B+f9B+f10B + 2*(f5B+f11B+f14B+f15B+f18B)); + + CxzA = 0.5*(f1A+f7A+f9A-f2A-f10A-f8A) - 0.3333333333333333*uxA; + CyzA = 0.5*(f3A+f7A+f10A-f4A-f9A-f8A) - 0.3333333333333333*uyA; + CxzB = 0.5*(f1B+f7B+f9B-f2B-f10B-f8B) - 0.3333333333333333*uxB; + CyzB = 0.5*(f3B+f7B+f10B-f4B-f9B-f8B) - 0.3333333333333333*uyB; + + f6A = f5A - 0.33333333333333338*uzA; + f12A = f11A - 0.16666666666666678*(uzA+uxA)+CxzA; + f13A = f14A - 0.16666666666666678*(uzA-uxA)-CxzA; + f16A = f15A - 0.16666666666666678*(uyA+uzA)+CyzA; + f17A = f18A - 0.16666666666666678*(uzA-uyA)-CyzA; + + f6B = f5B - 0.33333333333333338*uzB; + f12B = f11B - 0.16666666666666678*(uzB+uxB)+CxzB; + f13B = f14B - 0.16666666666666678*(uzB-uxB)-CxzB; + f16B = f15B - 0.16666666666666678*(uyB+uzB)+CyzB; + f17B = f18B - 0.16666666666666678*(uzB-uyB)-CyzB; + + distA[5*Np+n] = f6A; + distA[11*Np+n] = f12A; + distA[14*Np+n] = f13A; + distA[15*Np+n] = f16A; + distA[18*Np+n] = f17A; + + distB[5*Np+n] = f6B; + distB[11*Np+n] = f12B; + distB[14*Np+n] = f13B; + distB[15*Np+n] = f16B; + distB[18*Np+n] = f17B; + //................................................... + } +} + +__global__ void dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int Np) +{ + int idx, n; + int nread; + int nr5,nr11,nr14,nr15,nr18; + // distributions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A; + double f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B; + double f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double uxA,uyA,uzA,CyzA,CxzA; + double uxB,uyB,uzB,CyzB,CxzB; + uxA = uyA = 0.0; + uxB = uyB = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + if (idx < count){ + + n = list[idx]; + //........................................................................ + // Read distributions + //........................................................................ + f0A = distA[n]; + f0B = distB[n]; + + nread = d_neighborList[n]; + f1A = distA[nread]; + f1B = distB[nread]; + + nread = d_neighborList[n+2*Np]; + f3A = distA[nread]; + f3B = distB[nread]; + + nread = d_neighborList[n+6*Np]; + f7A = distA[nread]; + f7B = distB[nread]; + + nread = d_neighborList[n+8*Np]; + f9A = distA[nread]; + f9B = distB[nread]; + + nread = d_neighborList[n+12*Np]; + f13A = distA[nread]; + f13B = distB[nread]; + + nread = d_neighborList[n+16*Np]; + f17A = distA[nread]; + f17B = distB[nread]; + + nread = d_neighborList[n+Np]; + f2A = distA[nread]; + f2B = distB[nread]; + + nread = d_neighborList[n+3*Np]; + f4A = distA[nread]; + f4B = distB[nread]; + + nread = d_neighborList[n+5*Np]; + f6A = distA[nread]; + f6B = distB[nread]; + + nread = d_neighborList[n+7*Np]; + f8A = distA[nread]; + f8B = distB[nread]; + + nread = d_neighborList[n+9*Np]; + f10A = distA[nread]; + f10B = distB[nread]; + + nread = d_neighborList[n+11*Np]; + f12A = distA[nread]; + f12B = distB[nread]; + + nread = d_neighborList[n+15*Np]; + f16A = distA[nread]; + f16B = distB[nread]; + + // Unknown distributions + nr5 = d_neighborList[n+4*Np]; + nr11 = d_neighborList[n+10*Np]; + nr15 = d_neighborList[n+14*Np]; + nr14 = d_neighborList[n+13*Np]; + nr18 = d_neighborList[n+17*Np]; + + //................................................... + //........Determine the inlet flow velocity......... + //ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14); + //uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18); + uzA = dinA - (f0A+f1A+f2A+f3A+f4A+f7A+f8A+f9A+f10A + 2*(f6A+f12A+f13A+f16A+f17A)); + uzB = dinB - (f0B+f1B+f2B+f3B+f4B+f7B+f8B+f9B+f10B + 2*(f6B+f12B+f13B+f16B+f17B)); + + CxzA = 0.5*(f1A+f7A+f9A-f2A-f10A-f8A) - 0.3333333333333333*uxA; + CyzA = 0.5*(f3A+f7A+f10A-f4A-f9A-f8A) - 0.3333333333333333*uyA; + CxzB = 0.5*(f1B+f7B+f9B-f2B-f10B-f8B) - 0.3333333333333333*uxB; + CyzB = 0.5*(f3B+f7B+f10B-f4B-f9B-f8B) - 0.3333333333333333*uyB; + + f5A = f6A + 0.33333333333333338*uzA; + f11A = f12A + 0.16666666666666678*(uzA+uxA)-CxzA; + f14A = f13A + 0.16666666666666678*(uzA-uxA)+CxzA; + f15A = f16A + 0.16666666666666678*(uyA+uzA)-CyzA; + f18A = f17A + 0.16666666666666678*(uzA-uyA)+CyzA; + + f5B = f6B + 0.33333333333333338*uzB; + f11B = f12B + 0.16666666666666678*(uzB+uxB)-CxzB; + f14B = f13B + 0.16666666666666678*(uzB-uxB)+CxzB; + f15B = f16B + 0.16666666666666678*(uyB+uzB)-CyzB; + f18B = f17B + 0.16666666666666678*(uzB-uyB)+CyzB; + + //........Store in "opposite" memory location.......... + distA[nr5] = f5A; + distA[nr11] = f11A; + distA[nr14] = f14A; + distA[nr15] = f15A; + distA[nr18] = f18A; + + distB[nr5] = f5B; + distB[nr11] = f11B; + distB[nr14] = f14B; + distB[nr15] = f15B; + distB[nr18] = f18B; + } +} + +__global__ void dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int Np) +{ + int idx,n,nread; + int nr6,nr12,nr13,nr16,nr17; + // distributions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A; + double f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B; + double f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double uxA,uyA,uzA,CyzA,CxzA; + double uxB,uyB,uzB,CyzB,CxzB; + uxA = uyA = 0.0; + uxB = uyB = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + // Loop over the boundary - threadblocks delineated by start...finish + if ( idx < count ){ + + n = list[idx]; + //........................................................................ + // Read distributions + //........................................................................ + f0A = distA[n]; + f0B = distB[n]; + + nread = d_neighborList[n]; + f1A = distA[nread]; + f1B = distB[nread]; + + nread = d_neighborList[n+2*Np]; + f3A = distA[nread]; + f3B = distB[nread]; + + nread = d_neighborList[n+4*Np]; + f5A = distA[nread]; + f5B = distB[nread]; + + nread = d_neighborList[n+6*Np]; + f7A = distA[nread]; + f7B = distB[nread]; + + nread = d_neighborList[n+8*Np]; + f9A = distA[nread]; + f9B = distB[nread]; + + nread = d_neighborList[n+10*Np]; + f11A = distA[nread]; + f11B = distB[nread]; + + nread = d_neighborList[n+14*Np]; + f15A = distA[nread]; + f15B = distB[nread]; + + nread = d_neighborList[n+Np]; + f2A = distA[nread]; + f2B = distB[nread]; + + nread = d_neighborList[n+3*Np]; + f4A = distA[nread]; + f4B = distB[nread]; + + nread = d_neighborList[n+7*Np]; + f8A = distA[nread]; + f8B = distB[nread]; + + nread = d_neighborList[n+9*Np]; + f10A = distA[nread]; + f10B = distB[nread]; + + nread = d_neighborList[n+13*Np]; + f14A = distA[nread]; + f14B = distB[nread]; + + nread = d_neighborList[n+17*Np]; + f18A = distA[nread]; + f18B = distB[nread]; + + // unknown distributions + nr6 = d_neighborList[n+5*Np]; + nr12 = d_neighborList[n+11*Np]; + nr16 = d_neighborList[n+15*Np]; + nr17 = d_neighborList[n+16*Np]; + nr13 = d_neighborList[n+12*Np]; + + + //........Determine the outlet flow velocity......... + //ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + //uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + uzA = -doutA + (f0A+f1A+f2A+f3A+f4A+f7A+f8A+f9A+f10A + 2*(f5A+f11A+f14A+f15A+f18A)); + uzB = -doutB + (f0B+f1B+f2B+f3B+f4B+f7B+f8B+f9B+f10B + 2*(f5B+f11B+f14B+f15B+f18B)); + + CxzA = 0.5*(f1A+f7A+f9A-f2A-f10A-f8A) - 0.3333333333333333*uxA; + CyzA = 0.5*(f3A+f7A+f10A-f4A-f9A-f8A) - 0.3333333333333333*uyA; + CxzB = 0.5*(f1B+f7B+f9B-f2B-f10B-f8B) - 0.3333333333333333*uxB; + CyzB = 0.5*(f3B+f7B+f10B-f4B-f9B-f8B) - 0.3333333333333333*uyB; + + f6A = f5A - 0.33333333333333338*uzA; + f12A = f11A - 0.16666666666666678*(uzA+uxA)+CxzA; + f13A = f14A - 0.16666666666666678*(uzA-uxA)-CxzA; + f16A = f15A - 0.16666666666666678*(uyA+uzA)+CyzA; + f17A = f18A - 0.16666666666666678*(uzA-uyA)-CyzA; + + f6B = f5B - 0.33333333333333338*uzB; + f12B = f11B - 0.16666666666666678*(uzB+uxB)+CxzB; + f13B = f14B - 0.16666666666666678*(uzB-uxB)-CxzB; + f16B = f15B - 0.16666666666666678*(uyB+uzB)+CyzB; + f17B = f18B - 0.16666666666666678*(uzB-uyB)-CyzB; + + + //........Store in "opposite" memory location.......... + distA[nr6] = f6A; + distA[nr12] = f12A; + distA[nr13] = f13A; + distA[nr16] = f16A; + distA[nr17] = f17A; + + distB[nr6] = f6B; + distB[nr12] = f12B; + distB[nr13] = f13B; + distB[nr16] = f16B; + distB[nr17] = f17B; + //................................................... + } +} + extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA,double *distB, double *DenA, double *DenB, int Np){ dvc_ScaLBL_D3Q19_GreyscaleSC_Init<<>>(Map,distA,distB,DenA,DenB,Np); cudaError_t err = cudaGetLastError(); @@ -3308,3 +3762,58 @@ extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, d printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Gradient: %s \n",cudaGetErrorString(err)); } } + + +extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_BC_z<<>>(list, Map, DenA, DenB, vA, vB, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_BC_z: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_BC_Z<<>>(list, Map, DenA, DenB, vA, vB, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_BC_Z: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z<<>>(list, distA, distB, dinA, dinB, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z<<>>(list, distA, distB, doutA, doutB, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z<<>>(neighborList, list, distA, distB, dinA, dinB, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z<<>>(neighborList, list, distA, distB, doutA, doutB, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index bb73fdbb..8e01794f 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -15,7 +15,7 @@ void DeleteArray( const TYPE *p ) ScaLBL_GreyscaleSCModel::ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), -rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), +rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); @@ -46,9 +46,11 @@ void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ tolerance = 0.01; Fx = Fy = Fz = 0.0; Restart=false; - din=dout=1.0; + dinA=rhoA;//inlet density for fluid A + dinB=rhoB_minor;//inlet density for fluid B + doutA=rhoA_minor;//outlet denisty for fluid A + doutB=rhoB;//outlet density for fluid B flux=0.0; - dp = 10.0; //unit of 'dp': voxel // ---------------------- Greyscale Model parameters -----------------------// if (greyscaleSC_db->keyExists( "timestepMax" )){ @@ -66,12 +68,13 @@ void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ rhoB = greyscaleSC_db->getWithDefault( "rhoB", rhoB ); rhoA_minor = greyscaleSC_db->getWithDefault( "rhoA_minor", rhoA_minor ); rhoB_minor = greyscaleSC_db->getWithDefault( "rhoB_minor", rhoB_minor ); + dinA = greyscaleSC_db->getWithDefault( "dinA", dinA ); + dinB = greyscaleSC_db->getWithDefault( "dinB", dinB ); + doutA = greyscaleSC_db->getWithDefault( "doutA", doutA ); + doutB = greyscaleSC_db->getWithDefault( "doutB", doutB ); if (greyscaleSC_db->keyExists( "Gsc" )){ Gsc = greyscaleSC_db->getScalar( "Gsc" ); } - if (greyscaleSC_db->keyExists( "dp" )){ - dp = greyscaleSC_db->getScalar( "dp" ); - } if (greyscaleSC_db->keyExists( "F" )){ Fx = greyscaleSC_db->getVector( "F" )[0]; Fy = greyscaleSC_db->getVector( "F" )[1]; @@ -80,12 +83,6 @@ void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ if (greyscaleSC_db->keyExists( "Restart" )){ Restart = greyscaleSC_db->getScalar( "Restart" ); } - if (greyscaleSC_db->keyExists( "din" )){ - din = greyscaleSC_db->getScalar( "din" ); - } - if (greyscaleSC_db->keyExists( "dout" )){ - dout = greyscaleSC_db->getScalar( "dout" ); - } if (greyscaleSC_db->keyExists( "flux" )){ flux = greyscaleSC_db->getScalar( "flux" ); } @@ -182,6 +179,20 @@ void ScaLBL_GreyscaleSCModel::ReadInput(){ CalcDist(SignDist,id_solid,*Mask); if (rank == 0) cout << "Domain set." << endl; + + // Display boundary condition + switch (BoundaryCondition){ + case 0: + if (rank==0) printf("BoundaryCondition=%i: Periodic boundary condition\n",BoundaryCondition); + break; + case 3: + if (rank==0) printf("BoundaryCondition=%i: Constant pressure boundary condition\n",BoundaryCondition); + break; + default: + if (rank==0) printf("BoundaryCondition=%i: is currently not supported! Periodic boundary condition is used.\n",BoundaryCondition); + BoundaryCondition=0; + break; + } } void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() @@ -610,14 +621,33 @@ void ScaLBL_GreyscaleSCModel::Density_Init(){ } } } + //copy to device ScaLBL_CopyToDevice(DenA, DenA_temp, Nx*Ny*Nz*sizeof(double)); ScaLBL_CopyToDevice(DenB, DenB_temp, Nx*Ny*Nz*sizeof(double)); - //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); ScaLBL_DeviceBarrier(); delete [] DenA_temp; delete [] DenB_temp; - //delete [] Phi_temp; + + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,2); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-3); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-3); + } + } + } void ScaLBL_GreyscaleSCModel::Create(){ @@ -827,6 +857,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_DeviceBarrier(); ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryCondition > 0){ + ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); + ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + } + // Compute density gradient // fluid component A ScaLBL_Comm_Regular->SendHalo(DenA); @@ -845,7 +880,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } // Collsion ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, @@ -863,6 +902,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_DeviceBarrier(); ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryCondition > 0){ + ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); + ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + } + // Compute density gradient // fluid component A ScaLBL_Comm_Regular->SendHalo(DenA); @@ -881,7 +925,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } // Collsion ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index 5de29c3d..314b535d 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -43,8 +43,8 @@ public: double rhoA_minor,rhoB_minor;//dissolved density double tolerance; double Fx,Fy,Fz,flux; - double din,dout; - double dp;//solid particle diameter, unit in voxel + double dinA,doutA; + double dinB,doutB; double GreyPorosity; int Nx,Ny,Nz,N,Np; From 3dede0d93a110fbe4873ad50dc854dfdcf72fd9a Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 21 May 2020 22:00:47 -0400 Subject: [PATCH 149/270] GreyscaleSC: fix pressure BC inter-domain communication --- models/GreyscaleSCModel.cpp | 68 +++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 8e01794f..c1cd9460 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -855,12 +855,17 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryCondition > 0){ - ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); - ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - } + //if (BoundaryCondition > 0){ + // ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); + // ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + //} // Compute density gradient // fluid component A @@ -868,11 +873,27 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenA); ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + } + } ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // fluid component B ScaLBL_Comm_Regular->SendHalo(DenB); ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenB); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + } + } ScaLBL_DeviceBarrier(); ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); @@ -880,11 +901,6 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } // Collsion ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, @@ -900,12 +916,17 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryCondition > 0){ - ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); - ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - } + //if (BoundaryCondition > 0){ + // ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); + // ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + //} // Compute density gradient // fluid component A @@ -913,23 +934,34 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenA); ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + } + } ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // fluid component B ScaLBL_Comm_Regular->SendHalo(DenB); ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenB); ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + } + } ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); // Collsion ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } // Collsion ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, From 55981b06ce07e970d8dca3d20754a2fa8d62537b Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 22 May 2020 13:11:50 -0400 Subject: [PATCH 150/270] fix force adaptation --- models/ColorModel.cpp | 28 ++++++++++++++++------------ sample_scripts/configure_summit | 3 +++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 7b883657..3d9d651d 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -797,18 +797,22 @@ void ScaLBL_ColorModel::Run(){ if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ - RESCALE_FORCE = false; - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); + RESCALE_FORCE = false; + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ MORPH_ADAPT = true; diff --git a/sample_scripts/configure_summit b/sample_scripts/configure_summit index 99d17cd0..bcd8a221 100755 --- a/sample_scripts/configure_summit +++ b/sample_scripts/configure_summit @@ -7,6 +7,7 @@ module load cuda export HDF5_DIR=/ccs/proj/csc380/mcclurej/install/hdf5/1.8.12/ export SILO_DIR=/ccs/proj/csc380/mcclurej/install/silo/4.10.2/ +export NETCDF_DIR=/ccs/proj/geo136/install/netcdf/4.6.1 # configure rm -rf CMake* @@ -28,6 +29,8 @@ cmake \ -D USE_SILO=1 \ -D SILO_LIB="$SILO_DIR/lib/libsiloh5.a" \ -D SILO_DIRECTORY="$SILO_DIR" \ + -D USE_NETCDF=1 \ + -D NETCDF_DIRECTORY="$NETCDF_DIR" \ -D USE_DOXYGEN:BOOL=false \ -D USE_TIMER=0 \ ~/LBPM-WIA From 48fe85f4efb7a8beb8ac5af27624d6231cb536df Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 22 May 2020 13:12:54 -0400 Subject: [PATCH 151/270] fix force adaptation --- models/ColorModel.cpp | 410 ++++++++++++++++++++---------------------- 1 file changed, 191 insertions(+), 219 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 742f14aa..3d9d651d 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -489,7 +489,7 @@ void ScaLBL_ColorModel::Initialize(){ void ScaLBL_ColorModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - + int IMAGE_INDEX = 0; int IMAGE_COUNT = 0; std::vector ImageList; @@ -518,7 +518,7 @@ void ScaLBL_ColorModel::Run(){ double initial_volume = 0.0; double delta_volume = 0.0; double delta_volume_target = 0.0; - + /* history for morphological algoirthm */ double KRA_MORPH_FACTOR=0.5; double volA_prev = 0.0; @@ -535,7 +535,7 @@ void ScaLBL_ColorModel::Run(){ if (color_db->keyExists( "krA_morph_factor" )){ KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); } - + /* defaults for simulation protocols */ auto protocol = color_db->getWithDefault( "protocol", "none" ); if (protocol == "image sequence"){ @@ -657,7 +657,7 @@ void ScaLBL_ColorModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - //std::shared_ptr analysis_db; + //std::shared_ptr analysis_db; bool Regular = false; auto current_db = db->cloneDatabase(); runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); @@ -788,42 +788,131 @@ void ScaLBL_ColorModel::Run(){ double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); - + if ( morph_timesteps > morph_interval ){ - + bool isSteady = false; if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_STEADY_TIMESTEPS > MIN_STEADY_TIMESTEPS)) isSteady = true; if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ - RESCALE_FORCE = false; - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); + RESCALE_FORCE = false; + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ - /*if (SET_CAPILLARY_NUMBER && fabs(capillary_number - Ca) / capillary_number > 2.0){ - // reject steady points if they don't match the Ca well enough - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + MORPH_ADAPT = true; + CURRENT_MORPH_TIMESTEPS=0; + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); + log_krA = log(krA_TMP); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + } + } + log_krA_prev = log_krA; + volA_prev = volA; + //******************************** **/ + /** compute averages & write data **/ + Averages->Full(); + Averages->Write(timestep); + analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.finish(); + + if (rank==0){ + printf("** WRITE STEADY POINT *** "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + double h = Dm->voxel_length; + // pressures + double pA = Averages->gnb.p; + double pB = Averages->gwb.p; + double pAc = Averages->gnc.p; + double pBc = Averages->gwc.p; + double pAB = (pA-pB)/(h*5.796*alpha); + double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + // connected contribution + double Vol_nc = Averages->gnc.V/Dm->Volume; + double Vol_wc = Averages->gwc.V/Dm->Volume; + double Vol_nd = Averages->gnd.V/Dm->Volume; + double Vol_wd = Averages->gwd.V/Dm->Volume; + double Mass_n = Averages->gnc.M + Averages->gnd.M; + double Mass_w = Averages->gwc.M + Averages->gwd.M; + double vAc_x = Averages->gnc.Px/Mass_n; + double vAc_y = Averages->gnc.Py/Mass_n; + double vAc_z = Averages->gnc.Pz/Mass_n; + double vBc_x = Averages->gwc.Px/Mass_w; + double vBc_y = Averages->gwc.Py/Mass_w; + double vBc_z = Averages->gwc.Pz/Mass_w; + // disconnected contribution + double vAd_x = Averages->gnd.Px/Mass_n; + double vAd_y = Averages->gnd.Py/Mass_n; + double vAd_z = Averages->gnd.Pz/Mass_n; + double vBd_x = Averages->gwd.Px/Mass_w; + double vBd_y = Averages->gwd.Py/Mass_w; + double vBd_z = Averages->gwd.Pz/Mass_w; + + double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); + double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); + double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); + double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); + + double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); + double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); + + double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); + double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + + double kAeff = h*h*muA*(flow_rate_A)/(force_mag); + double kBeff = h*h*muB*(flow_rate_B)/(force_mag); + + double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; + double Mobility = muA/muB; + + bool WriteHeader=false; + FILE * kr_log_file = fopen("relperm.csv","r"); + if (kr_log_file != NULL) + fclose(kr_log_file); + else + WriteHeader=true; + kr_log_file = fopen("relperm.csv","a"); + if (WriteHeader) + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fclose(kr_log_file); + + printf(" Measured capillary number %f \n ",Ca); + } + if (SET_CAPILLARY_NUMBER ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; if (force_mag > 1e-3){ Fx *= 1e-3/force_mag; // impose ceiling for stability Fy *= 1e-3/force_mag; @@ -832,192 +921,75 @@ void ScaLBL_ColorModel::Run(){ if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); color_db->putVector("F",{Fx,Fy,Fz}); - // simulate the point again with new force - CURRENT_STEADY_TIMESTEPS=0; } - else { - */ - MORPH_ADAPT = true; - CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - //****** ENDPOINT ADAPTATION ********/ - double krA_TMP= fabs(muA*flow_rate_A / force_mag); - double krB_TMP= fabs(muB*flow_rate_B / force_mag); - log_krA = log(krA_TMP); - if (krA_TMP < 0.0){ - // cannot do endpoint adaptation if kr is negative - log_krA = log_krA_prev; - } - else if (krA_TMP < krB_TMP && morph_delta > 0.0){ - /** morphological target based on relative permeability for A **/ - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); - if (rank==0){ - printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); - } - } - log_krA_prev = log_krA; - volA_prev = volA; - //******************************** **/ - /** compute averages & write data **/ - Averages->Full(); - Averages->Write(timestep); - analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - analysis.finish(); - - if (rank==0){ - printf("** WRITE STEADY POINT *** "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - double h = Dm->voxel_length; - // pressures - double pA = Averages->gnb.p; - double pB = Averages->gwb.p; - double pAc = Averages->gnc.p; - double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*5.796*alpha); - double pAB_connected = (pAc-pBc)/(h*5.796*alpha); - // connected contribution - double Vol_nc = Averages->gnc.V/Dm->Volume; - double Vol_wc = Averages->gwc.V/Dm->Volume; - double Vol_nd = Averages->gnd.V/Dm->Volume; - double Vol_wd = Averages->gwd.V/Dm->Volume; - double Mass_n = Averages->gnc.M + Averages->gnd.M; - double Mass_w = Averages->gwc.M + Averages->gwd.M; - double vAc_x = Averages->gnc.Px/Mass_n; - double vAc_y = Averages->gnc.Py/Mass_n; - double vAc_z = Averages->gnc.Pz/Mass_n; - double vBc_x = Averages->gwc.Px/Mass_w; - double vBc_y = Averages->gwc.Py/Mass_w; - double vBc_z = Averages->gwc.Pz/Mass_w; - // disconnected contribution - double vAd_x = Averages->gnd.Px/Mass_n; - double vAd_y = Averages->gnd.Py/Mass_n; - double vAd_z = Averages->gnd.Pz/Mass_n; - double vBd_x = Averages->gwd.Px/Mass_w; - double vBd_y = Averages->gwd.Py/Mass_w; - double vBd_z = Averages->gwd.Pz/Mass_w; - - double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); - double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); - double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); - double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); - - double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); - double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); - - double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); - double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); - - double kAeff = h*h*muA*(flow_rate_A)/(force_mag); - double kBeff = h*h*muB*(flow_rate_B)/(force_mag); - - double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; - double Mobility = muA/muB; - - bool WriteHeader=false; - FILE * kr_log_file = fopen("relperm.csv","r"); - if (kr_log_file != NULL) - fclose(kr_log_file); - else - WriteHeader=true; - kr_log_file = fopen("relperm.csv","a"); - if (WriteHeader) - fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); - - fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); - fclose(kr_log_file); - - printf(" Measured capillary number %f \n ",Ca); - } - if (SET_CAPILLARY_NUMBER ){ - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - - else{ - if (rank==0){ - printf("** Continue to simulate steady *** \n "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - } - } - morph_timesteps=0; - Ca_previous = Ca; - /*} - THIS IS WHERE YOU DISABLE STEADY POINT REJECTION - */ + + CURRENT_STEADY_TIMESTEPS = 0; } - - if (MORPH_ADAPT ){ - CURRENT_MORPH_TIMESTEPS += analysis_interval; - if (USE_DIRECT){ - // Use image sequence - IMAGE_INDEX++; - MORPH_ADAPT = false; - if (IMAGE_INDEX < IMAGE_COUNT){ - std::string next_image = ImageList[IMAGE_INDEX]; - if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); - color_db->putScalar("image_index",IMAGE_INDEX); - ImageInit(next_image); - } - else{ - if (rank==0) printf("Finished simulating image sequence \n"); - timestep = timestepMax; - } - } - else if (USE_SEED){ - delta_volume = volA*Dm->Volume - initial_volume; - CURRENT_MORPH_TIMESTEPS += analysis_interval; - double massChange = SeedPhaseField(seed_water); - if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); - } - else if (USE_MORPHOPEN_OIL){ - delta_volume = volA*Dm->Volume - initial_volume; - if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); - MorphOpenConnected(delta_volume_target); - } - else { - if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); - //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A - delta_volume += MorphInit(beta,delta_volume_target-delta_volume); - } - - if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; - } - else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - RESCALE_FORCE = true; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; + else{ + if (rank==0){ + printf("** Continue to simulate steady *** \n "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); } } - morph_timesteps += analysis_interval; + morph_timesteps=0; + Ca_previous = Ca; } - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + + if (MORPH_ADAPT ){ + CURRENT_MORPH_TIMESTEPS += analysis_interval; + if (USE_DIRECT){ + // Use image sequence + IMAGE_INDEX++; + MORPH_ADAPT = false; + if (IMAGE_INDEX < IMAGE_COUNT){ + std::string next_image = ImageList[IMAGE_INDEX]; + if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); + color_db->putScalar("image_index",IMAGE_INDEX); + ImageInit(next_image); + } + else{ + if (rank==0) printf("Finished simulating image sequence \n"); + timestep = timestepMax; + } + } + else if (USE_SEED){ + delta_volume = volA*Dm->Volume - initial_volume; + CURRENT_MORPH_TIMESTEPS += analysis_interval; + double massChange = SeedPhaseField(seed_water); + if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); + } + else if (USE_MORPHOPEN_OIL){ + delta_volume = volA*Dm->Volume - initial_volume; + if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); + MorphOpenConnected(delta_volume_target); + } + else { + if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); + //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A + delta_volume += MorphInit(beta,delta_volume_target-delta_volume); + } + + if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; + } + else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + RESCALE_FORCE = true; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; + } + } + morph_timesteps += analysis_interval; } + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } analysis.finish(); PROFILE_STOP("Loop"); @@ -1043,7 +1015,7 @@ void ScaLBL_ColorModel::Run(){ } double ScaLBL_ColorModel::ImageInit(std::string Filename){ - + if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); Mask->Decomp(Filename); for (int i=0; iid[i]; // save what was read @@ -1071,16 +1043,16 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ Count=sumReduce( Dm->Comm, Count); PoreCount=sumReduce( Dm->Comm, PoreCount); - + if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - + ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); double saturation = Count/PoreCount; @@ -1089,14 +1061,14 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){ } double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ - + int nx = Nx; int ny = Ny; int nz = Nz; int n; int N = nx*ny*nz; double volume_change=0.0; - + if (target_volume_change < 0.0){ Array id_solid(nx,ny,nz); Array phase_label(nx,ny,nz); @@ -1162,7 +1134,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ signed char notwater=1; double SW=-(target_volume_change)/count_connected; MorphOpen(distance, id_connected, Dm, SW, water, notwater); - + for (int k=0; k Date: Tue, 26 May 2020 20:59:35 -0400 Subject: [PATCH 152/270] add some miscellaneous changes --- models/GreyscaleSCModel.cpp | 57 ++++++++++++++++++++++++++++++++++--- models/GreyscaleSCModel.h | 1 + 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index c1cd9460..50674365 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -201,6 +201,8 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() double *Poros, *Perm; Poros = new double[Np]; Perm = new double[Np]; + //relPermA_host = new double[Np]; + //relPermB_host = new double[Np]; double *SolidPotentialA_host = new double [Nx*Ny*Nz]; double *SolidPotentialB_host = new double [Nx*Ny*Nz]; double *SolidForceA_host = new double[3*Np]; @@ -210,11 +212,15 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() signed char VALUE=0; double POROSITY=0.f; double PERMEABILITY=0.f; + //double RELPERMA=0.f; + //double RELPERMB=0.f; double AFFINITY_A=0.f; double AFFINITY_B=0.f; auto PorosityList = greyscaleSC_db->getVector( "PorosityList" ); auto PermeabilityList = greyscaleSC_db->getVector( "PermeabilityList" ); + //auto RelPermListA = greyscaleSC_db->getVector( "RelPermListA" ); + //auto RelPermListB = greyscaleSC_db->getVector( "RelPermListB" ); auto LabelList = greyscaleSC_db->getVector( "ComponentLabels" ); auto AffinityListA = greyscaleSC_db->getVector( "ComponentAffinityA" ); auto AffinityListB = greyscaleSC_db->getVector( "ComponentAffinityB" ); @@ -231,11 +237,19 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) //4. Requirement for "PermeabilityList": // *for ComponentLabels <=2, does not matter, can leave it as 1.0 + //5. Requirement for "RelPermListA" and "RelPermListB": + // *for ComponentLabels <=2, does not matter, can leave both RelPermA and RelPermB as 1.0 NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ - ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList and PermeabilityList must all be the same length! \n"); + if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || + NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ + ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, and PermeabilityList must all be the same length! \n"); } +// if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || +// NLABELS != RelPermListA.size() || NLABELS != RelPermListB.size() || +// NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ +// ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, PermeabilityList, and RelPermListA/B must all be the same length! \n"); +// } double label_count[NLABELS]; double label_count_global[NLABELS]; @@ -297,6 +311,35 @@ void ScaLBL_GreyscaleSCModel::AssignGreyscaleAndSolidLabels() } } } +// // New way of initializing the relperm values +// for (int k=0;k0) && (VALUE == LabelList[idx])){ +// RELPERMA=PermeabilityList[idx]*RelPermListA[idx]; +// RELPERMB=PermeabilityList[idx]*RelPermListB[idx]; +// idx = NLABELS; +// //Mask->id[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// int idx = Map(i,j,k); +// if (!(idx < 0)){ +// if (RELPERMA<=0.0 || RELPERMB<=0.0){ +// ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); +// } +// else{ +// relPermA_host[idx] = RELPERMA/Dm->voxel_length/Dm->voxel_length; +// relPermB_host[idx] = RELPERMB/Dm->voxel_length/Dm->voxel_length; +// } +// } +// } +// } +// } //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase for (int k=0;kgetVector( "GreyNodeLabels" ); } - if (greyscaleSC_db->keyExists( "GreyNodeSw" )){ + if (greyscaleSC_db->keyExists( "GreyNodeSwInit" )){ SwList.clear(); - SwList = greyscaleSC_db->getVector( "GreyNodeSw" ); + SwList = greyscaleSC_db->getVector( "GreyNodeSwInit" ); } NLABELS=LabelList.size(); @@ -694,6 +741,8 @@ void ScaLBL_GreyscaleSCModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &DenA, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &DenB, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &relPermA, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &relPermB, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index 314b535d..667eca16 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -68,6 +68,7 @@ public: int *dvcMap; double *fqA, *fqB; double *Permeability;//grey voxel permeability + //double relPermA,relPermB;//grey voxel relperm double *Porosity; double *Velocity; double *Pressure_dvc; From 661246472a247a912f111a44f6eb74f6515b7c8d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 27 May 2020 10:25:06 -0400 Subject: [PATCH 153/270] initialize the development of greyscale color model --- models/GreyscaleColorModel.cpp | 1393 ++++++++++++++++++++++++++++++++ models/GreyscaleColorModel.h | 102 +++ 2 files changed, 1495 insertions(+) create mode 100644 models/GreyscaleColorModel.cpp create mode 100644 models/GreyscaleColorModel.h diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp new file mode 100644 index 00000000..b2d20948 --- /dev/null +++ b/models/GreyscaleColorModel.cpp @@ -0,0 +1,1393 @@ +/* +Greyscale lattice boltzmann model + */ +#include "models/GreyscaleColorModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include +#include + +template +void DeleteArray( const TYPE *p ) +{ + delete [] p; +} + +ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), +rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + SignDist.resize(Nx,Ny,Nz); + SignDist.fill(0); + +} +ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ + +} + +void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + greyscaleColor_db = db->getDatabase( "GreyscaleColor" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tauA = 1.0; + tauB = 1.0; + tauA_eff = tauA; + tauB_eff = tauB; + rhoA = rhoB = 1.0; + rhoA_minor = rhoB_minor = 0.0;//NOTE this is for open nodes -> no dissolution + alpha = 0.001; + beta = 0.95; + tolerance = 0.01; + Fx = Fy = Fz = 0.0; + Restart=false; + din = dout = 1.0; + flux=0.0; + + // ---------------------- Greyscale Model parameters -----------------------// + if (greyscaleColor_db->keyExists( "timestepMax" )){ + timestepMax = greyscaleColor_db->getScalar( "timestepMax" ); + } + if (greyscaleColor_db->keyExists( "tauA" )){ + tauA = greyscaleColor_db->getScalar( "tauA" ); + } + if (greyscaleColor_db->keyExists( "tauB" )){ + tauB = greyscaleColor_db->getScalar( "tauB" ); + } + tauA_eff = greyscaleColor_db->getWithDefault( "tauA_eff", tauA ); + tauB_eff = greyscaleColor_db->getWithDefault( "tauB_eff", tauB ); + rhoA = greyscaleColor_db->getWithDefault( "rhoA", rhoA ); + rhoB = greyscaleColor_db->getWithDefault( "rhoB", rhoB ); + rhoA_minor = greyscaleColor_db->getWithDefault( "rhoA_minor", rhoA_minor ); + rhoB_minor = greyscaleColor_db->getWithDefault( "rhoB_minor", rhoB_minor ); + din = greyscaleColor_db->getWithDefault( "din", din ); + dout = greyscaleColor_db->getWithDefault( "dout", dout ); + if (greyscaleColor_db->keyExists( "F" )){ + Fx = greyscaleColor_db->getVector( "F" )[0]; + Fy = greyscaleColor_db->getVector( "F" )[1]; + Fz = greyscaleColor_db->getVector( "F" )[2]; + } + if (greyscaleColor_db->keyExists( "Restart" )){ + Restart = greyscaleColor_db->getScalar( "Restart" ); + } + if (greyscaleColor_db->keyExists( "flux" )){ + flux = greyscaleColor_db->getScalar( "flux" ); + } + if (greyscaleColor_db->keyExists( "tolerance" )){ + tolerance = greyscaleColor_db->getScalar( "tolerance" ); + } + // ------------------------------------------------------------------------// + + //------------------------ Other Domain parameters ------------------------// + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + // ------------------------------------------------------------------------// +} + +void ScaLBL_GreyscaleColorModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + + SignDist.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + PorosityMap.resize(Nx,Ny,Nz); + Pressure.resize(Nx,Ny,Nz); + DenA_data.resize(Nx,Ny,Nz); + DenB_data.resize(Nx,Ny,Nz); + + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_GreyscaleColorModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + int count = 0; + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kgetVector( "PorosityList" ); + auto PermeabilityList = greyscaleColor_db->getVector( "PermeabilityList" ); + auto LabelList = greyscaleColor_db->getVector( "ComponentLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "ComponentAffinity" ); + + //1. Requirement for "ComponentLabels": + // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 + // *label = 1 and 2 are reserved for NW and W phase respectively. + //2. Requirement for "ComponentAffinity": + // *should be in the same length as "ComponentLabels" + // *for label=1 and 2, the affinity is +1 and -1, respectively + // *for label>2, these are the effective wettability (affinity) for the greynodes + //3. Requirement for "PorosityList": + // *for ComponentLables <=0, put porosity value = 0.0; + // *for ComponentLabels >=3, put the corresponding sub-resolution porosity + // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) + //4. Requirement for "PermeabilityList": + // *for ComponentLabels <=2, does not matter, can leave it as 1.0 + //5. Requirement for "RelPermListA" and "RelPermListB": + // *for ComponentLabels <=2, does not matter, can leave both RelPermA and RelPermB as 1.0 + + NLABELS=LabelList.size(); + if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || + NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ + ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, and PermeabilityList must all be the same length! \n"); + } +// if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || +// NLABELS != RelPermListA.size() || NLABELS != RelPermListB.size() || +// NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ +// ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, PermeabilityList, and RelPermListA/B must all be the same length! \n"); +// } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + + for (int idx=0; idx 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability + for (int k=0;k0) && (VALUE == LabelList[idx])){ + POROSITY=PorosityList[idx]; + label_count[idx] += 1.0; + idx = NLABELS; + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Poros[idx] = POROSITY; + } + } + } + } + } + + //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes + //For node_ID <= 0: these are solid nodes of various wettability + for (int k=0;k0) && (VALUE == LabelList[idx])){ + PERMEABILITY=PermeabilityList[idx]; + idx = NLABELS; + //Mask->id[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Perm[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + } + } + } + } + } +// // New way of initializing the relperm values +// for (int k=0;k0) && (VALUE == LabelList[idx])){ +// RELPERMA=PermeabilityList[idx]*RelPermListA[idx]; +// RELPERMB=PermeabilityList[idx]*RelPermListB[idx]; +// idx = NLABELS; +// //Mask->id[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// int idx = Map(i,j,k); +// if (!(idx < 0)){ +// if (RELPERMA<=0.0 || RELPERMB<=0.0){ +// ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); +// } +// else{ +// relPermA_host[idx] = RELPERMA/Dm->voxel_length/Dm->voxel_length; +// relPermB_host[idx] = RELPERMB/Dm->voxel_length/Dm->voxel_length; +// } +// } +// } +// } +// } + + //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase + for (int k=0;k=3){ + AFFINITY_A=AffinityListA[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 + AFFINITY_B=AffinityListB[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 + } + else{//i.e. label = 1 or 2 + AFFINITY_A=0.0; + AFFINITY_B=0.0; + } + idx = NLABELS; + } + } + //NOTE: node_ID = 1 and 2 are reserved + if ((VALUE == 1)||(VALUE == 2)){ + AFFINITY_A=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists + AFFINITY_B=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists + } + SolidPotentialA_host[n] = AFFINITY_A; + SolidPotentialB_host[n] = AFFINITY_B; + } + } + } + + // Calculate Shan-Chen fluid-solid forces + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f/18.f; + double w_edge = 1.f/36.f; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kid[nn] <= 0)||(Mask->id[nn]>=3)){ + double vec_x = double(ii-1); + double vec_y = double(jj-1); + double vec_z = double(kk-1); + double GWNS_A=SolidPotentialA_host[nn]; + double GWNS_B=SolidPotentialB_host[nn]; + phi_x_A += -1.0*GWNS_A*weight*vec_x; + phi_y_A += -1.0*GWNS_A*weight*vec_y; + phi_z_A += -1.0*GWNS_A*weight*vec_z; + phi_x_B += -1.0*GWNS_B*weight*vec_x; + phi_y_B += -1.0*GWNS_B*weight*vec_y; + phi_z_B += -1.0*GWNS_B*weight*vec_z; + } + } + } + } + SolidForceA_host[idx+0*Np] = phi_x_A; + SolidForceA_host[idx+1*Np] = phi_y_A; + SolidForceA_host[idx+2*Np] = phi_z_A; + SolidForceB_host[idx+0*Np] = phi_x_B; + SolidForceB_host[idx+1*Np] = phi_y_B; + SolidForceB_host[idx+2*Np] = phi_z_B; + } + } + } + } + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm.sumReduce(label_count[idx]); + + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); + } + + //Copy all data to device + ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); + //ScaLBL_CopyToDevice(relPermA, relPermA_host, Np*sizeof(double)); + //ScaLBL_CopyToDevice(relPermB, relPermB_host, Np*sizeof(double)); + ScaLBL_CopyToDevice(SolidForceA, SolidForceA_host, 3*Np*sizeof(double)); + ScaLBL_CopyToDevice(SolidForceB, SolidForceB_host, 3*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] SolidPotentialA_host; + delete [] SolidPotentialB_host; + delete [] SolidForceA_host; + delete [] SolidForceB_host; + delete [] Poros; + delete [] Perm; + //delete [] relPermA_host; + //delete [] relPermB_host; + delete [] Dst; +} + +//void ScaLBL_GreyscaleColorModel::Density_Init(){ +// +// size_t NLABELS=0; +// signed char VALUE=0; +// +// vector LabelList{1,2}; +// vector SwList{0.0,1.0}; +// +// if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ +// LabelList.clear(); +// LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); +// } +// if (greyscaleColor_db->keyExists( "GreyNodeSw" )){ +// SwList.clear(); +// SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); +// } +// +// NLABELS=LabelList.size(); +// if (NLABELS != SwList.size()){ +// ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); +// } +// +// double *Den_temp; +// Den_temp=new double [2*Np]; +// double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. +// double nB=0.5; +// +// //double *Phi_temp; +// //Phi_temp=new double [Np]; +// //double phi = 0.0; +// +// for (int k=0; kid[n]; +// if (VALUE>0){ +// for (unsigned int idx=0; idx < NLABELS; idx++){ +// if (VALUE == LabelList[idx]){ +// double Sw = SwList[idx]; +// if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); +// nB=Sw; +// nA=1.0-Sw; +// //phi = nA-nB; +// idx = NLABELS; +// } +// } +// if (VALUE==1){//label=1 reserved for NW phase +// //TODO; maybe need rho_major and rho_minor initialization +// nA=rhoA; +// nB=rhoB_minor; +// //phi = nA-nB; +// } +// else if(VALUE==2){//label=2 reserved for W phase +// //TODO; maybe need rho_major and rho_minor initialization +// nA=rhoA_minor; +// nB=rhoB; +// //phi = nA-nB; +// } +// int idx = Map(i,j,k); +// Den_temp[idx+0*Np] = nA; +// Den_temp[idx+1*Np] = nB; +// //Phi_temp[idx] = phi; +// } +// } +// } +// } +// //copy to device +// ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); +// //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); +// ScaLBL_DeviceBarrier(); +// delete [] Den_temp; +// //delete [] Phi_temp; +//} + +void ScaLBL_GreyscaleColorModel::Density_Init(){ + + size_t NLABELS=0; + signed char VALUE=0; + + vector LabelList{1,2}; + vector SwList{0.0,1.0}; + + if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ + LabelList.clear(); + LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); + } + if (greyscaleColor_db->keyExists( "GreyNodeSwInit" )){ + SwList.clear(); + SwList = greyscaleColor_db->getVector( "GreyNodeSwInit" ); + } + + NLABELS=LabelList.size(); + if (NLABELS != SwList.size()){ + ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); + } + + double *DenA_temp,*DenB_temp; + DenA_temp=new double [Nx*Ny*Nz]; + DenB_temp=new double [Nx*Ny*Nz]; + double nA=0.0;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. + double nB=0.0; + + //double *Phi_temp; + //Phi_temp=new double [Np]; + //double phi = 0.0; + + for (int k=0; kid[n]; + if (VALUE>0){ + for (unsigned int idx=0; idx < NLABELS; idx++){ + if (VALUE == LabelList[idx]){ + double Sw = SwList[idx]; + if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); + nB=Sw; + nA=1.0-Sw; + //phi = nA-nB; + idx = NLABELS; + } + } + if (VALUE==1){//label=1 reserved for NW phase + nA=rhoA; + nB=rhoB_minor; + //phi = nA-nB; + } + else if(VALUE==2){//label=2 reserved for W phase + nA=rhoA_minor; + nB=rhoB; + //phi = nA-nB; + } + DenA_temp[n] = nA; + DenB_temp[n] = nB; + } + else{ //for ID<=0, i.e. all sorts of solid minerals, density is zero + DenA_temp[n] = 0.0; + DenB_temp[n] = 0.0; + } + } + } + } + + //copy to device + ScaLBL_CopyToDevice(DenA, DenA_temp, Nx*Ny*Nz*sizeof(double)); + ScaLBL_CopyToDevice(DenB, DenB_temp, Nx*Ny*Nz*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] DenA_temp; + delete [] DenB_temp; + + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,2); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-3); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-3); + } + } + +} + +void ScaLBL_GreyscaleColorModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + //......................................................... + // don't perform computations at the eight corners + //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fqA, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &fqB, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &DenA, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &DenB, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &relPermA, sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &relPermB, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidForceA, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidForceB, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device neighbor list \n"); + fflush(stdout); + // Copy the Map to device + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nx*Ny*Nz ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); +} + +void ScaLBL_GreyscaleColorModel::Initialize(){ + if (Restart == true){ +// //TODO: Restart funtion is currently not working; need updates +// if (rank==0){ +// printf("Initializing density field and distributions from Restart! \n"); +// } +// // Read in the restart file to CPU buffers +// std::shared_ptr cfq; +// cfq = std::shared_ptr(new double[19*Np],DeleteArray); +// std::shared_ptr cDen; +// cDen = std::shared_ptr(new double[2*Np],DeleteArray); +// FILE *File; +// File=fopen(LocalRestartFile,"rb"); +// fread(cfq.get(),sizeof(double),19*Np,File); +// fread(cDen.get(),sizeof(double),2*Np,File); +// fclose(File); +// +// // Copy the restart data to the GPU +// ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); +// ScaLBL_DeviceBarrier(); +// MPI_Barrier(comm); +// +// //TODO need proper initialization ! +// +// //TODO need to initialize velocity field ! +// //this is required for calculating the pressure_dvc +// //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity + } + else{ + if (rank==0) printf ("Initializing solid affinities \n"); + AssignGreyscaleAndSolidLabels(); + if (rank==0) printf ("Initializing density field \n"); + Density_Init();//initialize density field + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_GreyscaleColor_Init(dvcMap,fqA, fqB, DenA,DenB, Np); + +// //debug +// DoubleArray PhaseField(Nx,Ny,Nz); +// //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); +// ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); +// FILE *AFILE; +// sprintf(LocalRankFilename,"A_init.%05i.raw",rank); +// AFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,AFILE); +// fclose(AFILE); +// +// //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); +// ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); +// FILE *BFILE; +// sprintf(LocalRankFilename,"B_init.%05i.raw",rank); +// BFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,BFILE); +// fclose(BFILE); + + //Velocity also needs initialization (for old incompressible momentum transport) + //if (rank==0) printf ("Initializing velocity field \n"); + //double *vel_init; + //vel_init = new double [3*Np]; + //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; + //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); + //ScaLBL_DeviceBarrier(); + //delete [] vel_init; + } +} + +void ScaLBL_GreyscaleColorModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int visualization_interval = 1000; + int restart_interval = 10000; // number of timesteps in between in saving distributions for restart + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "visualization_interval" )){ + visualization_interval = analysis_db->getScalar( "visualization_interval" ); + } + if (analysis_db->keyExists( "restart_interval" )){ + restart_interval = analysis_db->getScalar( "restart_interval" ); + } + if (greyscaleColor_db->keyExists( "timestep" )){ + timestep = greyscaleColor_db->getScalar( "timestep" ); + } + + if (rank==0){ + printf("********************************************************\n"); + printf("No. of timesteps: %i \n", timestepMax); + fflush(stdout); + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); + //......................................... + + Minkowski Morphology(Mask); + + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + auto current_db = db->cloneDatabase(); + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + // *************ODD TIMESTEP*************// + timestep++; + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_GreyscaleColor_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleColor_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleColor_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } + ScaLBL_D3Q19_AAodd_GreyscaleColor_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + + //if (BoundaryCondition > 0){ + // ScaLBL_Comm->GreyscaleColor_BC_z(dvcMap, DenA, DenB, dinA, dinB); + // ScaLBL_Comm->GreyscaleColor_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + //} + + // Compute density gradient + // fluid component A + ScaLBL_Comm_Regular->SendHalo(DenA); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenA); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + } + } + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + // fluid component B + ScaLBL_Comm_Regular->SendHalo(DenB); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenB); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + } + } + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + + // Collsion + ScaLBL_D3Q19_AAodd_GreyscaleColor_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + // Collsion + ScaLBL_D3Q19_AAodd_GreyscaleColor_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + + // *************EVEN TIMESTEP*************// + timestep++; + // Compute the density field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL + ScaLBL_D3Q19_AAeven_GreyscaleColor_Density(dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->GreyscaleColor_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); + ScaLBL_Comm->GreyscaleColor_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } + ScaLBL_D3Q19_AAeven_GreyscaleColor_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); + + //if (BoundaryCondition > 0){ + // ScaLBL_Comm->GreyscaleColor_BC_z(dvcMap, DenA, DenB, dinA, dinB); + // ScaLBL_Comm->GreyscaleColor_BC_Z(dvcMap, DenA, DenB, doutA, doutB); + //} + + // Compute density gradient + // fluid component A + ScaLBL_Comm_Regular->SendHalo(DenA); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenA); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); + } + } + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + // fluid component B + ScaLBL_Comm_Regular->SendHalo(DenB); + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(DenB); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); + } + } + ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + + // Collsion + ScaLBL_D3Q19_AAeven_GreyscaleColor_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + // Collsion + ScaLBL_D3Q19_AAeven_GreyscaleColor_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, + tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, + 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + //************************************************************************/ + +// if (timestep%analysis_interval==0){ +// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); +// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); +// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); +// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); +// +// double count_loc=0; +// double count; +// double vax,vay,vaz; +// double vax_loc,vay_loc,vaz_loc; +// //double px_loc,py_loc,pz_loc; +// //double px,py,pz; +// //double mass_loc,mass_glb; +// +// //parameters for domain average +// int64_t i,j,k,n,imin,jmin,kmin,kmax; +// // If external boundary conditions are set, do not average over the inlet and outlet +// kmin=1; kmax=Nz-1; +// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 +// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; +// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; +// +// imin=jmin=1; +// // If inlet/outlet layers exist use these as default +// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; +// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; +// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer +// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; +// +//// px_loc = py_loc = pz_loc = 0.f; +//// mass_loc = 0.f; +//// for (int k=kmin; k 0){ +//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); +//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); +//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); +//// mass_loc += Den*PorosityMap(i,j,k); +//// } +//// } +//// } +//// } +//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); +//// +//// vax = px/mass_glb; +//// vay = py/mass_glb; +//// vaz = pz/mass_glb; +// +// vax_loc = vay_loc = vaz_loc = 0.f; +// for (int k=kmin; k 0){ +// vax_loc += Velocity_x(i,j,k); +// vay_loc += Velocity_y(i,j,k); +// vaz_loc += Velocity_z(i,j,k); +// count_loc+=1.0; +// } +// } +// } +// } +// vax = Mask->Comm.sumReduce( vax_loc ); +// vay = Mask->Comm.sumReduce( vay_loc ); +// vaz = Mask->Comm.sumReduce( vaz_loc ); +// count = Mask->Comm.sumReduce( count_loc ); +// +// vax /= count; +// vay /= count; +// vaz /= count; +// +// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); +// double dir_x = Fx/force_mag; +// double dir_y = Fy/force_mag; +// double dir_z = Fz/force_mag; +// if (force_mag == 0.0){ +// // default to z direction +// dir_x = 0.0; +// dir_y = 0.0; +// dir_z = 1.0; +// force_mag = 1.0; +// } +// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; +// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); +// +// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); +// flow_rate_previous = flow_rate; +// +// //if (rank==0) printf("Computing Minkowski functionals \n"); +// Morphology.ComputeScalar(SignDist,0.f); +// //Morphology.PrintAll(); +// double mu = (tau-0.5)/3.f; +// double Vs = Morphology.V(); +// double As = Morphology.A(); +// double Hs = Morphology.H(); +// double Xs = Morphology.X(); +// Vs = Dm->Comm.sumReduce( Vs); +// As = Dm->Comm.sumReduce( As); +// Hs = Dm->Comm.sumReduce( Hs); +// Xs = Dm->Comm.sumReduce( Xs); +// +// double h = Dm->voxel_length; +// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; +// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; +// +// if (rank==0){ +// printf(" AbsPerm = %.5g [micron^2]\n",absperm); +// bool WriteHeader=false; +// FILE * log_file = fopen("Permeability.csv","r"); +// if (log_file != NULL) +// fclose(log_file); +// else +// WriteHeader=true; +// log_file = fopen("Permeability.csv","a"); +// if (WriteHeader) +// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", +// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); +// +// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, +// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); +// fclose(log_file); +// } +// } + + if (timestep%visualization_interval==0){ + WriteOutput(); + } + +// if (timestep%restart_interval==0){ +// //Use rank=0 write out Restart.db +// if (rank==0) { +// greyscaleColor_db->putScalar("timestep",timestep); +// greyscaleColor_db->putScalar( "Restart", true ); +// current_db->putDatabase("GreyscaleColor", greyscaleColor_db); +// std::ofstream OutStream("Restart.db"); +// current_db->print(OutStream, ""); +// OutStream.close(); +// +// } +// //Write out Restart data. +// std::shared_ptr cfq; +// cfq = std::shared_ptr(new double[19*Np],DeleteArray); +// ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU +// +// FILE *RESTARTFILE; +// RESTARTFILE=fopen(LocalRestartFile,"wb"); +// fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); +// fclose(RESTARTFILE); +// MPI_Barrier(comm); +// } + } + + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + +void ScaLBL_GreyscaleColorModel::WriteOutput(){ + +/* Minkowski Morphology(Mask); + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); + + memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); + Morphology.Initialize(); + Morphology.UpdateMeshValues(); + Morphology.ComputeLocal(); + Morphology.Reduce(); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int n=0; nLastExterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + + for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double mu = (tau-0.5)/3.f; + if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); + if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, + Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); + */ + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + auto PressureVar = std::make_shared(); + auto DenAVar = std::make_shared(); + auto DenBVar = std::make_shared(); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + PressureVar->name = "Pressure"; + PressureVar->type = IO::VariableType::VolumeVariable; + PressureVar->dim = 1; + PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PressureVar); + + DenAVar->name = "DenA"; + DenAVar->type = IO::VariableType::VolumeVariable; + DenAVar->dim = 1; + DenAVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(DenAVar); + DenBVar->name = "DenB"; + DenBVar->type = IO::VariableType::VolumeVariable; + DenBVar->dim = 1; + DenBVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(DenBVar); + + Array& SignData = visData[0].vars[0]->data; + Array& VelxData = visData[0].vars[1]->data; + Array& VelyData = visData[0].vars[2]->data; + Array& VelzData = visData[0].vars[3]->data; + Array& PressureData = visData[0].vars[4]->data; + Array& DenAData = visData[0].vars[5]->data; + Array& DenBData = visData[0].vars[6]->data; + + ASSERT(visData[0].vars[0]->name=="SignDist"); + ASSERT(visData[0].vars[1]->name=="Velocity_x"); + ASSERT(visData[0].vars[2]->name=="Velocity_y"); + ASSERT(visData[0].vars[3]->name=="Velocity_z"); + ASSERT(visData[0].vars[4]->name=="Pressure"); + ASSERT(visData[0].vars[5]->name=="DenA"); + ASSERT(visData[0].vars[6]->name=="DenB"); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); + ScaLBL_CopyToHost(DenA_data.data(), DenA, sizeof(double)*N); + ScaLBL_CopyToHost(DenB_data.data(), DenB, sizeof(double)*N); + + fillData.copy(SignDist,SignData); + fillData.copy(Velocity_x,VelxData); + fillData.copy(Velocity_y,VelyData); + fillData.copy(Velocity_z,VelzData); + fillData.copy(Pressure,PressureData); + fillData.copy(DenA_data,DenAData); + fillData.copy(DenB_data,DenBData); + + IO::writeData( timestep, visData, Dm->Comm ); + +} + +void ScaLBL_GreyscaleColorModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + + //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); + +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +// + //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + +// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); +// FILE *POROS_FILE; +// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); +// POROS_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,POROS_FILE); +// fclose(POROS_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); +// FILE *PERM_FILE; +// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); +// PERM_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,PERM_FILE); +// fclose(PERM_FILE); +} diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h new file mode 100644 index 00000000..667eca16 --- /dev/null +++ b/models/GreyscaleColorModel.h @@ -0,0 +1,102 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "common/MPI.h" +#include "common/Database.h" +#include "common/ScaLBL.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_GreyscaleSCModel{ +public: + ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_GreyscaleSCModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + void WriteOutput(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + int CollisionType; + double tauA,tauB; + double tauA_eff,tauB_eff; + double Gsc; + double rhoA,rhoB; + double rhoA_minor,rhoB_minor;//dissolved density + double tolerance; + double Fx,Fy,Fz,flux; + double dinA,doutA; + double dinB,doutB; + double GreyPorosity; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + std::shared_ptr ScaLBL_Comm_Regular; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr greyscaleSC_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + signed char *id; + int *NeighborList; + int *dvcMap; + double *fqA, *fqB; + double *Permeability;//grey voxel permeability + //double relPermA,relPermB;//grey voxel relperm + double *Porosity; + double *Velocity; + double *Pressure_dvc; + double *DenA, *DenB; + double *DenGradA,*DenGradB; + double *SolidForceA,*SolidForceB; + + IntArray Map; + DoubleArray SignDist; + DoubleArray Velocity_x; + DoubleArray Velocity_y; + DoubleArray Velocity_z; + DoubleArray PorosityMap; + DoubleArray Pressure; + DoubleArray DenA_data; + DoubleArray DenB_data; + +private: + MPI_Comm comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + void AssignGreyscaleAndSolidLabels(); + void Density_Init(); +}; + From f5f4c51d5fafa786eeef1a1b32b32a4807b4e291 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 28 May 2020 18:22:25 -0400 Subject: [PATCH 154/270] discard greyscaleColor model --- models/GreyscaleColorModel.cpp | 1393 -------------------------------- models/GreyscaleColorModel.h | 102 --- 2 files changed, 1495 deletions(-) delete mode 100644 models/GreyscaleColorModel.cpp delete mode 100644 models/GreyscaleColorModel.h diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp deleted file mode 100644 index b2d20948..00000000 --- a/models/GreyscaleColorModel.cpp +++ /dev/null @@ -1,1393 +0,0 @@ -/* -Greyscale lattice boltzmann model - */ -#include "models/GreyscaleColorModel.h" -#include "analysis/distance.h" -#include "analysis/morphology.h" -#include -#include - -template -void DeleteArray( const TYPE *p ) -{ - delete [] p; -} - -ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), -rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) -{ - SignDist.resize(Nx,Ny,Nz); - SignDist.fill(0); - -} -ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ - -} - -void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ - // read the input database - db = std::make_shared( filename ); - domain_db = db->getDatabase( "Domain" ); - greyscaleColor_db = db->getDatabase( "GreyscaleColor" ); - analysis_db = db->getDatabase( "Analysis" ); - vis_db = db->getDatabase( "Visualization" ); - - // set defaults - timestepMax = 100000; - tauA = 1.0; - tauB = 1.0; - tauA_eff = tauA; - tauB_eff = tauB; - rhoA = rhoB = 1.0; - rhoA_minor = rhoB_minor = 0.0;//NOTE this is for open nodes -> no dissolution - alpha = 0.001; - beta = 0.95; - tolerance = 0.01; - Fx = Fy = Fz = 0.0; - Restart=false; - din = dout = 1.0; - flux=0.0; - - // ---------------------- Greyscale Model parameters -----------------------// - if (greyscaleColor_db->keyExists( "timestepMax" )){ - timestepMax = greyscaleColor_db->getScalar( "timestepMax" ); - } - if (greyscaleColor_db->keyExists( "tauA" )){ - tauA = greyscaleColor_db->getScalar( "tauA" ); - } - if (greyscaleColor_db->keyExists( "tauB" )){ - tauB = greyscaleColor_db->getScalar( "tauB" ); - } - tauA_eff = greyscaleColor_db->getWithDefault( "tauA_eff", tauA ); - tauB_eff = greyscaleColor_db->getWithDefault( "tauB_eff", tauB ); - rhoA = greyscaleColor_db->getWithDefault( "rhoA", rhoA ); - rhoB = greyscaleColor_db->getWithDefault( "rhoB", rhoB ); - rhoA_minor = greyscaleColor_db->getWithDefault( "rhoA_minor", rhoA_minor ); - rhoB_minor = greyscaleColor_db->getWithDefault( "rhoB_minor", rhoB_minor ); - din = greyscaleColor_db->getWithDefault( "din", din ); - dout = greyscaleColor_db->getWithDefault( "dout", dout ); - if (greyscaleColor_db->keyExists( "F" )){ - Fx = greyscaleColor_db->getVector( "F" )[0]; - Fy = greyscaleColor_db->getVector( "F" )[1]; - Fz = greyscaleColor_db->getVector( "F" )[2]; - } - if (greyscaleColor_db->keyExists( "Restart" )){ - Restart = greyscaleColor_db->getScalar( "Restart" ); - } - if (greyscaleColor_db->keyExists( "flux" )){ - flux = greyscaleColor_db->getScalar( "flux" ); - } - if (greyscaleColor_db->keyExists( "tolerance" )){ - tolerance = greyscaleColor_db->getScalar( "tolerance" ); - } - // ------------------------------------------------------------------------// - - //------------------------ Other Domain parameters ------------------------// - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } - // ------------------------------------------------------------------------// -} - -void ScaLBL_GreyscaleColorModel::SetDomain(){ - Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - // domain parameters - Nx = Dm->Nx; - Ny = Dm->Ny; - Nz = Dm->Nz; - Lx = Dm->Lx; - Ly = Dm->Ly; - Lz = Dm->Lz; - N = Nx*Ny*Nz; - - SignDist.resize(Nx,Ny,Nz); - Velocity_x.resize(Nx,Ny,Nz); - Velocity_y.resize(Nx,Ny,Nz); - Velocity_z.resize(Nx,Ny,Nz); - PorosityMap.resize(Nx,Ny,Nz); - Pressure.resize(Nx,Ny,Nz); - DenA_data.resize(Nx,Ny,Nz); - DenB_data.resize(Nx,Ny,Nz); - - id = new signed char [N]; - for (int i=0; iid[i] = 1; // initialize this way - MPI_Barrier(comm); - Dm->CommInit(); - MPI_Barrier(comm); - // Read domain parameters - rank = Dm->rank(); - nprocx = Dm->nprocx(); - nprocy = Dm->nprocy(); - nprocz = Dm->nprocz(); -} - -void ScaLBL_GreyscaleColorModel::ReadInput(){ - - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - - if (domain_db->keyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask->Decomp(Filename); - } - else{ - if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); - Mask->ReadIDs(); - } - for (int i=0; iid[i]; // save what was read - - // Generate the signed distance map - // Initialize the domain and communication - Array id_solid(Nx,Ny,Nz); - int count = 0; - // Solve for the position of the solid phase - for (int k=0;kid[n]; - if (label > 0) id_solid(i,j,k) = 1; - else id_solid(i,j,k) = 0; - } - } - } - // Initialize the signed distance function - for (int k=0;kgetVector( "PorosityList" ); - auto PermeabilityList = greyscaleColor_db->getVector( "PermeabilityList" ); - auto LabelList = greyscaleColor_db->getVector( "ComponentLabels" ); - auto AffinityList = greyscaleColor_db->getVector( "ComponentAffinity" ); - - //1. Requirement for "ComponentLabels": - // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 - // *label = 1 and 2 are reserved for NW and W phase respectively. - //2. Requirement for "ComponentAffinity": - // *should be in the same length as "ComponentLabels" - // *for label=1 and 2, the affinity is +1 and -1, respectively - // *for label>2, these are the effective wettability (affinity) for the greynodes - //3. Requirement for "PorosityList": - // *for ComponentLables <=0, put porosity value = 0.0; - // *for ComponentLabels >=3, put the corresponding sub-resolution porosity - // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) - //4. Requirement for "PermeabilityList": - // *for ComponentLabels <=2, does not matter, can leave it as 1.0 - //5. Requirement for "RelPermListA" and "RelPermListB": - // *for ComponentLabels <=2, does not matter, can leave both RelPermA and RelPermB as 1.0 - - NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || - NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ - ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, and PermeabilityList must all be the same length! \n"); - } -// if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || -// NLABELS != RelPermListA.size() || NLABELS != RelPermListB.size() || -// NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ -// ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, PermeabilityList, and RelPermListA/B must all be the same length! \n"); -// } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - - for (int idx=0; idx 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - POROSITY=PorosityList[idx]; - label_count[idx] += 1.0; - idx = NLABELS; - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (POROSITY<=0.0){ - ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); - } - else{ - Poros[idx] = POROSITY; - } - } - } - } - } - - //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - PERMEABILITY=PermeabilityList[idx]; - idx = NLABELS; - //Mask->id[n] = 0; // set mask to zero since this is an immobile component - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (PERMEABILITY<=0.0){ - ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); - } - else{ - Perm[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; - } - } - } - } - } -// // New way of initializing the relperm values -// for (int k=0;k0) && (VALUE == LabelList[idx])){ -// RELPERMA=PermeabilityList[idx]*RelPermListA[idx]; -// RELPERMB=PermeabilityList[idx]*RelPermListB[idx]; -// idx = NLABELS; -// //Mask->id[n] = 0; // set mask to zero since this is an immobile component -// } -// } -// int idx = Map(i,j,k); -// if (!(idx < 0)){ -// if (RELPERMA<=0.0 || RELPERMB<=0.0){ -// ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); -// } -// else{ -// relPermA_host[idx] = RELPERMA/Dm->voxel_length/Dm->voxel_length; -// relPermB_host[idx] = RELPERMB/Dm->voxel_length/Dm->voxel_length; -// } -// } -// } -// } -// } - - //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase - for (int k=0;k=3){ - AFFINITY_A=AffinityListA[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 - AFFINITY_B=AffinityListB[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 - } - else{//i.e. label = 1 or 2 - AFFINITY_A=0.0; - AFFINITY_B=0.0; - } - idx = NLABELS; - } - } - //NOTE: node_ID = 1 and 2 are reserved - if ((VALUE == 1)||(VALUE == 2)){ - AFFINITY_A=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists - AFFINITY_B=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists - } - SolidPotentialA_host[n] = AFFINITY_A; - SolidPotentialB_host[n] = AFFINITY_B; - } - } - } - - // Calculate Shan-Chen fluid-solid forces - double *Dst; - Dst = new double [3*3*3]; - for (int kk=0; kk<3; kk++){ - for (int jj=0; jj<3; jj++){ - for (int ii=0; ii<3; ii++){ - int index = kk*9+jj*3+ii; - Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); - } - } - } - double w_face = 1.f/18.f; - double w_edge = 1.f/36.f; - double w_corner = 0.f; - //local - Dst[13] = 0.f; - //faces - Dst[4] = w_face; - Dst[10] = w_face; - Dst[12] = w_face; - Dst[14] = w_face; - Dst[16] = w_face; - Dst[22] = w_face; - // corners - Dst[0] = w_corner; - Dst[2] = w_corner; - Dst[6] = w_corner; - Dst[8] = w_corner; - Dst[18] = w_corner; - Dst[20] = w_corner; - Dst[24] = w_corner; - Dst[26] = w_corner; - // edges - Dst[1] = w_edge; - Dst[3] = w_edge; - Dst[5] = w_edge; - Dst[7] = w_edge; - Dst[9] = w_edge; - Dst[11] = w_edge; - Dst[15] = w_edge; - Dst[17] = w_edge; - Dst[19] = w_edge; - Dst[21] = w_edge; - Dst[23] = w_edge; - Dst[25] = w_edge; - - for (int k=1; kid[nn] <= 0)||(Mask->id[nn]>=3)){ - double vec_x = double(ii-1); - double vec_y = double(jj-1); - double vec_z = double(kk-1); - double GWNS_A=SolidPotentialA_host[nn]; - double GWNS_B=SolidPotentialB_host[nn]; - phi_x_A += -1.0*GWNS_A*weight*vec_x; - phi_y_A += -1.0*GWNS_A*weight*vec_y; - phi_z_A += -1.0*GWNS_A*weight*vec_z; - phi_x_B += -1.0*GWNS_B*weight*vec_x; - phi_y_B += -1.0*GWNS_B*weight*vec_y; - phi_z_B += -1.0*GWNS_B*weight*vec_z; - } - } - } - } - SolidForceA_host[idx+0*Np] = phi_x_A; - SolidForceA_host[idx+1*Np] = phi_y_A; - SolidForceA_host[idx+2*Np] = phi_z_A; - SolidForceB_host[idx+0*Np] = phi_x_B; - SolidForceB_host[idx+1*Np] = phi_y_B; - SolidForceB_host[idx+2*Np] = phi_z_B; - } - } - } - } - - // Set Dm to match Mask - for (int i=0; iid[i] = Mask->id[i]; - - for (int idx=0; idxComm.sumReduce(label_count[idx]); - - //Initialize a weighted porosity after considering grey voxels - GreyPorosity=0.0; - for (unsigned int idx=0; idxvoxel_length); - printf("Component labels: %lu \n",NLABELS); - for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); - printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); - } - printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); - } - - //Copy all data to device - ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); - ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); - //ScaLBL_CopyToDevice(relPermA, relPermA_host, Np*sizeof(double)); - //ScaLBL_CopyToDevice(relPermB, relPermB_host, Np*sizeof(double)); - ScaLBL_CopyToDevice(SolidForceA, SolidForceA_host, 3*Np*sizeof(double)); - ScaLBL_CopyToDevice(SolidForceB, SolidForceB_host, 3*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] SolidPotentialA_host; - delete [] SolidPotentialB_host; - delete [] SolidForceA_host; - delete [] SolidForceB_host; - delete [] Poros; - delete [] Perm; - //delete [] relPermA_host; - //delete [] relPermB_host; - delete [] Dst; -} - -//void ScaLBL_GreyscaleColorModel::Density_Init(){ -// -// size_t NLABELS=0; -// signed char VALUE=0; -// -// vector LabelList{1,2}; -// vector SwList{0.0,1.0}; -// -// if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ -// LabelList.clear(); -// LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); -// } -// if (greyscaleColor_db->keyExists( "GreyNodeSw" )){ -// SwList.clear(); -// SwList = greyscaleColor_db->getVector( "GreyNodeSw" ); -// } -// -// NLABELS=LabelList.size(); -// if (NLABELS != SwList.size()){ -// ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); -// } -// -// double *Den_temp; -// Den_temp=new double [2*Np]; -// double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. -// double nB=0.5; -// -// //double *Phi_temp; -// //Phi_temp=new double [Np]; -// //double phi = 0.0; -// -// for (int k=0; kid[n]; -// if (VALUE>0){ -// for (unsigned int idx=0; idx < NLABELS; idx++){ -// if (VALUE == LabelList[idx]){ -// double Sw = SwList[idx]; -// if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); -// nB=Sw; -// nA=1.0-Sw; -// //phi = nA-nB; -// idx = NLABELS; -// } -// } -// if (VALUE==1){//label=1 reserved for NW phase -// //TODO; maybe need rho_major and rho_minor initialization -// nA=rhoA; -// nB=rhoB_minor; -// //phi = nA-nB; -// } -// else if(VALUE==2){//label=2 reserved for W phase -// //TODO; maybe need rho_major and rho_minor initialization -// nA=rhoA_minor; -// nB=rhoB; -// //phi = nA-nB; -// } -// int idx = Map(i,j,k); -// Den_temp[idx+0*Np] = nA; -// Den_temp[idx+1*Np] = nB; -// //Phi_temp[idx] = phi; -// } -// } -// } -// } -// //copy to device -// ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); -// //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); -// ScaLBL_DeviceBarrier(); -// delete [] Den_temp; -// //delete [] Phi_temp; -//} - -void ScaLBL_GreyscaleColorModel::Density_Init(){ - - size_t NLABELS=0; - signed char VALUE=0; - - vector LabelList{1,2}; - vector SwList{0.0,1.0}; - - if (greyscaleColor_db->keyExists( "GreyNodeLabels" )){ - LabelList.clear(); - LabelList = greyscaleColor_db->getVector( "GreyNodeLabels" ); - } - if (greyscaleColor_db->keyExists( "GreyNodeSwInit" )){ - SwList.clear(); - SwList = greyscaleColor_db->getVector( "GreyNodeSwInit" ); - } - - NLABELS=LabelList.size(); - if (NLABELS != SwList.size()){ - ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); - } - - double *DenA_temp,*DenB_temp; - DenA_temp=new double [Nx*Ny*Nz]; - DenB_temp=new double [Nx*Ny*Nz]; - double nA=0.0;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. - double nB=0.0; - - //double *Phi_temp; - //Phi_temp=new double [Np]; - //double phi = 0.0; - - for (int k=0; kid[n]; - if (VALUE>0){ - for (unsigned int idx=0; idx < NLABELS; idx++){ - if (VALUE == LabelList[idx]){ - double Sw = SwList[idx]; - if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); - nB=Sw; - nA=1.0-Sw; - //phi = nA-nB; - idx = NLABELS; - } - } - if (VALUE==1){//label=1 reserved for NW phase - nA=rhoA; - nB=rhoB_minor; - //phi = nA-nB; - } - else if(VALUE==2){//label=2 reserved for W phase - nA=rhoA_minor; - nB=rhoB; - //phi = nA-nB; - } - DenA_temp[n] = nA; - DenB_temp[n] = nB; - } - else{ //for ID<=0, i.e. all sorts of solid minerals, density is zero - DenA_temp[n] = 0.0; - DenB_temp[n] = 0.0; - } - } - } - } - - //copy to device - ScaLBL_CopyToDevice(DenA, DenA_temp, Nx*Ny*Nz*sizeof(double)); - ScaLBL_CopyToDevice(DenB, DenB_temp, Nx*Ny*Nz*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] DenA_temp; - delete [] DenB_temp; - - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,2); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-3); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-3); - } - } - -} - -void ScaLBL_GreyscaleColorModel::Create(){ - /* - * This function creates the variables needed to run a LBM - */ - //......................................................... - // don't perform computations at the eight corners - //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - - //......................................................... - // Initialize communication structures in averaging domain - for (int i=0; iid[i] = Mask->id[i]; - Mask->CommInit(); - Np=Mask->PoreCount(); - //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - // Create a communicator for the device (will use optimized layout) - // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); - ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); - - int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); - Map.resize(Nx,Ny,Nz); Map.fill(-2); - auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - MPI_Barrier(comm); - - //........................................................................... - // MAIN VARIABLES ALLOCATED HERE - //........................................................................... - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - //......................device distributions................................. - dist_mem_size = Np*sizeof(double); - neighborSize=18*(Np*sizeof(int)); - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &fqA, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &fqB, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &DenA, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &DenB, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &relPermA, sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &relPermB, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &SolidForceA, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &SolidForceB, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device neighbor list \n"); - fflush(stdout); - // Copy the Map to device - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; kLastExterior(); idx++){ - auto n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n", n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - auto n = TmpMap[idx]; - if ( n > Nx*Ny*Nz ){ - printf("Bad value! idx=%i \n",n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; - // copy the neighbor list - ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); -} - -void ScaLBL_GreyscaleColorModel::Initialize(){ - if (Restart == true){ -// //TODO: Restart funtion is currently not working; need updates -// if (rank==0){ -// printf("Initializing density field and distributions from Restart! \n"); -// } -// // Read in the restart file to CPU buffers -// std::shared_ptr cfq; -// cfq = std::shared_ptr(new double[19*Np],DeleteArray); -// std::shared_ptr cDen; -// cDen = std::shared_ptr(new double[2*Np],DeleteArray); -// FILE *File; -// File=fopen(LocalRestartFile,"rb"); -// fread(cfq.get(),sizeof(double),19*Np,File); -// fread(cDen.get(),sizeof(double),2*Np,File); -// fclose(File); -// -// // Copy the restart data to the GPU -// ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); -// ScaLBL_DeviceBarrier(); -// MPI_Barrier(comm); -// -// //TODO need proper initialization ! -// -// //TODO need to initialize velocity field ! -// //this is required for calculating the pressure_dvc -// //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity - } - else{ - if (rank==0) printf ("Initializing solid affinities \n"); - AssignGreyscaleAndSolidLabels(); - if (rank==0) printf ("Initializing density field \n"); - Density_Init();//initialize density field - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_GreyscaleColor_Init(dvcMap,fqA, fqB, DenA,DenB, Np); - -// //debug -// DoubleArray PhaseField(Nx,Ny,Nz); -// //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A_init.%05i.raw",rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B_init.%05i.raw",rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); - - //Velocity also needs initialization (for old incompressible momentum transport) - //if (rank==0) printf ("Initializing velocity field \n"); - //double *vel_init; - //vel_init = new double [3*Np]; - //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; - //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); - //ScaLBL_DeviceBarrier(); - //delete [] vel_init; - } -} - -void ScaLBL_GreyscaleColorModel::Run(){ - int nprocs=nprocx*nprocy*nprocz; - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - int analysis_interval = 1000; // number of timesteps in between in situ analysis - int visualization_interval = 1000; - int restart_interval = 10000; // number of timesteps in between in saving distributions for restart - if (analysis_db->keyExists( "analysis_interval" )){ - analysis_interval = analysis_db->getScalar( "analysis_interval" ); - } - if (analysis_db->keyExists( "visualization_interval" )){ - visualization_interval = analysis_db->getScalar( "visualization_interval" ); - } - if (analysis_db->keyExists( "restart_interval" )){ - restart_interval = analysis_db->getScalar( "restart_interval" ); - } - if (greyscaleColor_db->keyExists( "timestep" )){ - timestep = greyscaleColor_db->getScalar( "timestep" ); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("No. of timesteps: %i \n", timestepMax); - fflush(stdout); - } - - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... - - Minkowski Morphology(Mask); - - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - auto current_db = db->cloneDatabase(); - double error = 1.0; - double flow_rate_previous = 0.0; - while (timestep < timestepMax && error > tolerance) { - //************************************************************************/ - // *************ODD TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleColor_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleColor_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleColor_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - ScaLBL_D3Q19_AAodd_GreyscaleColor_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - - //if (BoundaryCondition > 0){ - // ScaLBL_Comm->GreyscaleColor_BC_z(dvcMap, DenA, DenB, dinA, dinB); - // ScaLBL_Comm->GreyscaleColor_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - //} - - // Compute density gradient - // fluid component A - ScaLBL_Comm_Regular->SendHalo(DenA); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenA); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - // fluid component B - ScaLBL_Comm_Regular->SendHalo(DenB); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenB); - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - - // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleColor_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleColor_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - - // *************EVEN TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleColor_Density(dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleColor_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleColor_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - ScaLBL_D3Q19_AAeven_GreyscaleColor_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - - //if (BoundaryCondition > 0){ - // ScaLBL_Comm->GreyscaleColor_BC_z(dvcMap, DenA, DenB, dinA, dinB); - // ScaLBL_Comm->GreyscaleColor_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - //} - - // Compute density gradient - // fluid component A - ScaLBL_Comm_Regular->SendHalo(DenA); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenA); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - // fluid component B - ScaLBL_Comm_Regular->SendHalo(DenB); - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenB); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleColor_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - - // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleColor_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleColor_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - //************************************************************************/ - -// if (timestep%analysis_interval==0){ -// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); -// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); -// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); -// -// double count_loc=0; -// double count; -// double vax,vay,vaz; -// double vax_loc,vay_loc,vaz_loc; -// //double px_loc,py_loc,pz_loc; -// //double px,py,pz; -// //double mass_loc,mass_glb; -// -// //parameters for domain average -// int64_t i,j,k,n,imin,jmin,kmin,kmax; -// // If external boundary conditions are set, do not average over the inlet and outlet -// kmin=1; kmax=Nz-1; -// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 -// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; -// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; -// -// imin=jmin=1; -// // If inlet/outlet layers exist use these as default -// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; -// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; -// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer -// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; -// -//// px_loc = py_loc = pz_loc = 0.f; -//// mass_loc = 0.f; -//// for (int k=kmin; k 0){ -//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); -//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); -//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); -//// mass_loc += Den*PorosityMap(i,j,k); -//// } -//// } -//// } -//// } -//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// -//// vax = px/mass_glb; -//// vay = py/mass_glb; -//// vaz = pz/mass_glb; -// -// vax_loc = vay_loc = vaz_loc = 0.f; -// for (int k=kmin; k 0){ -// vax_loc += Velocity_x(i,j,k); -// vay_loc += Velocity_y(i,j,k); -// vaz_loc += Velocity_z(i,j,k); -// count_loc+=1.0; -// } -// } -// } -// } -// vax = Mask->Comm.sumReduce( vax_loc ); -// vay = Mask->Comm.sumReduce( vay_loc ); -// vaz = Mask->Comm.sumReduce( vaz_loc ); -// count = Mask->Comm.sumReduce( count_loc ); -// -// vax /= count; -// vay /= count; -// vaz /= count; -// -// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); -// double dir_x = Fx/force_mag; -// double dir_y = Fy/force_mag; -// double dir_z = Fz/force_mag; -// if (force_mag == 0.0){ -// // default to z direction -// dir_x = 0.0; -// dir_y = 0.0; -// dir_z = 1.0; -// force_mag = 1.0; -// } -// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; -// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); -// -// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); -// flow_rate_previous = flow_rate; -// -// //if (rank==0) printf("Computing Minkowski functionals \n"); -// Morphology.ComputeScalar(SignDist,0.f); -// //Morphology.PrintAll(); -// double mu = (tau-0.5)/3.f; -// double Vs = Morphology.V(); -// double As = Morphology.A(); -// double Hs = Morphology.H(); -// double Xs = Morphology.X(); -// Vs = Dm->Comm.sumReduce( Vs); -// As = Dm->Comm.sumReduce( As); -// Hs = Dm->Comm.sumReduce( Hs); -// Xs = Dm->Comm.sumReduce( Xs); -// -// double h = Dm->voxel_length; -// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; -// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; -// -// if (rank==0){ -// printf(" AbsPerm = %.5g [micron^2]\n",absperm); -// bool WriteHeader=false; -// FILE * log_file = fopen("Permeability.csv","r"); -// if (log_file != NULL) -// fclose(log_file); -// else -// WriteHeader=true; -// log_file = fopen("Permeability.csv","a"); -// if (WriteHeader) -// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", -// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); -// -// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, -// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); -// fclose(log_file); -// } -// } - - if (timestep%visualization_interval==0){ - WriteOutput(); - } - -// if (timestep%restart_interval==0){ -// //Use rank=0 write out Restart.db -// if (rank==0) { -// greyscaleColor_db->putScalar("timestep",timestep); -// greyscaleColor_db->putScalar( "Restart", true ); -// current_db->putDatabase("GreyscaleColor", greyscaleColor_db); -// std::ofstream OutStream("Restart.db"); -// current_db->print(OutStream, ""); -// OutStream.close(); -// -// } -// //Write out Restart data. -// std::shared_ptr cfq; -// cfq = std::shared_ptr(new double[19*Np],DeleteArray); -// ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU -// -// FILE *RESTARTFILE; -// RESTARTFILE=fopen(LocalRestartFile,"wb"); -// fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); -// fclose(RESTARTFILE); -// MPI_Barrier(comm); -// } - } - - PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_greyscale_simulator",1); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // ************************************************************************ -} - -void ScaLBL_GreyscaleColorModel::WriteOutput(){ - -/* Minkowski Morphology(Mask); - int SIZE=Np*sizeof(double); - ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); - - memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); - Morphology.Initialize(); - Morphology.UpdateMeshValues(); - Morphology.ComputeLocal(); - Morphology.Reduce(); - - double count_loc=0; - double count; - double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - vax_loc = vay_loc = vaz_loc = 0.f; - for (int n=0; nLastExterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - - for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - - vax /= count; - vay /= count; - vaz /= count; - - double mu = (tau-0.5)/3.f; - if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); - if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, - Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); - */ - - std::vector visData; - fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); - - auto VxVar = std::make_shared(); - auto VyVar = std::make_shared(); - auto VzVar = std::make_shared(); - auto SignDistVar = std::make_shared(); - auto PressureVar = std::make_shared(); - auto DenAVar = std::make_shared(); - auto DenBVar = std::make_shared(); - - IO::initialize("","silo","false"); - // Create the MeshDataStruct - visData.resize(1); - visData[0].meshName = "domain"; - visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); - SignDistVar->name = "SignDist"; - SignDistVar->type = IO::VariableType::VolumeVariable; - SignDistVar->dim = 1; - SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(SignDistVar); - - VxVar->name = "Velocity_x"; - VxVar->type = IO::VariableType::VolumeVariable; - VxVar->dim = 1; - VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VxVar); - VyVar->name = "Velocity_y"; - VyVar->type = IO::VariableType::VolumeVariable; - VyVar->dim = 1; - VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VyVar); - VzVar->name = "Velocity_z"; - VzVar->type = IO::VariableType::VolumeVariable; - VzVar->dim = 1; - VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VzVar); - - PressureVar->name = "Pressure"; - PressureVar->type = IO::VariableType::VolumeVariable; - PressureVar->dim = 1; - PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(PressureVar); - - DenAVar->name = "DenA"; - DenAVar->type = IO::VariableType::VolumeVariable; - DenAVar->dim = 1; - DenAVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(DenAVar); - DenBVar->name = "DenB"; - DenBVar->type = IO::VariableType::VolumeVariable; - DenBVar->dim = 1; - DenBVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(DenBVar); - - Array& SignData = visData[0].vars[0]->data; - Array& VelxData = visData[0].vars[1]->data; - Array& VelyData = visData[0].vars[2]->data; - Array& VelzData = visData[0].vars[3]->data; - Array& PressureData = visData[0].vars[4]->data; - Array& DenAData = visData[0].vars[5]->data; - Array& DenBData = visData[0].vars[6]->data; - - ASSERT(visData[0].vars[0]->name=="SignDist"); - ASSERT(visData[0].vars[1]->name=="Velocity_x"); - ASSERT(visData[0].vars[2]->name=="Velocity_y"); - ASSERT(visData[0].vars[3]->name=="Velocity_z"); - ASSERT(visData[0].vars[4]->name=="Pressure"); - ASSERT(visData[0].vars[5]->name=="DenA"); - ASSERT(visData[0].vars[6]->name=="DenB"); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); - ScaLBL_CopyToHost(DenA_data.data(), DenA, sizeof(double)*N); - ScaLBL_CopyToHost(DenB_data.data(), DenB, sizeof(double)*N); - - fillData.copy(SignDist,SignData); - fillData.copy(Velocity_x,VelxData); - fillData.copy(Velocity_y,VelyData); - fillData.copy(Velocity_z,VelzData); - fillData.copy(Pressure,PressureData); - fillData.copy(DenA_data,DenAData); - fillData.copy(DenB_data,DenBData); - - IO::writeData( timestep, visData, Dm->Comm ); - -} - -void ScaLBL_GreyscaleColorModel::WriteDebug(){ - // Copy back final phase indicator field and convert to regular layout - DoubleArray PhaseField(Nx,Ny,Nz); - - //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); - -// FILE *OUTFILE; -// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); -// OUTFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,OUTFILE); -// fclose(OUTFILE); -// - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); - FILE *AFILE; - sprintf(LocalRankFilename,"A.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); - FILE *BFILE; - sprintf(LocalRankFilename,"B.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); - - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); - FILE *PFILE; - sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); - PFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PFILE); - fclose(PFILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); - FILE *VELX_FILE; - sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); - VELX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELX_FILE); - fclose(VELX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); - FILE *VELY_FILE; - sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); - VELY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELY_FILE); - fclose(VELY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); - FILE *VELZ_FILE; - sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); - VELZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELZ_FILE); - fclose(VELZ_FILE); - -// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); -// FILE *POROS_FILE; -// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); -// POROS_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,POROS_FILE); -// fclose(POROS_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); -// FILE *PERM_FILE; -// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); -// PERM_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,PERM_FILE); -// fclose(PERM_FILE); -} diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h deleted file mode 100644 index 667eca16..00000000 --- a/models/GreyscaleColorModel.h +++ /dev/null @@ -1,102 +0,0 @@ -/* -Implementation of color lattice boltzmann model - */ -#include -#include -#include -#include -#include -#include -#include - -#include "common/Communication.h" -#include "common/MPI.h" -#include "common/Database.h" -#include "common/ScaLBL.h" -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" - -class ScaLBL_GreyscaleSCModel{ -public: - ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleSCModel(); - - // functions in they should be run - void ReadParams(string filename); - void ReadParams(std::shared_ptr db0); - void SetDomain(); - void ReadInput(); - void Create(); - void Initialize(); - void Run(); - void WriteDebug(); - void WriteOutput(); - - bool Restart,pBC; - int timestep,timestepMax; - int BoundaryCondition; - int CollisionType; - double tauA,tauB; - double tauA_eff,tauB_eff; - double Gsc; - double rhoA,rhoB; - double rhoA_minor,rhoB_minor;//dissolved density - double tolerance; - double Fx,Fy,Fz,flux; - double dinA,doutA; - double dinB,doutB; - double GreyPorosity; - - int Nx,Ny,Nz,N,Np; - int rank,nprocx,nprocy,nprocz,nprocs; - double Lx,Ly,Lz; - - std::shared_ptr Dm; // this domain is for analysis - std::shared_ptr Mask; // this domain is for lbm - std::shared_ptr ScaLBL_Comm; - std::shared_ptr ScaLBL_Comm_Regular; - - // input database - std::shared_ptr db; - std::shared_ptr domain_db; - std::shared_ptr greyscaleSC_db; - std::shared_ptr analysis_db; - std::shared_ptr vis_db; - - signed char *id; - int *NeighborList; - int *dvcMap; - double *fqA, *fqB; - double *Permeability;//grey voxel permeability - //double relPermA,relPermB;//grey voxel relperm - double *Porosity; - double *Velocity; - double *Pressure_dvc; - double *DenA, *DenB; - double *DenGradA,*DenGradB; - double *SolidForceA,*SolidForceB; - - IntArray Map; - DoubleArray SignDist; - DoubleArray Velocity_x; - DoubleArray Velocity_y; - DoubleArray Velocity_z; - DoubleArray PorosityMap; - DoubleArray Pressure; - DoubleArray DenA_data; - DoubleArray DenB_data; - -private: - MPI_Comm comm; - - int dist_mem_size; - int neighborSize; - // filenames - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - - void AssignGreyscaleAndSolidLabels(); - void Density_Init(); -}; - From a404f4c8af2fb4948ff167f01fc5d2700d167c4a Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 29 May 2020 10:28:11 -0400 Subject: [PATCH 155/270] GreyscaleSC model: update the density initialization on grey nodes --- models/GreyscaleSCModel.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index 50674365..f1dc9ab2 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -606,20 +606,25 @@ void ScaLBL_GreyscaleSCModel::Density_Init(){ signed char VALUE=0; vector LabelList{1,2}; - vector SwList{0.0,1.0}; + vector GreyDenAList{rhoA,rhoB_minor}; + vector GreyDenBList{rhoB,rhoA_minor}; if (greyscaleSC_db->keyExists( "GreyNodeLabels" )){ LabelList.clear(); LabelList = greyscaleSC_db->getVector( "GreyNodeLabels" ); } - if (greyscaleSC_db->keyExists( "GreyNodeSwInit" )){ - SwList.clear(); - SwList = greyscaleSC_db->getVector( "GreyNodeSwInit" ); + if (greyscaleSC_db->keyExists( "GreyNodeDenAInit" )){ + GreyDenAList.clear(); + GreyDenAList = greyscaleSC_db->getVector( "GreyNodeDenAInit" ); + } + if (greyscaleSC_db->keyExists( "GreyNodeDenBInit" )){ + GreyDenBList.clear(); + GreyDenBList = greyscaleSC_db->getVector( "GreyNodeDenBInit" ); } NLABELS=LabelList.size(); - if (NLABELS != SwList.size()){ - ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); + if (NLABELS != GreyDenAList.size() || NLABELS != GreyDenBList.size()){ + ERROR("Error: GreyNodeLabels, GreyNodeDenAInit, and GreyNodeDenBInit must all be the same length! \n"); } double *DenA_temp,*DenB_temp; @@ -640,10 +645,8 @@ void ScaLBL_GreyscaleSCModel::Density_Init(){ if (VALUE>0){ for (unsigned int idx=0; idx < NLABELS; idx++){ if (VALUE == LabelList[idx]){ - double Sw = SwList[idx]; - if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); - nB=Sw; - nA=1.0-Sw; + nA=GreyDenAList[idx]; + nB=GreyDenBList[idx]; //phi = nA-nB; idx = NLABELS; } From 120595ae7b34ea2f0e15eea268d42d6379d5c217 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 4 Jun 2020 16:40:47 -0400 Subject: [PATCH 156/270] try to fix fluxBC for greyscaleSC model but didn't work; save the work --- models/GreyscaleSCModel.cpp | 40 +++++++++++++++++++++++++++++-------- models/GreyscaleSCModel.h | 3 ++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp index f1dc9ab2..6fa125fd 100644 --- a/models/GreyscaleSCModel.cpp +++ b/models/GreyscaleSCModel.cpp @@ -15,7 +15,7 @@ void DeleteArray( const TYPE *p ) ScaLBL_GreyscaleSCModel::ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), -rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),flux(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), +rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),fluxA(0),fluxB(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { SignDist.resize(Nx,Ny,Nz); @@ -50,7 +50,7 @@ void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ dinB=rhoB_minor;//inlet density for fluid B doutA=rhoA_minor;//outlet denisty for fluid A doutB=rhoB;//outlet density for fluid B - flux=0.0; + fluxA=fluxB=0.0; // ---------------------- Greyscale Model parameters -----------------------// if (greyscaleSC_db->keyExists( "timestepMax" )){ @@ -83,8 +83,11 @@ void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ if (greyscaleSC_db->keyExists( "Restart" )){ Restart = greyscaleSC_db->getScalar( "Restart" ); } - if (greyscaleSC_db->keyExists( "flux" )){ - flux = greyscaleSC_db->getScalar( "flux" ); + if (greyscaleSC_db->keyExists( "fluxA" )){ + fluxA = greyscaleSC_db->getScalar( "fluxA" ); + } + if (greyscaleSC_db->keyExists( "fluxB" )){ + fluxB = greyscaleSC_db->getScalar( "fluxB" ); } if (greyscaleSC_db->keyExists( "tolerance" )){ tolerance = greyscaleSC_db->getScalar( "tolerance" ); @@ -188,6 +191,9 @@ void ScaLBL_GreyscaleSCModel::ReadInput(){ case 3: if (rank==0) printf("BoundaryCondition=%i: Constant pressure boundary condition\n",BoundaryCondition); break; + case 4: + if (rank==0) printf("BoundaryCondition=%i: Constant flux boundary condition\n",BoundaryCondition); + break; default: if (rank==0) printf("BoundaryCondition=%i: is currently not supported! Periodic boundary condition is used.\n",BoundaryCondition); BoundaryCondition=0; @@ -912,6 +918,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); } + if (BoundaryCondition == 4){ + dinA = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqA, fluxA, timestep); + dinB = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqB, fluxB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); //if (BoundaryCondition > 0){ @@ -925,7 +936,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenA); ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ + if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) if (Dm->kproc()==0){ ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); } @@ -938,7 +949,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_Comm_Regular->SendHalo(DenB); ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenB); - if (BoundaryCondition >0 ){ + if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) if (Dm->kproc()==0){ ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); } @@ -973,6 +984,11 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); } + if (BoundaryCondition == 4){ + dinA = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqA, fluxA, timestep); + dinB = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqB, fluxB, timestep); + ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); + } ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); //if (BoundaryCondition > 0){ @@ -986,7 +1002,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenA); ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ + if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) if (Dm->kproc()==0){ ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); } @@ -1000,7 +1016,7 @@ void ScaLBL_GreyscaleSCModel::Run(){ ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(DenB); ScaLBL_DeviceBarrier(); - if (BoundaryCondition >0 ){ + if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) if (Dm->kproc()==0){ ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); } @@ -1149,6 +1165,14 @@ void ScaLBL_GreyscaleSCModel::Run(){ // fclose(log_file); // } // } + if (timestep==2&&BoundaryCondition==4){ + if (rank==0) printf(" Timestep dinA dinB doutA doutB\n"); + } + if (timestep%analysis_interval==0){ + if (BoundaryCondition==4){ + if (rank==0) printf(" %i %.3g %.3g %.3g %.3g\n",timestep,dinA,dinB,doutA,doutB); + } + } if (timestep%visualization_interval==0){ WriteOutput(); diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h index 667eca16..e5e79d68 100644 --- a/models/GreyscaleSCModel.h +++ b/models/GreyscaleSCModel.h @@ -42,7 +42,8 @@ public: double rhoA,rhoB; double rhoA_minor,rhoB_minor;//dissolved density double tolerance; - double Fx,Fy,Fz,flux; + double Fx,Fy,Fz; + double fluxA,fluxB; double dinA,doutA; double dinB,doutB; double GreyPorosity; From 2e47e9c306ef361b841c5b31bbfb78083324fc79 Mon Sep 17 00:00:00 2001 From: James McClure Date: Sun, 7 Jun 2020 20:54:09 -0400 Subject: [PATCH 157/270] fix bug in Euler characteristic at domain boundary --- analysis/Minkowski.cpp | 6 ++-- models/ColorModel.cpp | 74 +++++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index faac6142..e98cfdc7 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=1; k 0.01*volume_initial) - target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); + if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); + double target_delta_volume_incremental = target_delta_volume; + if (fabs(target_delta_volume) > 0.01*volume_initial) + target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - //phase(i,j,k) = -1.0; - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } + for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + //phase(i,j,k) = -1.0; + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } + } + } + fillDouble.fill(phase); //} count = 0.f; From f0b150c7c58e42cb5852baf2a6bdb8e6094401e6 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 9 Jun 2020 15:42:45 -0400 Subject: [PATCH 158/270] initialize greyscaleColor model --- models/GreyscaleColorModel.cpp | 1663 ++++++++++++++++++++++++++++++++ models/GreyscaleColorModel.h | 88 ++ 2 files changed, 1751 insertions(+) create mode 100644 models/GreyscaleColorModel.cpp create mode 100644 models/GreyscaleColorModel.h diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp new file mode 100644 index 00000000..3fef03d1 --- /dev/null +++ b/models/GreyscaleColorModel.cpp @@ -0,0 +1,1663 @@ +/* +color lattice boltzmann model + */ +#include "models/ColorModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include "common/Communication.h" +#include "common/ReadMicroCT.h" +#include +#include + +ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + REVERSE_FLOW_DIRECTION = false; +} +ScaLBL_ColorModel::~ScaLBL_ColorModel(){ + +} + +/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) +{ + int q,n; + double value; + ofstream File(FILENAME,ios::binary); + for (n=0; n( filename ); + domain_db = db->getDatabase( "Domain" ); + color_db = db->getDatabase( "Color" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tauA = tauB = 1.0; + rhoA = rhoB = 1.0; + Fx = Fy = Fz = 0.0; + alpha=1e-3; + beta=0.95; + Restart=false; + din=dout=1.0; + flux=0.0; + + // Color Model parameters + if (color_db->keyExists( "timestepMax" )){ + timestepMax = color_db->getScalar( "timestepMax" ); + } + if (color_db->keyExists( "tauA" )){ + tauA = color_db->getScalar( "tauA" ); + } + if (color_db->keyExists( "tauB" )){ + tauB = color_db->getScalar( "tauB" ); + } + if (color_db->keyExists( "rhoA" )){ + rhoA = color_db->getScalar( "rhoA" ); + } + if (color_db->keyExists( "rhoB" )){ + rhoB = color_db->getScalar( "rhoB" ); + } + if (color_db->keyExists( "F" )){ + Fx = color_db->getVector( "F" )[0]; + Fy = color_db->getVector( "F" )[1]; + Fz = color_db->getVector( "F" )[2]; + } + if (color_db->keyExists( "alpha" )){ + alpha = color_db->getScalar( "alpha" ); + } + if (color_db->keyExists( "beta" )){ + beta = color_db->getScalar( "beta" ); + } + if (color_db->keyExists( "Restart" )){ + Restart = color_db->getScalar( "Restart" ); + } + if (color_db->keyExists( "din" )){ + din = color_db->getScalar( "din" ); + } + if (color_db->keyExists( "dout" )){ + dout = color_db->getScalar( "dout" ); + } + if (color_db->keyExists( "flux" )){ + flux = color_db->getScalar( "flux" ); + } + inletA=1.f; + inletB=0.f; + outletA=0.f; + outletB=1.f; + //if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details) + + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + + // Override user-specified boundary condition for specific protocols + auto protocol = color_db->getWithDefault( "protocol", "none" ); + if (protocol == "seed water"){ + if (BoundaryCondition != 0 ){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } + else if (protocol == "open connected oil"){ + if (BoundaryCondition != 0 ){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } + else if (protocol == "shell aggregation"){ + if (BoundaryCondition != 0 ){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } +} + +void ScaLBL_ColorModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object + comm.barrier(); + Dm->CommInit(); + comm.barrier(); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_ColorModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (color_db->keyExists( "image_sequence" )){ + auto ImageList = color_db->getVector( "image_sequence"); + int IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + std::string first_image = ImageList[IMAGE_INDEX]; + Mask->Decomp(first_image); + IMAGE_INDEX++; + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, comm ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0; + } + } + } +// MeanFilter(Averages->SDs); + if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + CalcDist(Averages->SDs,id_solid,*Mask); + + if (rank == 0) cout << "Domain set." << endl; + + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); +} + +void ScaLBL_ColorModel::AssignComponentLabels(double *phase) +{ + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = color_db->getVector( "ComponentLabels" ); + auto AffinityList = color_db->getVector( "ComponentAffinity" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: ComponentLabels and ComponentAffinity must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (size_t idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + // fluid labels are reserved + if (VALUE == 1) AFFINITY=1.0; + else if (VALUE == 2) AFFINITY=-1.0; + phase[n] = AFFINITY; + } + } + } + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (size_t idx=0; idxComm.sumReduce( label_count[idx] ); + + if (rank==0){ + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + comm.barrier(); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nx*Ny*Nz ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; + + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + // initialize phi based on PhaseLabel (include solid component labels) + double *PhaseLabel; + PhaseLabel = new double[N]; + AssignComponentLabels(PhaseLabel); + ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); +} + +/******************************************************** + * AssignComponentLabels * + ********************************************************/ + +void ScaLBL_ColorModel::Initialize(){ + + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + /* + * This function initializes model + */ + if (Restart == true){ + if (rank==0){ + printf("Reading restart file! \n"); + } + + // Read in the restart file to CPU buffers + int *TmpMap; + TmpMap = new int[Np]; + + double *cPhi, *cDist, *cDen; + cPhi = new double[N]; + cDen = new double[2*Np]; + cDist = new double[19*Np]; + ScaLBL_CopyToHost(TmpMap, dvcMap, Np*sizeof(int)); + ScaLBL_CopyToHost(cPhi, Phi, N*sizeof(double)); + + ifstream File(LocalRestartFile,ios::binary); + int idx; + double value,va,vb; + for (int n=0; nLastExterior(); n++){ + va = cDen[n]; + vb = cDen[Np + n]; + value = (va-vb)/(va+vb); + idx = TmpMap[n]; + if (!(idx < 0) && idxFirstInterior(); nLastInterior(); n++){ + va = cDen[n]; + vb = cDen[Np + n]; + value = (va-vb)/(va+vb); + idx = TmpMap[n]; + if (!(idx < 0) && idxLastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); +} + +void ScaLBL_ColorModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + int IMAGE_INDEX = 0; + int IMAGE_COUNT = 0; + std::vector ImageList; + bool SET_CAPILLARY_NUMBER = false; + bool MORPH_ADAPT = false; + bool USE_MORPH = false; + bool USE_SEED = false; + bool USE_DIRECT = false; + bool USE_MORPHOPEN_OIL = false; + int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine + int MIN_STEADY_TIMESTEPS = 100000; + int MAX_STEADY_TIMESTEPS = 200000; + int RAMP_TIMESTEPS = 0;//50000; // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in) + int CURRENT_MORPH_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) + int CURRENT_STEADY_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) + int morph_interval = 100000; + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int morph_timesteps = 0; + double morph_delta = 0.0; + double seed_water = 0.0; + double capillary_number = 0.0; + double tolerance = 0.01; + double Ca_previous = 0.f; + double initial_volume = 0.0; + double delta_volume = 0.0; + double delta_volume_target = 0.0; + double RESIDUAL_ENDPOINT_THRESHOLD = 0.04; + double NOISE_THRESHOLD = 0.0; + double BUMP_RATE = 2.0; + bool USE_BUMP_RATE = false; + + /* history for morphological algoirthm */ + double KRA_MORPH_FACTOR=0.8; + double volA_prev = 0.0; + double log_krA_prev = 1.0; + double log_krA_target = 1.0; + double log_krA = 0.0; + double slope_krA_volume = 0.0; + if (color_db->keyExists( "vol_A_previous" )){ + volA_prev = color_db->getScalar( "vol_A_previous" ); + } + if (color_db->keyExists( "log_krA_previous" )){ + log_krA_prev = color_db->getScalar( "log_krA_previous" ); + } + if (color_db->keyExists( "krA_morph_factor" )){ + KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); + } + + /* defaults for simulation protocols */ + auto protocol = color_db->getWithDefault( "protocol", "none" ); + if (protocol == "image sequence"){ + // Get the list of images + USE_DIRECT = true; + ImageList = color_db->getVector( "image_sequence"); + IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + IMAGE_COUNT = ImageList.size(); + morph_interval = 10000; + USE_MORPH = true; + } + else if (protocol == "seed water"){ + morph_delta = 0.05; + seed_water = 0.01; + USE_SEED = true; + USE_MORPH = true; + } + else if (protocol == "open connected oil"){ + morph_delta = 0.05; + USE_MORPH = true; + USE_MORPHOPEN_OIL = true; + } + else if (protocol == "shell aggregation"){ + morph_delta = 0.05; + USE_MORPH = true; + } + + if (color_db->keyExists( "residual_endpoint_threshold" )){ + RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar( "residual_endpoint_threshold" ); + } + NULL_USE( RESIDUAL_ENDPOINT_THRESHOLD ); + if (color_db->keyExists( "noise_threshold" )){ + NOISE_THRESHOLD = color_db->getScalar( "noise_threshold" ); + USE_BUMP_RATE = true; + } + if (color_db->keyExists( "bump_rate" )){ + BUMP_RATE = color_db->getScalar( "bump_rate" ); + USE_BUMP_RATE = true; + } + if (color_db->keyExists( "capillary_number" )){ + capillary_number = color_db->getScalar( "capillary_number" ); + SET_CAPILLARY_NUMBER=true; + //RESCALE_FORCE_MAX = 1; + } +// if (analysis_db->keyExists( "rescale_force_count" )){ +// RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); +// } + if (color_db->keyExists( "timestep" )){ + timestep = color_db->getScalar( "timestep" ); + } + if (BoundaryCondition != 0 && SET_CAPILLARY_NUMBER==true){ + if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 \n"); + SET_CAPILLARY_NUMBER=false; + } + if (analysis_db->keyExists( "seed_water" )){ + seed_water = analysis_db->getScalar( "seed_water" ); + if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water); + USE_SEED = true; + } + if (analysis_db->keyExists( "morph_delta" )){ + morph_delta = analysis_db->getScalar( "morph_delta" ); + if (rank == 0) printf("Target volume change %f (morph_delta) \n",morph_delta); + } + if (analysis_db->keyExists( "morph_interval" )){ + morph_interval = analysis_db->getScalar( "morph_interval" ); + USE_MORPH = true; + } + if (analysis_db->keyExists( "use_morphopen_oil" )){ + USE_MORPHOPEN_OIL = analysis_db->getScalar( "use_morphopen_oil" ); + if (rank == 0 && USE_MORPHOPEN_OIL) printf("Volume change by morphological opening \n"); + USE_MORPH = true; + } + if (analysis_db->keyExists( "tolerance" )){ + tolerance = analysis_db->getScalar( "tolerance" ); + } + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "min_steady_timesteps" )){ + MIN_STEADY_TIMESTEPS = analysis_db->getScalar( "min_steady_timesteps" ); + } + if (analysis_db->keyExists( "max_steady_timesteps" )){ + MAX_STEADY_TIMESTEPS = analysis_db->getScalar( "max_steady_timesteps" ); + } + if (analysis_db->keyExists( "max_morph_timesteps" )){ + MAX_MORPH_TIMESTEPS = analysis_db->getScalar( "max_morph_timesteps" ); + } + + + if (rank==0){ + printf("********************************************************\n"); + if (protocol == "image sequence"){ + printf(" using protocol = image sequence \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + std::string first_image = ImageList[IMAGE_INDEX]; + printf(" first image in sequence: %s ***\n", first_image.c_str()); + } + else if (protocol == "seed water"){ + printf(" using protocol = seed water \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + printf(" seed_water = %f \n",seed_water); + } + else if (protocol == "open connected oil"){ + printf(" using protocol = open connected oil \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + } + else if (protocol == "shell aggregation"){ + printf(" using protocol = shell aggregation \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + } + printf("No. of timesteps: %i \n", timestepMax); + fflush(stdout); + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + comm.barrier(); + starttime = Utilities::MPI::time(); + //......................................... + + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + //std::shared_ptr analysis_db; + bool Regular = false; + auto current_db = db->cloneDatabase(); + runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); + //analysis.createThreads( analysis_method, 4 ); + while (timestep < timestepMax ) { + //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } + PROFILE_START("Update"); + // *************ODD TIMESTEP************* + timestep++; + // Compute the Phase indicator field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + if (BoundaryCondition > 0){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + // Halo exchange for phase field + ScaLBL_Comm_Regular->SendHalo(Phi); + + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); + comm.barrier(); + + // *************EVEN TIMESTEP************* + timestep++; + // Compute the Phase indicator field + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + // Halo exchange for phase field + if (BoundaryCondition > 0){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); + comm.barrier(); + //************************************************************************ + + PROFILE_STOP("Update"); + + if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ + printf("%i %f \n",timestep,din); + } + // Run the analysis + analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + + + // allow initial ramp-up to get closer to steady state + if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ + analysis.finish(); + CURRENT_STEADY_TIMESTEPS += analysis_interval; + + double volB = Averages->gwb.V; + double volA = Averages->gnb.V; + volA /= Dm->Volume; + volB /= Dm->Volume;; + initial_volume = volA*Dm->Volume; + double vA_x = Averages->gnb.Px/Averages->gnb.M; + double vA_y = Averages->gnb.Py/Averages->gnb.M; + double vA_z = Averages->gnb.Pz/Averages->gnb.M; + double vB_x = Averages->gwb.Px/Averages->gwb.M; + double vB_y = Averages->gwb.Py/Averages->gwb.M; + double vB_z = Averages->gwb.Pz/Averages->gwb.M; + double muA = rhoA*(tauA-0.5)/3.f; + double muB = rhoB*(tauB-0.5)/3.f; + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double current_saturation = volB/(volA+volB); + double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); + double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); + double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); + + if (SET_CAPILLARY_NUMBER && CURRENT_STEADY_TIMESTEPS%MIN_STEADY_TIMESTEPS < analysis_interval ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + + if ( morph_timesteps > morph_interval ){ + + bool isSteady = false; + if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_STEADY_TIMESTEPS > MIN_STEADY_TIMESTEPS)) + isSteady = true; + if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) + isSteady = true; + + if ( isSteady ){ + MORPH_ADAPT = true; + CURRENT_MORPH_TIMESTEPS=0; + //delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + /** morphological target based on relative permeability for A **/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + log_krA = log(krA_TMP); + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); + log_krA_prev = log_krA; + volA_prev = volA; + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + /** compute averages & write data **/ + Averages->Full(); + Averages->Write(timestep); + analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.finish(); + + if (rank==0){ + printf("** WRITE STEADY POINT *** "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + double h = Dm->voxel_length; + // pressures + double pA = Averages->gnb.p; + double pB = Averages->gwb.p; + double pAc = Averages->gnc.p; + double pBc = Averages->gwc.p; + double pAB = (pA-pB)/(h*5.796*alpha); + double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + // connected contribution + double Vol_nc = Averages->gnc.V/Dm->Volume; + double Vol_wc = Averages->gwc.V/Dm->Volume; + double Vol_nd = Averages->gnd.V/Dm->Volume; + double Vol_wd = Averages->gwd.V/Dm->Volume; + double Mass_n = Averages->gnc.M + Averages->gnd.M; + double Mass_w = Averages->gwc.M + Averages->gwd.M; + double vAc_x = Averages->gnc.Px/Mass_n; + double vAc_y = Averages->gnc.Py/Mass_n; + double vAc_z = Averages->gnc.Pz/Mass_n; + double vBc_x = Averages->gwc.Px/Mass_w; + double vBc_y = Averages->gwc.Py/Mass_w; + double vBc_z = Averages->gwc.Pz/Mass_w; + // disconnected contribution + double vAd_x = Averages->gnd.Px/Mass_n; + double vAd_y = Averages->gnd.Py/Mass_n; + double vAd_z = Averages->gnd.Pz/Mass_n; + double vBd_x = Averages->gwd.Px/Mass_w; + double vBd_y = Averages->gwd.Py/Mass_w; + double vBd_z = Averages->gwd.Pz/Mass_w; + + double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); + double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); + double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); + double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); + + double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); + double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); + + double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); + double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + + double kAeff = h*h*muA*(flow_rate_A)/(force_mag); + double kBeff = h*h*muB*(flow_rate_B)/(force_mag); + + double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; + double Mobility = muA/muB; + + bool WriteHeader=false; + FILE * kr_log_file = fopen("relperm.csv","r"); + if (kr_log_file != NULL) + fclose(kr_log_file); + else + WriteHeader=true; + kr_log_file = fopen("relperm.csv","a"); + if (WriteHeader) + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fclose(kr_log_file); + + printf(" Measured capillary number %f \n ",Ca); + } + if (SET_CAPILLARY_NUMBER ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (flow_rate_A < NOISE_THRESHOLD && USE_BUMP_RATE){ + if (rank==0) printf("Hit noise threshold (%f): bumping capillary number by %f X \n",NOISE_THRESHOLD,BUMP_RATE); + Fx *= BUMP_RATE; // impose bump condition + Fy *= BUMP_RATE; + Fz *= BUMP_RATE; + capillary_number *= BUMP_RATE; + color_db->putScalar("capillary_number",capillary_number); + current_db->putDatabase("Color", color_db); + MORPH_ADAPT = false; // re-run current point if below noise threshold + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + + CURRENT_STEADY_TIMESTEPS = 0; + } + else{ + if (rank==0){ + printf("** Continue to simulate steady *** \n "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + } + } + morph_timesteps=0; + Ca_previous = Ca; + } + + if (MORPH_ADAPT ){ + CURRENT_MORPH_TIMESTEPS += analysis_interval; + if (USE_DIRECT){ + // Use image sequence + IMAGE_INDEX++; + MORPH_ADAPT = false; + if (IMAGE_INDEX < IMAGE_COUNT){ + std::string next_image = ImageList[IMAGE_INDEX]; + if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); + color_db->putScalar("image_index",IMAGE_INDEX); + ImageInit(next_image); + } + else{ + if (rank==0) printf("Finished simulating image sequence \n"); + timestep = timestepMax; + } + } + else if (USE_SEED){ + delta_volume = volA*Dm->Volume - initial_volume; + CURRENT_MORPH_TIMESTEPS += analysis_interval; + double massChange = SeedPhaseField(seed_water); + if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); + } + else if (USE_MORPHOPEN_OIL){ + delta_volume = volA*Dm->Volume - initial_volume; + if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); + MorphOpenConnected(delta_volume_target); + } + else { + if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); + //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A + delta_volume += MorphInit(beta,delta_volume_target-delta_volume); + } + + if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + if (USE_DIRECT){ + //BoundaryCondition = 0; + //ScaLBL_Comm->BoundaryCondition = 0; + //ScaLBL_Comm_Regular->BoundaryCondition = 0; + //Fx = capillary_number*dir_x*force_mag / Ca; + //Fy = capillary_number*dir_y*force_mag / Ca; + //Fz = capillary_number*dir_z*force_mag / Ca; + } + } + else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + } + if ( REVERSE_FLOW_DIRECTION ){ + //if (rank==0) printf("*****REVERSE FLOW DIRECTION***** \n"); + delta_volume = 0.0; + // flow direction will reverse after next steady point + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + //morph_delta *= (-1.0); + REVERSE_FLOW_DIRECTION = false; + } + comm.barrier(); + } + morph_timesteps += analysis_interval; + } + } + analysis.finish(); + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_color_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + comm.barrier(); + stoptime = Utilities::MPI::time(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + +double ScaLBL_ColorModel::ImageInit(std::string Filename){ + + if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); + Mask->Decomp(Filename); + for (int i=0; iid[i]; // save what was read + for (int i=0; iid[i] = Mask->id[i]; // save what was read + + double *PhaseLabel; + PhaseLabel = new double[Nx*Ny*Nz]; + AssignComponentLabels(PhaseLabel); + + double Count = 0.0; + double PoreCount = 0.0; + for (int k=1; kComm.sumReduce( Count ); + PoreCount = Dm->Comm.sumReduce( PoreCount ); + + if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); + ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); + comm.barrier(); + + ScaLBL_D3Q19_Init(fq, Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + comm.barrier(); + + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); + + double saturation = Count/PoreCount; + return saturation; + +} + +double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ + + int nx = Nx; + int ny = Ny; + int nz = Nz; + int n; + int N = nx*ny*nz; + double volume_change=0.0; + + if (target_volume_change < 0.0){ + Array id_solid(nx,ny,nz); + Array phase_label(nx,ny,nz); + DoubleArray distance(Nx,Ny,Nz); + DoubleArray phase(nx,ny,nz); + signed char *id_connected; + id_connected = new signed char [nx*ny*nz]; + + ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); + + // Extract only the connected part of NWP + BlobIDstruct new_index; + double vF=0.0; double vS=0.0; + ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); + Dm->Comm.barrier(); + + long long count_connected=0; + long long count_porespace=0; + long long count_water=0; + for (int k=1; k 0){ + count_porespace++; + } + if (id[n] == 2){ + count_water++; + } + } + } + } + count_connected = Dm->Comm.sumReduce( count_connected); + count_porespace = Dm->Comm.sumReduce( count_porespace); + count_water = Dm->Comm.sumReduce( count_water); + + for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } + } + } + + int count_morphopen=0.0; + for (int k=1; kComm.sumReduce( count_morphopen); + volume_change = double(count_morphopen - count_connected); + + if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); + + ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + } + return(volume_change); +} + +double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ + srand(time(NULL)); + double mass_loss =0.f; + double count =0.f; + double *Aq_tmp, *Bq_tmp; + double *Vel_tmp; + + Aq_tmp = new double [7*Np]; + Bq_tmp = new double [7*Np]; + Vel_tmp = new double [3*Np]; + + ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Vel_tmp, Velocity, 7*Np*sizeof(double)); + + //Extract averged velocity + double vx_glb = (Averages->gnb.Px+Averages->gwb.Px)/(Averages->gnb.M+Averages->gwb.M); + double vy_glb = (Averages->gnb.Py+Averages->gwb.Py)/(Averages->gnb.M+Averages->gwb.M); + double vz_glb = (Averages->gnb.Pz+Averages->gwb.Pz)/(Averages->gnb.M+Averages->gwb.M); + double v_mag_glb = sqrt(vx_glb*vx_glb+vy_glb*vy_glb+vz_glb*vz_glb); + + for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ + double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); + double weight = (v_mag_local 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } + + for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ + double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); + double weight = (v_mag_local 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } + + count = Dm->Comm.sumReduce( count ); + mass_loss = Dm->Comm.sumReduce( mass_loss ); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + + // Need to initialize Aq, Bq, Den, Phi directly + //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); + + return(mass_loss); +} + +//double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ +// srand(time(NULL)); +// double mass_loss =0.f; +// double count =0.f; +// double *Aq_tmp, *Bq_tmp; +// +// Aq_tmp = new double [7*Np]; +// Bq_tmp = new double [7*Np]; +// +// ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); +// ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); +// +///* for (int k=1; kSDs(i,j,k) < 0.f){ +// // skip +// } +// else if (phase(i,j,k) > 0.f ){ +// phase(i,j,k) -= random_value*seed_water_in_oil; +// mass_loss += random_value*seed_water_in_oil; +// count++; +// } +// else { +// +// } +// } +// } +// } +// */ +// for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ +// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; +// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; +// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; +// double phase_id = (dA - dB) / (dA + dB); +// if (phase_id > 0.0){ +// Aq_tmp[n] -= 0.3333333333333333*random_value; +// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; +// +// Bq_tmp[n] += 0.3333333333333333*random_value; +// Bq_tmp[n+Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; +// } +// mass_loss += random_value*seed_water_in_oil; +// } +// +// for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ +// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; +// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; +// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; +// double phase_id = (dA - dB) / (dA + dB); +// if (phase_id > 0.0){ +// Aq_tmp[n] -= 0.3333333333333333*random_value; +// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; +// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; +// +// Bq_tmp[n] += 0.3333333333333333*random_value; +// Bq_tmp[n+Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; +// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; +// } +// mass_loss += random_value*seed_water_in_oil; +// } +// +// count = Dm->Comm.sumReduce( count ); +// mass_loss = Dm->Comm.sumReduce( mass_loss ); +// if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); +// +// // Need to initialize Aq, Bq, Den, Phi directly +// //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); +// ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); +// +// return(mass_loss); +//} + +double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + double vF = 0.f; + double vS = 0.f; + double delta_volume; + + DoubleArray phase(Nx,Ny,Nz); + IntArray phase_label(Nx,Ny,Nz);; + DoubleArray phase_distance(Nx,Ny,Nz); + Array phase_id(Nx,Ny,Nz); + fillHalo fillDouble(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); + + + // Basic algorithm to + // 1. Copy phase field to CPU + ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); + + double count = 0.f; + for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f) count+=1.f; + } + } + } + double volume_initial = Dm->Comm.sumReduce( count); + /* + sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); + FILE *INPUT = fopen(LocalRankFilename,"wb"); + fwrite(phase.data(),8,N,INPUT); + fclose(INPUT); + */ + // 2. Identify connected components of phase field -> phase_label + BlobIDstruct new_index; + ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); + comm.barrier(); + + // only operate on component "0" + count = 0.0; + double second_biggest = 0.0; + + for (int k=0; kComm.sumReduce( count ); + second_biggest = Dm->Comm.sumReduce( second_biggest ); + + /*int reach_x, reach_y, reach_z; + for (int k=0; k phase_distance + CalcDist(phase_distance,phase_id,*Dm); + + double temp,value; + double factor=0.5/beta; + for (int k=0; k 1.f) value=1.f; + if (value < -1.f) value=-1.f; + // temp -- distance based on analytical form McClure, Prins et al, Comp. Phys. Comm. + temp = -factor*log((1.0+value)/(1.0-value)); + /// use this approximation close to the object + if (fabs(value) < 0.8 && Averages->SDs(i,j,k) > 1.f ){ + phase_distance(i,j,k) = temp; + } + // erase the original object + phase(i,j,k) = -1.0; + } + } + } + } + + if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){ + // if connected volume is less than 2% just delete the whole thing + if (rank==0) printf("Connected region has shrunk! \n"); + REVERSE_FLOW_DIRECTION = true; + } +/* else{*/ + if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest ); + if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); + double target_delta_volume_incremental = target_delta_volume; + if (fabs(target_delta_volume) > 0.01*volume_initial) + target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental); + + for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + //phase(i,j,k) = -1.0; + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } + } + } + fillDouble.fill(phase); + //} + + count = 0.f; + for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f){ + count+=1.f; + } + } + } + } + double volume_final = Dm->Comm.sumReduce( count ); + + delta_volume = (volume_final-volume_initial); + if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); + if (rank == 0) printf(" new saturation = %f \n", volume_final/(0.238323*double((Nx-2)*(Ny-2)*(Nz-2)*nprocs))); + + // 6. copy back to the device + //if (rank==0) printf("MorphInit: copy data back to device\n"); + ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); + /* + sprintf(LocalRankFilename,"dist_final.%05i.raw",rank); + FILE *DIST = fopen(LocalRankFilename,"wb"); + fwrite(phase_distance.data(),8,N,DIST); + fclose(DIST); + + sprintf(LocalRankFilename,"phi_final.%05i.raw",rank); + FILE *PHI = fopen(LocalRankFilename,"wb"); + fwrite(phase.data(),8,N,PHI); + fclose(PHI); + */ + // 7. Re-initialize phase field and density + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (BoundaryCondition >0 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + return delta_volume; +} + +void ScaLBL_ColorModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), Phi, sizeof(double)*N); + + FILE *OUTFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); +// FILE *CGX_FILE; +// sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); +// CGX_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGX_FILE); +// fclose(CGX_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); +// FILE *CGY_FILE; +// sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); +// CGY_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGY_FILE); +// fclose(CGY_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); +// FILE *CGZ_FILE; +// sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); +// CGZ_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGZ_FILE); +// fclose(CGZ_FILE); + +} diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h new file mode 100644 index 00000000..c52f04c3 --- /dev/null +++ b/models/GreyscaleColorModel.h @@ -0,0 +1,88 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "analysis/TwoPhase.h" +#include "analysis/runAnalysis.h" +#include "common/MPI.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_ColorModel{ +public: + ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM); + ~ScaLBL_ColorModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + + bool Restart,pBC; + bool REVERSE_FLOW_DIRECTION; + int timestep,timestepMax; + int BoundaryCondition; + double tauA,tauB,rhoA,rhoB,alpha,beta; + double Fx,Fy,Fz,flux; + double din,dout,inletA,inletB,outletA,outletB; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + std::shared_ptr ScaLBL_Comm_Regular; + //std::shared_ptr Averages; + std::shared_ptr Averages; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr color_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + IntArray Map; + signed char *id; + int *NeighborList; + int *dvcMap; + double *fq, *Aq, *Bq; + double *Den, *Phi; + double *ColorGrad; + double *Velocity; + double *Pressure; + +private: + Utilities::MPI comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); + void AssignComponentLabels(double *phase); + double ImageInit(std::string filename); + double MorphInit(const double beta, const double morph_delta); + double SeedPhaseField(const double seed_water_in_oil); + double MorphOpenConnected(double target_volume_change); +}; + From 2621a7718f93e1f6211497ba5b919ee05f78a782 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 15 Jun 2020 22:41:01 -0400 Subject: [PATCH 159/270] Greyscale Color model; both CPU and GPU versions are ready --- common/ScaLBL.cpp | 4 +- common/ScaLBL.h | 20 +- cpu/Greyscale.cpp | 2 +- cpu/GreyscaleColor.cpp | 1366 +++++++++ cpu/GreyscaleFE.cpp | 3113 +++++++++++++++++++ cpu/GreyscaleSC.cpp | 3625 +++++++++++++++++++++++ gpu/GreyscaleColor.cu | 1429 +++++++++ gpu/GreyscaleSC.cu | 12 +- models/GreyscaleColorModel.cpp | 663 ++++- models/GreyscaleColorModel.h | 20 +- tests/CMakeLists.txt | 1 + tests/lbpm_greyscaleColor_simulator.cpp | 58 + 12 files changed, 10152 insertions(+), 161 deletions(-) create mode 100644 cpu/GreyscaleColor.cpp create mode 100644 cpu/GreyscaleFE.cpp create mode 100644 cpu/GreyscaleSC.cpp create mode 100644 gpu/GreyscaleColor.cu create mode 100644 tests/lbpm_greyscaleColor_simulator.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index c8feffbe..8aee6f33 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2079,7 +2079,7 @@ void ScaLBL_Communicator::GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, { if (kproc == 0) { // Set the density field on the z inlet - ScaLBL_GreyscaleSC_BC_z(dvcSendList_z, Map, DenA, DenB, vA, vB, sendCount_z, N); + ScaLBL_GreyscaleSC_BC_z(dvcSendList_z, Map, DenA, DenB, vA, vB, sendCount_z); } } @@ -2087,7 +2087,7 @@ void ScaLBL_Communicator::GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, { if (kproc == nprocz-1){ // Set the density field on the Z outlet - ScaLBL_GreyscaleSC_BC_Z(dvcSendList_Z, Map, DenA, DenB, vA, vB, sendCount_Z, N); + ScaLBL_GreyscaleSC_BC_Z(dvcSendList_Z, Map, DenA, DenB, vA, vB, sendCount_Z); } } diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 0f31224b..a74c54a2 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -146,9 +146,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, dou extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np); -extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); +extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); -extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); +extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); @@ -158,6 +158,19 @@ extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *l extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); +// GREYSCALE COLOR MODEL (Two-component) +//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np); + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, + double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, + double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); @@ -224,9 +237,6 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice); -extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); - -extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np); class ScaLBL_Communicator{ diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index 8fa54275..affe6c2b 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1436,7 +1436,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis //-------------------- IMRT collison where body force has NO higher-order terms -------------// //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*Den+19*Den(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m1 = m1 + rlx_setA*((-30*Den+19*Den*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); m2 = m2 + rlx_setA*((12*Den - 5.5*Den*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); jx = jx + Fx; m4 = m4 + rlx_setB*((-0.6666666666666666*ux*Den) - m4) diff --git a/cpu/GreyscaleColor.cpp b/cpu/GreyscaleColor.cpp new file mode 100644 index 00000000..48f84e09 --- /dev/null +++ b/cpu/GreyscaleColor.cpp @@ -0,0 +1,1366 @@ +#include + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Velocity, + double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, + double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ + + int n,nn,ijk,nread; + int nr1,nr2,nr3,nr4,nr5,nr6; + int nr7,nr8,nr9,nr10; + int nr11,nr12,nr13,nr14; + //int nr15,nr16,nr17,nr18; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double phi,tau,rho0,rlx_setA,rlx_setB; + + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double tau_eff; + double mu_eff;//kinematic viscosity + double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + for (n=start; n even part of dist) + //fq = dist[nread]; // reading the f2 data into register fq + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + //nread = neighborList[n+2*Np]; // neighbor 4 + //fq = dist[nread]; + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + //nread = neighborList[n+3*Np]; // neighbor 3 + //fq = dist[nread]; + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + //nread = neighborList[n+4*Np]; + //fq = dist[nread]; + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + //nread = neighborList[n+5*Np]; + //fq = dist[nread]; + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + //nread = neighborList[n+6*Np]; + //fq = dist[nread]; + nr7 = neighborList[n+6*Np]; + fq = dist[nr7]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + //nread = neighborList[n+7*Np]; + //fq = dist[nread]; + nr8 = neighborList[n+7*Np]; + fq = dist[nr8]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + //nread = neighborList[n+8*Np]; + //fq = dist[nread]; + nr9 = neighborList[n+8*Np]; + fq = dist[nr9]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + //nread = neighborList[n+9*Np]; + //fq = dist[nread]; + nr10 = neighborList[n+9*Np]; + fq = dist[nr10]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + //nread = neighborList[n+10*Np]; + //fq = dist[nread]; + nr11 = neighborList[n+10*Np]; + fq = dist[nr11]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + //nread = neighborList[n+11*Np]; + //fq = dist[nread]; + nr12 = neighborList[n+11*Np]; + fq = dist[nr12]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + //nread = neighborList[n+12*Np]; + //fq = dist[nread]; + nr13 = neighborList[n+12*Np]; + fq = dist[nr13]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + //nread = neighborList[n+13*Np]; + //fq = dist[nread]; + nr14 = neighborList[n+13*Np]; + fq = dist[nr14]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + + // Compute greyscale related parameters + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx); + vy = jy/rho0+0.5*(porosity*Gy); + vz = jz/rho0+0.5*(porosity*Gz); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=rho0*(Gx); + Fy=rho0*(Gy); + Fz=rho0*(Gz); + } + + // write the velocity + Velocity[n] = ux; + Velocity[Np+n] = uy; + Velocity[2*Np+n] = uz; + + //........................................................................ + //..............carry out relaxation process.............................. + //..........Toelke, Fruediger et. al. 2006................................ + if (C == 0.0) nx = ny = nz = 0.0; + m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); + m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); + m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); + m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); + m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + //nread = neighborList[n+7*Np]; + dist[nr8] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + //nread = neighborList[n+6*Np]; + dist[nr7] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + //nread = neighborList[n+9*Np]; + dist[nr10] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + //nread = neighborList[n+8*Np]; + dist[nr9] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + //nread = neighborList[n+11*Np]; + dist[nr12] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + //nread = neighborList[n+10*Np]; + dist[nr11]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + //nread = neighborList[n+13*Np]; + dist[nr14] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + //nread = neighborList[n+12*Np]; + dist[nr13] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Velocity, + double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, + double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ + + int ijk,nn,n; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double phi,tau,rho0,rlx_setA,rlx_setB; + + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double tau_eff; + double mu_eff;//kinematic viscosity + double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + for (n=start; n0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + } +} + + +//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np){ +// int n; +// double porosity; +// for (n=0; n +#include + +extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Poros,double *Velocity, + double *Pressure, double rhoA,double rhoB, int N){ + + int n; + double ux,uy,uz,u_mag; + double pressure; + double porosity; + double rho0; + double phi; + double nA,nB; + + for (n=0; n0.0)-Gsc*nB*nA_gradx*int(phi<0.0); +// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); +// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); + Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); + Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + dist[1*Np+n] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + dist[2*Np+n] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[3*Np+n] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[4*Np+n] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[5*Np+n] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[6*Np+n] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + dist[7*Np+n] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + dist[8*Np+n] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + dist[9*Np+n] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + dist[10*Np+n] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + dist[11*Np+n] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + dist[12*Np+n] = fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + dist[13*Np+n] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + dist[14*Np+n] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + dist[15*Np+n] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + dist[16*Np+n] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + dist[17*Np+n] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + dist[18*Np+n] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // Calculate the color gradient + nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); + ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); + nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + double ColorMag = C; + if (C==0.0) ColorMag=1.0; + nx = nx/ColorMag; + ny = ny/ColorMag; + nz = nz/ColorMag; + if (C == 0.0) nx = ny = nz = 0.0; + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, + double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure){ + + int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho0; + double phi; + double nx,ny,nz,C; + double nA,nB; + double a1,b1,a2,b2,nAB,delta; + double beta=0.95; + double nA_gradx,nA_grady,nA_gradz; + double nB_gradx,nB_grady,nB_gradz; + double Gff_x,Gff_y,Gff_z; + double Gfs_x,Gfs_y,Gfs_z; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + for (n=start; n 10Np => odd part of dist) + fq = dist[nr1]; // reading the f1 data into register fq + pressure = fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + pressure += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + pressure += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + pressure += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// + // fluid-fluid force +// Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); +// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); +// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); + Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); + Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); + Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + porosity = Poros[n]; + // use local saturation as an estimation of effective relperm values + perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + +// //..............carry out relaxation process............................................... +// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) +// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; +// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) +// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) +// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; +// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) +// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; +// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) +// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; +// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) +// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; +// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) +// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; +// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) +// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; +// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) +// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// //....................................................................................................... + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); + m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); + m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); + m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); + m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); + m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // Calculate the color gradient + nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); + ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); + nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); + //...........Normalize the Color Gradient................................. + C = sqrt(nx*nx+ny*ny+nz*nz); + double ColorMag = C; + if (C==0.0) ColorMag=1.0; + nx = nx/ColorMag; + ny = ny/ColorMag; + nz = nz/ColorMag; + if (C == 0.0) nx = ny = nz = 0.0; + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ + + int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho,rho0; + double phi; + double phi_lap;//laplacian of phase field + double nA,nB; + double Gfs_x,Gfs_y,Gfs_z; + double Gff_x,Gff_y,Gff_z; + double chem; + double rlx_phi; + double a1,a2;//PDF of phase field + // *---------------------------------Pressure Tensor Gradient------------------------------------*// + double Pxx_x,Pyy_y,Pzz_z; + double Pxy_x,Pxy_y; + double Pyz_y,Pyz_z; + double Pxz_x,Pxz_z; + double px,py,pz; //pressure gradient + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + for (n=start; n0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + //Load pressure gradient + px=PressureGrad[0*Np+n]; + py=PressureGrad[1*Np+n]; + pz=PressureGrad[2*Np+n]; + + //Load pressure tensor gradient + //For reference full list of PressTensorGrad + //PressTensorGrad[n+0*Np] = Pxx_x + //PressTensorGrad[n+1*Np] = Pxx_y + //PressTensorGrad[n+2*Np] = Pxx_z + //PressTensorGrad[n+3*Np] = Pyy_x + //PressTensorGrad[n+4*Np] = Pyy_y + //PressTensorGrad[n+5*Np] = Pyy_z + //PressTensorGrad[n+6*Np] = Pzz_x + //PressTensorGrad[n+7*Np] = Pzz_y + //PressTensorGrad[n+8*Np] = Pzz_z + //PressTensorGrad[n+9*Np] = Pxy_x + //PressTensorGrad[n+10*Np] = Pxy_y + //PressTensorGrad[n+11*Np] = Pxy_z + //PressTensorGrad[n+12*Np] = Pyz_x + //PressTensorGrad[n+13*Np] = Pyz_y + //PressTensorGrad[n+14*Np] = Pyz_z + //PressTensorGrad[n+15*Np] = Pxz_x + //PressTensorGrad[n+16*Np] = Pxz_y + //PressTensorGrad[n+17*Np] = Pxz_z + Pxx_x = PressTensorGrad[0*Np+n]; + Pyy_y = PressTensorGrad[4*Np+n]; + Pzz_z = PressTensorGrad[8*Np+n]; + Pxy_x = PressTensorGrad[9*Np+n]; + Pxz_x = PressTensorGrad[15*Np+n]; + Pxy_y = PressTensorGrad[10*Np+n]; + Pyz_y = PressTensorGrad[13*Np+n]; + Pyz_z = PressTensorGrad[14*Np+n]; + Pxz_z = PressTensorGrad[17*Np+n]; + //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... + //TODO double check if you need porosity as a fre-factor + Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); + Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); + Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + // local density + rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); + // local relaxation time + tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); + rlx_setA = 1.f/tau; + rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); + mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity + + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + // q=0 + fq = dist[n]; + rho = fq; + m1 = -30.0*fq; + m2 = 12.0*fq; + + // q=1 + nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist) + fq = dist[nr1]; // reading the f1 data into register fq + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + pressure=rho/3.0; + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); + m10 = m10 + rlx_setA*( - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*( - m12); + m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // calcuale chemical potential + chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; + //rlx_phi = 3.f-sqrt(3.f); + rlx_phi = 1.0; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + //a1 = Cq[nr2]; + //a2 = Cq[nr1]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); + + // q = 1 + //nread = neighborList[n+Np]; + Cq[nr2] = a1; + // q=2 + //nread = neighborList[n]; + Cq[nr1] = a2; + + //............................................... + // Cq = {0,1,0} + //a1 = Cq[nr4]; + //a2 = Cq[nr3]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); + + // q = 3 + //nread = neighborList[n+3*Np]; + Cq[nr4] = a1; + // q = 4 + //nread = neighborList[n+2*Np]; + Cq[nr3] = a2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + //a1 = Cq[nr6]; + //a2 = Cq[nr5]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); + + // q = 5 + //nread = neighborList[n+5*Np]; + Cq[nr6] = a1; + // q = 6 + //nread = neighborList[n+4*Np]; + Cq[nr5] = a2; + //............................................... + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + //a1=Cq[n]; + //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); + Cq[n] = phi-3.0*gamma*chem; + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, + double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, + double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + // currently disable 'GeoFun' + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double tau,tau_eff,rlx_setA,rlx_setB; + double mu_eff;//effective kinematic viscosity for Darcy term + double rho,rho0; + double phi; + double phi_lap;//laplacian of phase field + double nA,nB; + double Gfs_x,Gfs_y,Gfs_z; + double Gff_x,Gff_y,Gff_z; + double chem; + double rlx_phi; + double a1,a2;//PDF of phase field + // *---------------------------------Pressure Tensor Gradient------------------------------------*// + double Pxx_x,Pyy_y,Pzz_z; + double Pxy_x,Pxy_y; + double Pyz_y,Pyz_z; + double Pxz_x,Pxz_z; + double px,py,pz; //pressure gradient + + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + for (n=start; n0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); + + //Load pressure gradient + px=PressureGrad[0*Np+n]; + py=PressureGrad[1*Np+n]; + pz=PressureGrad[2*Np+n]; + + //Load pressure tensor gradient + //For reference full list of PressTensorGrad + //PressTensorGrad[n+0*Np] = Pxx_x + //PressTensorGrad[n+1*Np] = Pxx_y + //PressTensorGrad[n+2*Np] = Pxx_z + //PressTensorGrad[n+3*Np] = Pyy_x + //PressTensorGrad[n+4*Np] = Pyy_y + //PressTensorGrad[n+5*Np] = Pyy_z + //PressTensorGrad[n+6*Np] = Pzz_x + //PressTensorGrad[n+7*Np] = Pzz_y + //PressTensorGrad[n+8*Np] = Pzz_z + //PressTensorGrad[n+9*Np] = Pxy_x + //PressTensorGrad[n+10*Np] = Pxy_y + //PressTensorGrad[n+11*Np] = Pxy_z + //PressTensorGrad[n+12*Np] = Pyz_x + //PressTensorGrad[n+13*Np] = Pyz_y + //PressTensorGrad[n+14*Np] = Pyz_z + //PressTensorGrad[n+15*Np] = Pxz_x + //PressTensorGrad[n+16*Np] = Pxz_y + //PressTensorGrad[n+17*Np] = Pxz_z + Pxx_x = PressTensorGrad[0*Np+n]; + Pyy_y = PressTensorGrad[4*Np+n]; + Pzz_z = PressTensorGrad[8*Np+n]; + Pxy_x = PressTensorGrad[9*Np+n]; + Pxz_x = PressTensorGrad[15*Np+n]; + Pxy_y = PressTensorGrad[10*Np+n]; + Pyz_y = PressTensorGrad[13*Np+n]; + Pyz_z = PressTensorGrad[14*Np+n]; + Pxz_z = PressTensorGrad[17*Np+n]; + //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... + //TODO double check if you need porosity as a fre-factor + Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); + Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); + Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); + // fluid-solid force + Gfs_x = (nA-nB)*SolidForce[n+0*Np]; + Gfs_y = (nA-nB)*SolidForce[n+1*Np]; + Gfs_z = (nA-nB)*SolidForce[n+2*Np]; + + // local density + rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); + // local relaxation time + tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); + rlx_setA = 1.f/tau; + rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); + mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity + + + //........................................................................ + // READ THE DISTRIBUTIONS + // (read from opposite array due to previous swap operation) + //........................................................................ + // q=0 + fq = dist[n]; + rho = fq; + m1 = -30.0*fq; + m2 = 12.0*fq; + + // q=1 + fq = dist[2*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // f2 = dist[10*Np+n]; + fq = dist[1*Np+n]; + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + fq = dist[4*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + fq = dist[3*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + fq = dist[6*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q = 6 + fq = dist[5*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + fq = dist[7*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + fq = dist[10*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + fq = dist[12*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + fq = dist[11*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + fq = dist[14*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + fq = dist[13*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + fq = dist[16*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + fq = dist[15*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + fq = dist[18*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); + vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); + vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); + if (porosity==1.0){ + Fx=rho0*(Gx + Gff_x + Gfs_x); + Fy=rho0*(Gy + Gff_y + Gfs_y); + Fz=rho0*(Gz + Gff_z + Gfs_z); + } + + //Calculate pressure for Incompressible-MRT model + //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); + pressure=rho/3.0; + + //-------------------- IMRT collison where body force has NO higher-order terms -------------// + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); + m10 = m10 + rlx_setA*( - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*( - m12); + m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + dist[1*Np+n] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + dist[2*Np+n] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[3*Np+n] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + dist[4*Np+n] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[5*Np+n] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + dist[6*Np+n] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + dist[7*Np+n] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); + dist[8*Np+n] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + dist[9*Np+n] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + dist[10*Np+n] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); + dist[11*Np+n] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + dist[12*Np+n] = fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); + dist[13*Np+n] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); + dist[14*Np+n] = fq; + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + dist[15*Np+n] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + dist[16*Np+n] = fq; + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + dist[17*Np+n] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + dist[18*Np+n] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + + //-----------------------Mass transport------------------------// + // calcuale chemical potential + chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; + //rlx_phi = 3.f-sqrt(3.f); + rlx_phi = 1.0; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + //a1 = Cq[1*Np+n]; + //a2 = Cq[2*Np+n]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); + + Cq[1*Np+n] = a1; + Cq[2*Np+n] = a2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + //a1 = Cq[3*Np+n]; + //a2 = Cq[4*Np+n]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); + + Cq[3*Np+n] = a1; + Cq[4*Np+n] = a2; + //............................................... + // q = 4 + // Cq = {0,0,1} + //a1 = Cq[5*Np+n]; + //a2 = Cq[6*Np+n]; + //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); + //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); + a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); + a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); + + Cq[5*Np+n] = a1; + Cq[6*Np+n] = a2; + //............................................... + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + //a1=Cq[n]; + //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); + Cq[n] = phi-3.0*gamma*chem; + } +} + +extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ + int n; + double phi; + double nA,nB; + double Den0; + for (n=0; n +#include + +//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList,int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np){ +// +// int ijk; +// int n, nread; +// double vx,vy,vz,v_mag; +// double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; +// double ux,uy,uz; +// // conserved momemnts +// double jxA,jyA,jzA; +// double jxB,jyB,jzB; +// double rhoA,rhoB; +// double nA,nB; +// // non-conserved moments +// double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; +// double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; +// double fq; +// //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double permA,permB;//effective relative perm +// double c0, c1; //Guo's model parameters +// double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity +// double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity +// double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) +// double FxB, FyB, FzB; +// double rlx_setA,rlx_setB; +// double nA_gradx,nA_grady,nA_gradz; +// double nB_gradx,nB_grady,nB_gradz; +// double GffA_x,GffA_y,GffA_z; +// double GfsA_x,GfsA_y,GfsA_z; +// double GffB_x,GffB_y,GffB_z; +// double GfsB_x,GfsB_y,GfsB_z; +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s even part of dist) +// fq = distA[nread]; // reading the f2 data into register fq +// rhoA += fq; +// m1A -= 11.0*(fq); +// m2A -= 4.0*(fq); +// jxA -= fq; +// m4A += 4.0*(fq); +// m9A += 2.0*(fq); +// m10A -= 4.0*(fq); +// +// // q=3 +// nread = neighborList[n+2*Np]; // neighbor 4 +// fq = distA[nread]; +// rhoA += fq; +// m1A -= 11.0*fq; +// m2A -= 4.0*fq; +// jyA = fq; +// m6A = -4.0*fq; +// m9A -= fq; +// m10A += 2.0*fq; +// m11A = fq; +// m12A = -2.0*fq; +// +// // q = 4 +// nread = neighborList[n+3*Np]; // neighbor 3 +// fq = distA[nread]; +// rhoA += fq; +// m1A -= 11.0*fq; +// m2A -= 4.0*fq; +// jyA -= fq; +// m6A += 4.0*fq; +// m9A -= fq; +// m10A += 2.0*fq; +// m11A += fq; +// m12A -= 2.0*fq; +// +// // q=5 +// nread = neighborList[n+4*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A -= 11.0*fq; +// m2A -= 4.0*fq; +// jzA = fq; +// m8A = -4.0*fq; +// m9A -= fq; +// m10A += 2.0*fq; +// m11A -= fq; +// m12A += 2.0*fq; +// +// +// // q = 6 +// nread = neighborList[n+5*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A -= 11.0*fq; +// m2A -= 4.0*fq; +// jzA -= fq; +// m8A += 4.0*fq; +// m9A -= fq; +// m10A += 2.0*fq; +// m11A -= fq; +// m12A += 2.0*fq; +// +// // q=7 +// nread = neighborList[n+6*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA += fq; +// m4A += fq; +// jyA += fq; +// m6A += fq; +// m9A += fq; +// m10A += fq; +// m11A += fq; +// m12A += fq; +// m13A = fq; +// m16A = fq; +// m17A = -fq; +// +// // q = 8 +// nread = neighborList[n+7*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA -= fq; +// m4A -= fq; +// jyA -= fq; +// m6A -= fq; +// m9A += fq; +// m10A += fq; +// m11A += fq; +// m12A += fq; +// m13A += fq; +// m16A -= fq; +// m17A += fq; +// +// // q=9 +// nread = neighborList[n+8*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA += fq; +// m4A += fq; +// jyA -= fq; +// m6A -= fq; +// m9A += fq; +// m10A += fq; +// m11A += fq; +// m12A += fq; +// m13A -= fq; +// m16A += fq; +// m17A += fq; +// +// // q = 10 +// nread = neighborList[n+9*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA -= fq; +// m4A -= fq; +// jyA += fq; +// m6A += fq; +// m9A += fq; +// m10A += fq; +// m11A += fq; +// m12A += fq; +// m13A -= fq; +// m16A -= fq; +// m17A -= fq; +// +// // q=11 +// nread = neighborList[n+10*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA += fq; +// m4A += fq; +// jzA += fq; +// m8A += fq; +// m9A += fq; +// m10A += fq; +// m11A -= fq; +// m12A -= fq; +// m15A = fq; +// m16A -= fq; +// m18A = fq; +// +// // q=12 +// nread = neighborList[n+11*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA -= fq; +// m4A -= fq; +// jzA -= fq; +// m8A -= fq; +// m9A += fq; +// m10A += fq; +// m11A -= fq; +// m12A -= fq; +// m15A += fq; +// m16A += fq; +// m18A -= fq; +// +// // q=13 +// nread = neighborList[n+12*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA += fq; +// m4A += fq; +// jzA -= fq; +// m8A -= fq; +// m9A += fq; +// m10A += fq; +// m11A -= fq; +// m12A -= fq; +// m15A -= fq; +// m16A -= fq; +// m18A -= fq; +// +// // q=14 +// nread = neighborList[n+13*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jxA -= fq; +// m4A -= fq; +// jzA += fq; +// m8A += fq; +// m9A += fq; +// m10A += fq; +// m11A -= fq; +// m12A -= fq; +// m15A -= fq; +// m16A += fq; +// m18A += fq; +// +// // q=15 +// nread = neighborList[n+14*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jyA += fq; +// m6A += fq; +// jzA += fq; +// m8A += fq; +// m9A -= 2.0*fq; +// m10A -= 2.0*fq; +// m14A = fq; +// m17A += fq; +// m18A -= fq; +// +// // q=16 +// nread = neighborList[n+15*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jyA -= fq; +// m6A -= fq; +// jzA -= fq; +// m8A -= fq; +// m9A -= 2.0*fq; +// m10A -= 2.0*fq; +// m14A += fq; +// m17A -= fq; +// m18A += fq; +// +// // q=17 +// nread = neighborList[n+16*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jyA += fq; +// m6A += fq; +// jzA -= fq; +// m8A -= fq; +// m9A -= 2.0*fq; +// m10A -= 2.0*fq; +// m14A -= fq; +// m17A += fq; +// m18A += fq; +// +// // q=18 +// nread = neighborList[n+17*Np]; +// fq = distA[nread]; +// rhoA += fq; +// m1A += 8.0*fq; +// m2A += fq; +// jyA -= fq; +// m6A -= fq; +// jzA += fq; +// m8A += fq; +// m9A -= 2.0*fq; +// m10A -= 2.0*fq; +// m14A -= fq; +// m17A -= fq; +// m18A -= fq; +// //---------------------------------------------------------------------// +// +// // ------------------- Fluid component B ---------------------------------// +// //........................................................................ +// // READ THE DISTRIBUTIONS +// // (read from opposite array due to previous swap operation) +// //........................................................................ +// // q=0 +// fq = distB[n]; +// rhoB = fq; +// m1B = -30.0*fq; +// m2B = 12.0*fq; +// +// // q=1 +// nread = neighborList[n]; // neighbor 2 +// fq = distB[nread]; // reading the f1 data into register fq +// rhoB += fq; +// m1B -= 11.0*fq; +// m2B -= 4.0*fq; +// jxB = fq; +// m4B = -4.0*fq; +// m9B = 2.0*fq; +// m10B = -4.0*fq; +// +// // q=2 +// nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) +// fq = distB[nread]; // reading the f2 data into register fq +// rhoB += fq; +// m1B -= 11.0*(fq); +// m2B -= 4.0*(fq); +// jxB -= fq; +// m4B += 4.0*(fq); +// m9B += 2.0*(fq); +// m10B -= 4.0*(fq); +// +// // q=3 +// nread = neighborList[n+2*Np]; // neighbor 4 +// fq = distB[nread]; +// rhoB += fq; +// m1B -= 11.0*fq; +// m2B -= 4.0*fq; +// jyB = fq; +// m6B = -4.0*fq; +// m9B -= fq; +// m10B += 2.0*fq; +// m11B = fq; +// m12B = -2.0*fq; +// +// // q = 4 +// nread = neighborList[n+3*Np]; // neighbor 3 +// fq = distB[nread]; +// rhoB += fq; +// m1B -= 11.0*fq; +// m2B -= 4.0*fq; +// jyB -= fq; +// m6B += 4.0*fq; +// m9B -= fq; +// m10B += 2.0*fq; +// m11B += fq; +// m12B -= 2.0*fq; +// +// // q=5 +// nread = neighborList[n+4*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B -= 11.0*fq; +// m2B -= 4.0*fq; +// jzB = fq; +// m8B = -4.0*fq; +// m9B -= fq; +// m10B += 2.0*fq; +// m11B -= fq; +// m12B += 2.0*fq; +// +// +// // q = 6 +// nread = neighborList[n+5*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B -= 11.0*fq; +// m2B -= 4.0*fq; +// jzB -= fq; +// m8B += 4.0*fq; +// m9B -= fq; +// m10B += 2.0*fq; +// m11B -= fq; +// m12B += 2.0*fq; +// +// // q=7 +// nread = neighborList[n+6*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB += fq; +// m4B += fq; +// jyB += fq; +// m6B += fq; +// m9B += fq; +// m10B += fq; +// m11B += fq; +// m12B += fq; +// m13B = fq; +// m16B = fq; +// m17B = -fq; +// +// // q = 8 +// nread = neighborList[n+7*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB -= fq; +// m4B -= fq; +// jyB -= fq; +// m6B -= fq; +// m9B += fq; +// m10B += fq; +// m11B += fq; +// m12B += fq; +// m13B += fq; +// m16B -= fq; +// m17B += fq; +// +// // q=9 +// nread = neighborList[n+8*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB += fq; +// m4B += fq; +// jyB -= fq; +// m6B -= fq; +// m9B += fq; +// m10B += fq; +// m11B += fq; +// m12B += fq; +// m13B -= fq; +// m16B += fq; +// m17B += fq; +// +// // q = 10 +// nread = neighborList[n+9*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB -= fq; +// m4B -= fq; +// jyB += fq; +// m6B += fq; +// m9B += fq; +// m10B += fq; +// m11B += fq; +// m12B += fq; +// m13B -= fq; +// m16B -= fq; +// m17B -= fq; +// +// // q=11 +// nread = neighborList[n+10*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB += fq; +// m4B += fq; +// jzB += fq; +// m8B += fq; +// m9B += fq; +// m10B += fq; +// m11B -= fq; +// m12B -= fq; +// m15B = fq; +// m16B -= fq; +// m18B = fq; +// +// // q=12 +// nread = neighborList[n+11*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB -= fq; +// m4B -= fq; +// jzB -= fq; +// m8B -= fq; +// m9B += fq; +// m10B += fq; +// m11B -= fq; +// m12B -= fq; +// m15B += fq; +// m16B += fq; +// m18B -= fq; +// +// // q=13 +// nread = neighborList[n+12*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB += fq; +// m4B += fq; +// jzB -= fq; +// m8B -= fq; +// m9B += fq; +// m10B += fq; +// m11B -= fq; +// m12B -= fq; +// m15B -= fq; +// m16B -= fq; +// m18B -= fq; +// +// // q=14 +// nread = neighborList[n+13*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jxB -= fq; +// m4B -= fq; +// jzB += fq; +// m8B += fq; +// m9B += fq; +// m10B += fq; +// m11B -= fq; +// m12B -= fq; +// m15B -= fq; +// m16B += fq; +// m18B += fq; +// +// // q=15 +// nread = neighborList[n+14*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jyB += fq; +// m6B += fq; +// jzB += fq; +// m8B += fq; +// m9B -= 2.0*fq; +// m10B -= 2.0*fq; +// m14B = fq; +// m17B += fq; +// m18B -= fq; +// +// // q=16 +// nread = neighborList[n+15*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jyB -= fq; +// m6B -= fq; +// jzB -= fq; +// m8B -= fq; +// m9B -= 2.0*fq; +// m10B -= 2.0*fq; +// m14B += fq; +// m17B -= fq; +// m18B += fq; +// +// // q=17 +// nread = neighborList[n+16*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jyB += fq; +// m6B += fq; +// jzB -= fq; +// m8B -= fq; +// m9B -= 2.0*fq; +// m10B -= 2.0*fq; +// m14B -= fq; +// m17B += fq; +// m18B += fq; +// +// // q=18 +// nread = neighborList[n+17*Np]; +// fq = distB[nread]; +// rhoB += fq; +// m1B += 8.0*fq; +// m2B += fq; +// jyB -= fq; +// m6B -= fq; +// jzB += fq; +// m8B += fq; +// m9B -= 2.0*fq; +// m10B -= 2.0*fq; +// m14B -= fq; +// m17B -= fq; +// m18B -= fq; +// //---------------------------------------------------------------------// +// +// +// // Compute SC fluid-fluid interaction force +// GffA_x = -Gsc*nB_gradx; +// GffA_y = -Gsc*nB_grady; +// GffA_z = -Gsc*nB_gradz; +// GffB_x = -Gsc*nA_gradx; +// GffB_y = -Gsc*nA_grady; +// GffB_z = -Gsc*nA_gradz; +// // Compute SC fluid-solid force +// GfsA_x = SolidForceA[n+0*Np]; +// GfsA_y = SolidForceA[n+1*Np]; +// GfsA_z = SolidForceA[n+2*Np]; +// GfsB_x = SolidForceB[n+0*Np]; +// GfsB_y = SolidForceB[n+1*Np]; +// GfsB_z = SolidForceB[n+2*Np]; +// +// // Compute greyscale related parameters +// // ------------------- Fluid Component A -----------------------// +// c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(permA); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); +// vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); +// vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// FxA = nA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); +// FyA = nA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); +// FzA = nA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); +// if (porosity==1.0){ +// FxA=nA*(Gx + GffA_x + GfsA_x); +// FyA=nA*(Gy + GffA_y + GfsA_y); +// FzA=nA*(Gz + GffA_z + GfsA_z); +// } +// // ------------------- Fluid Component B -----------------------// +// // Compute greyscale related parameters +// c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(permB); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); +// vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); +// vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// FxB = nB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); +// FyB = nB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); +// FzB = nB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); +// if (porosity==1.0){ +// FxB=nB*(Gx + GffB_x + GfsB_x); +// FyB=nB*(Gy + GffB_y + GfsB_y); +// FzB=nB*(Gz + GffB_z + GfsB_z); +// } +// +// // Calculate barycentric velocity of the fluid mixture +// ux = (nA*ux_A+nB*ux_B)/(nA+nB); +// uy = (nA*uy_A+nB*uy_B)/(nA+nB); +// uz = (nA*uz_A+nB*uz_B)/(nA+nB); +// +// // ------------------- Fluid Component A -----------------------// +// rlx_setA = 1.0/tauA; +// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); +// //-------------------- MRT collison where body force has NO higher-order terms -------------// +// //..............carry out relaxation process............................................... +// //TODO need to incoporate porosity +// m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA) - m1A) +// + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); +// m2A = m2A + rlx_setA*((3*rhoA - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) +// + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); +// jxA = jxA + FxA; +// m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); +// jyA = jyA + FyA; +// m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA)- m6A) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); +// jzA = jzA + FzA; +// m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA)- m8A) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); +// m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) +// + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); +// //m10A = m10A + rlx_setA*( - m10A) +// // + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); +// m10A = m10A + rlx_setA*( -0.5*(rhoA*(2*ux*ux-uy*uy-uz*uz))- m10A) +// + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); +// m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) +// + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); +// //m12A = m12A + rlx_setA*( - m12A) +// // + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); +// m12A = m12A + rlx_setA*( -0.5*(rhoA*(uy*uy-uz*uz))- m12A) +// + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); +// m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) +// + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); +// m14A = m14A + rlx_setA*( rhoA*(uy*uz) - m14A) +// + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); +// m15A = m15A + rlx_setA*( rhoA*(ux*uz) - m15A) +// + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); +// m16A = m16A + rlx_setB*( - m16A); +// m17A = m17A + rlx_setB*( - m17A); +// m18A = m18A + rlx_setB*( - m18A); +// //....................................................................................................... +// +// +// // ------------------- Fluid Component A -----------------------// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rhoA-mrt_V2*m1A+mrt_V3*m2A; +// distA[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); +// nread = neighborList[n+Np]; +// distA[nread] = fq; +// +// // q=2 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); +// nread = neighborList[n]; +// distA[nread] = fq; +// +// // q = 3 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); +// nread = neighborList[n+3*Np]; +// distA[nread] = fq; +// +// // q = 4 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); +// nread = neighborList[n+2*Np]; +// distA[nread] = fq; +// +// // q = 5 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); +// nread = neighborList[n+5*Np]; +// distA[nread] = fq; +// +// // q = 6 +// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); +// nread = neighborList[n+4*Np]; +// distA[nread] = fq; +// +// // q = 7 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); +// nread = neighborList[n+7*Np]; +// distA[nread] = fq; +// +// // q = 8 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); +// nread = neighborList[n+6*Np]; +// distA[nread] = fq; +// +// // q = 9 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); +// nread = neighborList[n+9*Np]; +// distA[nread] = fq; +// +// // q = 10 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); +// nread = neighborList[n+8*Np]; +// distA[nread] = fq; +// +// // q = 11 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); +// nread = neighborList[n+11*Np]; +// distA[nread] = fq; +// +// // q = 12 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); +// nread = neighborList[n+10*Np]; +// distA[nread]= fq; +// +// // q = 13 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); +// nread = neighborList[n+13*Np]; +// distA[nread] = fq; +// +// // q= 14 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); +// nread = neighborList[n+12*Np]; +// distA[nread] = fq; +// +// // q = 15 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); +// nread = neighborList[n+15*Np]; +// distA[nread] = fq; +// +// // q = 16 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); +// nread = neighborList[n+14*Np]; +// distA[nread] = fq; +// +// // q = 17 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); +// nread = neighborList[n+17*Np]; +// distA[nread] = fq; +// +// // q = 18 +// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); +// nread = neighborList[n+16*Np]; +// distA[nread] = fq; +// //........................................................................ +// +// // ------------------- Fluid Component B -----------------------// +// rlx_setA = 1.0/tauB; +// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); +// //-------------------- MRT collison where body force has NO higher-order terms -------------// +// //..............carry out relaxation process............................................... +// //TODO need to incoporate porosity +// m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB) - m1B) +// + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); +// m2B = m2B + rlx_setA*((3*rhoB - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) +// + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); +// jxB = jxB + FxB; +// m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); +// jyB = jyB + FyB; +// m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB)- m6B) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); +// jzB = jzB + FzB; +// m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB)- m8B) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); +// m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) +// + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); +// //m10B = m10B + rlx_setA*( - m10B) +// // + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); +// m10B = m10B + rlx_setA*( -0.5*(rhoB*(2*ux*ux-uy*uy-uz*uz))- m10B) +// + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); +// m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) +// + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); +// //m12B = m12B + rlx_setA*( - m12B) +// // + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); +// m12B = m12B + rlx_setA*( -0.5*(rhoB*(uy*uy-uz*uz))- m12B) +// + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); +// m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) +// + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); +// m14B = m14B + rlx_setA*( rhoB*(uy*uz) - m14B) +// + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); +// m15B = m15B + rlx_setA*( rhoB*(ux*uz) - m15B) +// + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); +// m16B = m16B + rlx_setB*( - m16B); +// m17B = m17B + rlx_setB*( - m17B); +// m18B = m18B + rlx_setB*( - m18B); +// //....................................................................................................... +// +// +// // ------------------- Fluid Component B -----------------------// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rhoB-mrt_V2*m1B+mrt_V3*m2B; +// distB[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); +// nread = neighborList[n+Np]; +// distB[nread] = fq; +// +// // q=2 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); +// nread = neighborList[n]; +// distB[nread] = fq; +// +// // q = 3 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); +// nread = neighborList[n+3*Np]; +// distB[nread] = fq; +// +// // q = 4 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); +// nread = neighborList[n+2*Np]; +// distB[nread] = fq; +// +// // q = 5 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); +// nread = neighborList[n+5*Np]; +// distB[nread] = fq; +// +// // q = 6 +// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); +// nread = neighborList[n+4*Np]; +// distB[nread] = fq; +// +// // q = 7 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); +// nread = neighborList[n+7*Np]; +// distB[nread] = fq; +// +// // q = 8 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); +// nread = neighborList[n+6*Np]; +// distB[nread] = fq; +// +// // q = 9 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); +// nread = neighborList[n+9*Np]; +// distB[nread] = fq; +// +// // q = 10 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); +// nread = neighborList[n+8*Np]; +// distB[nread] = fq; +// +// // q = 11 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); +// nread = neighborList[n+11*Np]; +// distB[nread] = fq; +// +// // q = 12 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); +// nread = neighborList[n+10*Np]; +// distB[nread]= fq; +// +// // q = 13 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); +// nread = neighborList[n+13*Np]; +// distB[nread] = fq; +// +// // q= 14 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); +// nread = neighborList[n+12*Np]; +// distB[nread] = fq; +// +// // q = 15 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); +// nread = neighborList[n+15*Np]; +// distB[nread] = fq; +// +// // q = 16 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); +// nread = neighborList[n+14*Np]; +// distB[nread] = fq; +// +// // q = 17 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); +// nread = neighborList[n+17*Np]; +// distB[nread] = fq; +// +// // q = 18 +// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); +// nread = neighborList[n+16*Np]; +// distB[nread] = fq; +// //........................................................................ +// +// //Update velocity on device +// Velocity[0*Np+n] = ux; +// Velocity[1*Np+n] = uy; +// Velocity[2*Np+n] = uz; +// //Update pressure on device +// Pressure[n] = (nA+nB+Gsc*nA*nB)/3.0; +// } +// } +//} + +//__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np){ +// +// int ijk; +// int n; +// double vx,vy,vz,v_mag; +// double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; +// double ux,uy,uz; +// // conserved momemnts +// double jxA,jyA,jzA; +// double jxB,jyB,jzB; +// double rhoA,rhoB; +// double nA,nB; +// // non-conserved moments +// double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; +// double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; +// double fq; +// //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double permA,permB;//effective relative perm +// double c0, c1; //Guo's model parameters +// double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity +// double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity +// double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) +// double FxB, FyB, FzB; +// double rlx_setA,rlx_setB; +// double nA_gradx,nA_grady,nA_gradz; +// double nB_gradx,nB_grady,nB_gradz; +// double GffA_x,GffA_y,GffA_z; +// double GfsA_x,GfsA_y,GfsA_z; +// double GffB_x,GffB_y,GffB_z; +// double GfsB_x,GfsB_y,GfsB_z; +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s 10Np => odd part of dist) + f1A = distA[nr1]; // reading the f1 data into register fq + f1B = distB[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2A = distA[nr2]; // reading the f2 data into register fq + f2B = distB[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3A = distA[nr3]; + f3B = distB[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4A = distA[nr4]; + f4B = distB[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5A = distA[nr5]; + f5B = distB[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6A = distA[nr6]; + f6B = distB[nr6]; + + // q=7 + nr7 = neighborList[n+6*Np]; + f7A = distA[nr7]; + f7B = distB[nr7]; + + // q = 8 + nr8 = neighborList[n+7*Np]; + f8A = distA[nr8]; + f8B = distB[nr8]; + + // q=9 + nr9 = neighborList[n+8*Np]; + f9A = distA[nr9]; + f9B = distB[nr9]; + + // q = 10 + nr10 = neighborList[n+9*Np]; + f10A = distA[nr10]; + f10B = distB[nr10]; + + // q=11 + nr11 = neighborList[n+10*Np]; + f11A = distA[nr11]; + f11B = distB[nr11]; + + // q=12 + nr12 = neighborList[n+11*Np]; + f12A = distA[nr12]; + f12B = distB[nr12]; + + // q=13 + nr13 = neighborList[n+12*Np]; + f13A = distA[nr13]; + f13B = distB[nr13]; + + // q=14 + nr14 = neighborList[n+13*Np]; + f14A = distA[nr14]; + f14B = distB[nr14]; + + // q=15 + nr15 = neighborList[n+14*Np]; + f15A = distA[nr15]; + f15B = distB[nr15]; + + // q=16 + nr16 = neighborList[n+15*Np]; + f16A = distA[nr16]; + f16B = distB[nr16]; + + // q=17 + //fq = dist[18*Np+n]; + nr17 = neighborList[n+16*Np]; + f17A = distA[nr17]; + f17B = distB[nr17]; + + // q=18 + nr18 = neighborList[n+17*Np]; + f18A = distA[nr18]; + f18B = distB[nr18]; + //---------------------------------------------------------------------// + + // Compute SC fluid-fluid interaction force + GffA_x = -Gsc*rhoB_gradx; + GffA_y = -Gsc*rhoB_grady; + GffA_z = -Gsc*rhoB_gradz; + GffB_x = -Gsc*rhoA_gradx; + GffB_y = -Gsc*rhoA_grady; + GffB_z = -Gsc*rhoA_gradz; + // Compute SC fluid-solid force + GfsA_x = SolidForceA[n+0*Np]; + GfsA_y = SolidForceA[n+1*Np]; + GfsA_z = SolidForceA[n+2*Np]; + GfsB_x = SolidForceB[n+0*Np]; + GfsB_y = SolidForceB[n+1*Np]; + GfsB_z = SolidForceB[n+2*Np]; + + // Compute greyscale related parameters + // ------------------- Fluid Component A -----------------------// + jxA = f1A-f2A+f7A-f8A+f9A-f10A+f11A-f12A+f13A-f14A; + jyA = f3A-f4A+f7A-f8A-f9A+f10A+f15A-f16A+f17A-f18A; + jzA = f5A-f6A+f11A-f12A-f13A+f14A+f15A-f16A-f17A+f18A; + + c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permA); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); + vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); + vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); + FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); + FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); + if (porosity==1.0){ + FxA=rhoA*(Gx + GffA_x + GfsA_x); + FyA=rhoA*(Gy + GffA_y + GfsA_y); + FzA=rhoA*(Gz + GffA_z + GfsA_z); + } + // ------------------- Fluid Component B -----------------------// + // Compute greyscale related parameters + jxB = f1B-f2B+f7B-f8B+f9B-f10B+f11B-f12B+f13B-f14B; + jyB = f3B-f4B+f7B-f8B-f9B+f10B+f15B-f16B+f17B-f18B; + jzB = f5B-f6B+f11B-f12B-f13B+f14B+f15B-f16B-f17B+f18B; + + c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(permB); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); + vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); + vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); + FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); + FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); + if (porosity==1.0){ + FxB=rhoB*(Gx + GffB_x + GfsB_x); + FyB=rhoB*(Gy + GffB_y + GfsB_y); + FzB=rhoB*(Gz + GffB_z + GfsB_z); + } + + // Calculate barycentric velocity of the fluid mixture + ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); + uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); + uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); + + //..............carry out relaxation process............................................... + // ------------------- Fluid Component A -----------------------// + // q=0 + distA[n] = f0A*(1.0-rlx) + rlx*0.3333333333333333*rhoA*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + + 0.3333333333333333*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 1 + distA[nr2] = f1A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q=2 + distA[nr1] = f2A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(-3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 3 + distA[nr4] = f3A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 4 + distA[nr3] = f4A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); + + // q = 5 + distA[nr6] = f5A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(3. + (6.*uz)/porosity)); + + // q = 6 + distA[nr5] = f6A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(-3. + (6.*uz)/porosity)); + + // q = 7 + distA[nr8] = f7A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 8 + distA[nr7] = f8A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyA*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 9 + distA[nr10] = f9A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyA*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 10 + distA[nr9] = f10A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + + FzA*(0. - (3.*uz)/porosity)); + + // q = 11 + distA[nr12] = f11A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + + // q = 12 + distA[nr11] = f12A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + + FzA*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + + // q = 13 + distA[nr14] = f13A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + + FzA*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + + // q= 14 + distA[nr13] = f14A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + + // q = 15 + distA[nr16] = f15A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + + // q = 16 + distA[nr15] = f16A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + + FzA*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 17 + distA[nr18] = f17A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + + FzA*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 18 + distA[nr17] = f18A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + + FzA*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + + + // ------------------- Fluid Component B -----------------------// + // q=0 + distB[n] = f0B*(1.0-rlx) + rlx*0.3333333333333333*rhoB*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + + 0.3333333333333333*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 1 + distB[nr2] = f1B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q=2 + distB[nr1] = f2B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(-3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 3 + distB[nr4] = f3B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 4 + distB[nr3] = f4B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); + + // q = 5 + distB[nr6] = f5B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(3. + (6.*uz)/porosity)); + + // q = 6 + distB[nr5] = f6B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) + +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(-3. + (6.*uz)/porosity)); + + // q = 7 + distB[nr8] = f7B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 8 + distB[nr7] = f8B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyB*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 9 + distB[nr10] = f9B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyB*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 10 + distB[nr9] = f10B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + + FzB*(0. - (3.*uz)/porosity)); + + // q = 11 + distB[nr12] = f11B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); + + // q = 12 + distB[nr11] = f12B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + + FzB*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); + + // q = 13 + distB[nr14] = f13B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + + FzB*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); + + // q= 14 + distB[nr13] = f14B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); + + // q = 15 + distB[nr16] = f15B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); + + // q = 16 + distB[nr15] = f16B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + + FzB*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 17 + distB[nr18] = f17B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + + FzB*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); + + // q = 18 + distB[nr17] = f18B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) + +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + + FzB*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); + + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map,double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, + double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, + double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, + int start, int finish, int Np){ + + int n; + int ijk; + double vx,vy,vz,v_mag; + double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; + double ux,uy,uz; + double rhoA,rhoB; + double jxA,jyA,jzA; + double jxB,jyB,jzB; + // distribution functions + double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A,f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; + double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B,f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double permA,permB;//effective relative perm + double c0, c1; //Guo's model parameters + double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity + double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity + double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double FxB, FyB, FzB; + double tau,rlx; + double phi;//phase field indicator + double rhoA_gradx,rhoA_grady,rhoA_gradz; + double rhoB_gradx,rhoB_grady,rhoB_gradz; + double GffA_x,GffA_y,GffA_z; + double GfsA_x,GfsA_y,GfsA_z; + double GffB_x,GffB_y,GffB_z; + double GfsB_x,GfsB_y,GfsB_z; + + for (n=start; n + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, + double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ + + int n,nn,ijk,nread; + int nr1,nr2,nr3,nr4,nr5,nr6; + int nr7,nr8,nr9,nr10; + int nr11,nr12,nr13,nr14; + //int nr15,nr16,nr17,nr18; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double phi,tau,rho0,rlx_setA,rlx_setB; + + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double tau_eff; + double mu_eff;//kinematic viscosity + double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s even part of dist) + //fq = dist[nread]; // reading the f2 data into register fq + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + //nread = neighborList[n+2*Np]; // neighbor 4 + //fq = dist[nread]; + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + //nread = neighborList[n+3*Np]; // neighbor 3 + //fq = dist[nread]; + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + //nread = neighborList[n+4*Np]; + //fq = dist[nread]; + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + //nread = neighborList[n+5*Np]; + //fq = dist[nread]; + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + //nread = neighborList[n+6*Np]; + //fq = dist[nread]; + nr7 = neighborList[n+6*Np]; + fq = dist[nr7]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + //nread = neighborList[n+7*Np]; + //fq = dist[nread]; + nr8 = neighborList[n+7*Np]; + fq = dist[nr8]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + //nread = neighborList[n+8*Np]; + //fq = dist[nread]; + nr9 = neighborList[n+8*Np]; + fq = dist[nr9]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + //nread = neighborList[n+9*Np]; + //fq = dist[nread]; + nr10 = neighborList[n+9*Np]; + fq = dist[nr10]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + //nread = neighborList[n+10*Np]; + //fq = dist[nread]; + nr11 = neighborList[n+10*Np]; + fq = dist[nr11]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + //nread = neighborList[n+11*Np]; + //fq = dist[nread]; + nr12 = neighborList[n+11*Np]; + fq = dist[nr12]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + //nread = neighborList[n+12*Np]; + //fq = dist[nread]; + nr13 = neighborList[n+12*Np]; + fq = dist[nr13]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + //nread = neighborList[n+13*Np]; + //fq = dist[nread]; + nr14 = neighborList[n+13*Np]; + fq = dist[nr14]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + + // Compute greyscale related parameters + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*(porosity*Gx); + vy = jy/rho0+0.5*(porosity*Gy); + vz = jz/rho0+0.5*(porosity*Gz); + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=rho0*(Gx); + Fy=rho0*(Gy); + Fz=rho0*(Gz); + } + + // write the velocity + Velocity[n] = ux; + Velocity[Np+n] = uy; + Velocity[2*Np+n] = uz; + + //........................................................................ + //..............carry out relaxation process.............................. + //..........Toelke, Fruediger et. al. 2006................................ + if (C == 0.0) nx = ny = nz = 0.0; + m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); + m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); + m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); + m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); + m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + //nread = neighborList[n+7*Np]; + dist[nr8] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + //nread = neighborList[n+6*Np]; + dist[nr7] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + //nread = neighborList[n+9*Np]; + dist[nr10] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + //nread = neighborList[n+8*Np]; + dist[nr9] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + //nread = neighborList[n+11*Np]; + dist[nr12] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + //nread = neighborList[n+10*Np]; + dist[nr11]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + //nread = neighborList[n+13*Np]; + dist[nr14] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + //nread = neighborList[n+12*Np]; + dist[nr13] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + Aq[n] = 0.3333333333333333*nA; + Bq[n] = 0.3333333333333333*nB; + + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + // q = 1 + //nread = neighborList[n+Np]; + Aq[nr2] = a1; + Bq[nr2] = b1; + // q=2 + //nread = neighborList[n]; + Aq[nr1] = a2; + Bq[nr1] = b2; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + // q = 3 + //nread = neighborList[n+3*Np]; + Aq[nr4] = a1; + Bq[nr4] = b1; + // q = 4 + //nread = neighborList[n+2*Np]; + Aq[nr3] = a2; + Bq[nr3] = b2; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + // q = 5 + //nread = neighborList[n+5*Np]; + Aq[nr6] = a1; + Bq[nr6] = b1; + // q = 6 + //nread = neighborList[n+4*Np]; + Aq[nr5] = a2; + Bq[nr5] = b2; + //............................................... + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, + double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ + int ijk,nn,n; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double m3,m5,m7; + double nA,nB; // number density + double a1,b1,a2,b2,nAB,delta; + double C,nx,ny,nz; //color gradient magnitude and direction + double phi,tau,rho0,rlx_setA,rlx_setB; + + double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double tau_eff; + double mu_eff;//kinematic viscosity + double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + Aq[1*Np+n] = a1; + Bq[1*Np+n] = b1; + Aq[2*Np+n] = a2; + Bq[2*Np+n] = b2; + + //............................................... + // q = 2 + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + Aq[3*Np+n] = a1; + Bq[3*Np+n] = b1; + Aq[4*Np+n] = a2; + Bq[4*Np+n] = b2; + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + Aq[5*Np+n] = a1; + Bq[5*Np+n] = b1; + Aq[6*Np+n] = a2; + Bq[6*Np+n] = b2; + //............................................... + + } + } +} + + +//__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np) +//{ +// int n; +// int S = Np/NBLOCKS/NTHREADS + 1; +// double porosity; +// for (int s=0; s>>(dist,Porosity,Np); +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q19_GreyscaleColor_Init: %s \n",cudaGetErrorString(err)); +// } +//} + +extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, + double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ + + //cudaProfilerStart(); + //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor, cudaFuncCachePreferL1); + + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, + rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColor: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); + +} + +extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, + double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ + + //cudaProfilerStart(); + //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor, cudaFuncCachePreferL1); + + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel, + rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff,alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColor: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu index e22b7a50..309a9371 100644 --- a/gpu/GreyscaleSC.cu +++ b/gpu/GreyscaleSC.cu @@ -3215,7 +3215,7 @@ __global__ void dvc_ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Ma } } -__global__ void dvc_ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np) +__global__ void dvc_ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count) { int idx,n,nm; // Fill the outlet with component b @@ -3228,7 +3228,7 @@ __global__ void dvc_ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, } } -__global__ void dvc_ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np) +__global__ void dvc_ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count) { int idx,n,nm; // Fill the outlet with component b @@ -3764,18 +3764,18 @@ extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, d } -extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np){ +extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count){ int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_BC_z<<>>(list, Map, DenA, DenB, vA, vB, count, Np); + dvc_ScaLBL_GreyscaleSC_BC_z<<>>(list, Map, DenA, DenB, vA, vB, count); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_GreyscaleSC_BC_z: %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count, int Np){ +extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count){ int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_BC_Z<<>>(list, Map, DenA, DenB, vA, vB, count, Np); + dvc_ScaLBL_GreyscaleSC_BC_Z<<>>(list, Map, DenA, DenB, vA, vB, count); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ printf("CUDA error in ScaLBL_GreyscaleSC_BC_Z: %s \n",cudaGetErrorString(err)); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 3fef03d1..c7cabe04 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -1,7 +1,7 @@ /* color lattice boltzmann model */ -#include "models/ColorModel.h" +#include "models/GreyscaleColorModel.h" #include "analysis/distance.h" #include "analysis/morphology.h" #include "common/Communication.h" @@ -9,18 +9,18 @@ color lattice boltzmann model #include #include -ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),greyMode(0),Lx(0),Ly(0),Lz(0),comm(COMM) { REVERSE_FLOW_DIRECTION = false; } -ScaLBL_ColorModel::~ScaLBL_ColorModel(){ +ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ } -/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) +/*void ScaLBL_GreyscaleColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) { int q,n; double value; @@ -39,7 +39,7 @@ ScaLBL_ColorModel::~ScaLBL_ColorModel(){ } -void ScaLBL_ColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np) +void ScaLBL_GreyscaleColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np) { int q=0, n=0; double value=0; @@ -58,11 +58,11 @@ void ScaLBL_ColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq */ -void ScaLBL_ColorModel::ReadParams(string filename){ +void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); - color_db = db->getDatabase( "Color" ); + greyscaleColor_db = db->getDatabase( "Color" ); analysis_db = db->getDatabase( "Analysis" ); vis_db = db->getDatabase( "Visualization" ); @@ -76,46 +76,55 @@ void ScaLBL_ColorModel::ReadParams(string filename){ Restart=false; din=dout=1.0; flux=0.0; + greyMode=false; // Color Model parameters - if (color_db->keyExists( "timestepMax" )){ - timestepMax = color_db->getScalar( "timestepMax" ); + if (greyscaleColor_db->keyExists( "timestepMax" )){ + timestepMax = greyscaleColor_db->getScalar( "timestepMax" ); } - if (color_db->keyExists( "tauA" )){ - tauA = color_db->getScalar( "tauA" ); + if (greyscaleColor_db->keyExists( "tauA" )){ + tauA = greyscaleColor_db->getScalar( "tauA" ); } - if (color_db->keyExists( "tauB" )){ - tauB = color_db->getScalar( "tauB" ); + if (greyscaleColor_db->keyExists( "tauB" )){ + tauB = greyscaleColor_db->getScalar( "tauB" ); } - if (color_db->keyExists( "rhoA" )){ - rhoA = color_db->getScalar( "rhoA" ); + tauA_eff = greyscaleColor_db->getWithDefault( "tauA_eff", tauA ); + tauB_eff = greyscaleColor_db->getWithDefault( "tauB_eff", tauB ); + if (greyscaleColor_db->keyExists( "rhoA" )){ + rhoA = greyscaleColor_db->getScalar( "rhoA" ); } - if (color_db->keyExists( "rhoB" )){ - rhoB = color_db->getScalar( "rhoB" ); + if (greyscaleColor_db->keyExists( "rhoB" )){ + rhoB = greyscaleColor_db->getScalar( "rhoB" ); } - if (color_db->keyExists( "F" )){ - Fx = color_db->getVector( "F" )[0]; - Fy = color_db->getVector( "F" )[1]; - Fz = color_db->getVector( "F" )[2]; + if (greyscaleColor_db->keyExists( "F" )){ + Fx = greyscaleColor_db->getVector( "F" )[0]; + Fy = greyscaleColor_db->getVector( "F" )[1]; + Fz = greyscaleColor_db->getVector( "F" )[2]; } - if (color_db->keyExists( "alpha" )){ - alpha = color_db->getScalar( "alpha" ); + if (greyscaleColor_db->keyExists( "alpha" )){ + alpha = greyscaleColor_db->getScalar( "alpha" ); } - if (color_db->keyExists( "beta" )){ - beta = color_db->getScalar( "beta" ); + if (greyscaleColor_db->keyExists( "beta" )){ + beta = greyscaleColor_db->getScalar( "beta" ); } - if (color_db->keyExists( "Restart" )){ - Restart = color_db->getScalar( "Restart" ); + if (greyscaleColor_db->keyExists( "Restart" )){ + Restart = greyscaleColor_db->getScalar( "Restart" ); } - if (color_db->keyExists( "din" )){ - din = color_db->getScalar( "din" ); + if (greyscaleColor_db->keyExists( "din" )){ + din = greyscaleColor_db->getScalar( "din" ); } - if (color_db->keyExists( "dout" )){ - dout = color_db->getScalar( "dout" ); + if (greyscaleColor_db->keyExists( "dout" )){ + dout = greyscaleColor_db->getScalar( "dout" ); } - if (color_db->keyExists( "flux" )){ - flux = color_db->getScalar( "flux" ); + if (greyscaleColor_db->keyExists( "flux" )){ + flux = greyscaleColor_db->getScalar( "flux" ); } + if (greyscaleColor_db->keyExists("GreySolidLabels")|| + greyscaleColor_db->keyExists("GreySolidAffinity")|| + greyscaleColor_db->keyExists("PorosityList")|| + greyscaleColor_db->keyExists("PermeabilityList")){ + greyMode = true; + } inletA=1.f; inletB=0.f; outletA=0.f; @@ -128,7 +137,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ } // Override user-specified boundary condition for specific protocols - auto protocol = color_db->getWithDefault( "protocol", "none" ); + auto protocol = greyscaleColor_db->getWithDefault( "protocol", "none" ); if (protocol == "seed water"){ if (BoundaryCondition != 0 ){ BoundaryCondition = 0; @@ -152,7 +161,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ } } -void ScaLBL_ColorModel::SetDomain(){ +void ScaLBL_GreyscaleColorModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases // domain parameters @@ -177,15 +186,15 @@ void ScaLBL_ColorModel::SetDomain(){ nprocz = Dm->nprocz(); } -void ScaLBL_ColorModel::ReadInput(){ +void ScaLBL_GreyscaleColorModel::ReadInput(){ sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - if (color_db->keyExists( "image_sequence" )){ - auto ImageList = color_db->getVector( "image_sequence"); - int IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + if (greyscaleColor_db->keyExists( "image_sequence" )){ + auto ImageList = greyscaleColor_db->getVector( "image_sequence"); + int IMAGE_INDEX = greyscaleColor_db->getWithDefault( "image_index", 0 ); std::string first_image = ImageList[IMAGE_INDEX]; Mask->Decomp(first_image); IMAGE_INDEX++; @@ -245,14 +254,28 @@ void ScaLBL_ColorModel::ReadInput(){ Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); } -void ScaLBL_ColorModel::AssignComponentLabels(double *phase) +void ScaLBL_GreyscaleColorModel::AssignComponentLabels() { + // Initialize impermeability solid nodes and grey nodes + // Key input parameters: + // 1. ComponentLabels + // labels for various impermeable minerals and grey nodes + // 2. ComponentAffinity + // for impermeable minerals, this is same as the wettability phase field in the normal color model + // for grey nodes, this is effectively the initial phase field values + // **Convention for ComponentLabels: + // (1) zero and negative integers are for impermeability minerals + // (2) positive integers > 2 are for grey nodes + // (3) label = 1 and 2 are always conserved for open node of non-wetting and wetting phase, respectively. + double *phase; + phase = new double[N]; + size_t NLABELS=0; signed char VALUE=0; double AFFINITY=0.f; - auto LabelList = color_db->getVector( "ComponentLabels" ); - auto AffinityList = color_db->getVector( "ComponentAffinity" ); + auto LabelList = greyscaleColor_db->getVector( "ComponentLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "ComponentAffinity" ); NLABELS=LabelList.size(); if (NLABELS != AffinityList.size()){ @@ -304,10 +327,300 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) } } + ScaLBL_CopyToDevice(Phi, phase, N*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] phase; +} + +void ScaLBL_GreyscaleColorModel::AssignGreySolidLabels() +{ + // ONLY initialize grey nodes + // Key input parameters: + // 1. GreySolidLabels + // labels for grey nodes + // 2. GreySolidAffinity + // affinity ranges [-1,1] + // oil-wet > 0 + // water-wet < 0 + // neutral = 0 + double *SolidPotential_host = new double [Nx*Ny*Nz]; + double *GreySolidGrad_host = new double [3*Np]; + + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (size_t idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + SolidPotential_host[n] = AFFINITY; + } + } + } + + // Calculate grey-solid color-gradient + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f; + double w_edge = 0.5; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kComm.sumReduce( label_count[idx] ); + + if (rank==0){ + printf("Grey-solid labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxgetVector( "GreySolidLabels" ); + auto PorosityList = greyscaleColor_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscaleColor_db->getVector( "PermeabilityList" ); + + NLABELS=LabelList.size(); + if (LabelList.size() != PorosityList.size()){ + ERROR("Error: GreySolidLabels and PorosityList must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Porosity[idx] = POROSITY; + } + } + } + } + } + + if (NLABELS != PermeabilityList.size()){ + ERROR("Error: GreySolidLabels and PermeabilityList must be the same length! \n"); + } + for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + } + } + } + } + } + + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm.sumReduce(label_count[idx]); + + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); + } + + ScaLBL_CopyToDevice(Porosity_dvc, Porosity, Np*sizeof(double)); + ScaLBL_CopyToDevice(Permeability_dvc, Permeability, Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] Porosity; + delete [] Permeability; } -void ScaLBL_ColorModel::Create(){ +void ScaLBL_GreyscaleColorModel::Create(){ /* * This function creates the variables needed to run a LBM */ @@ -353,7 +666,12 @@ void ScaLBL_ColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + if (greyMode==true){ + ScaLBL_AllocateDeviceMemory((void **) &GreySolidGrad, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Permeability_dvc, sizeof(double)*Np); + } + //ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); @@ -390,21 +708,21 @@ void ScaLBL_ColorModel::Create(){ // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - // initialize phi based on PhaseLabel (include solid component labels) - double *PhaseLabel; - PhaseLabel = new double[N]; - AssignComponentLabels(PhaseLabel); - ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); + + // initialize phi based on PhaseLabel (include solid component labels) + AssignComponentLabels();//do open/black/grey nodes initialization + if (greyMode==true){ + AssignGreySolidLabels(); + AssignGreyPoroPermLabels(); + } } -/******************************************************** - * AssignComponentLabels * - ********************************************************/ -void ScaLBL_ColorModel::Initialize(){ +void ScaLBL_GreyscaleColorModel::Initialize(){ if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); + //ScaLBL_D3Q19_GreyscaleColor_Init(fq, Porosity_dvc, Np); /* * This function initializes model */ @@ -487,7 +805,7 @@ void ScaLBL_ColorModel::Initialize(){ ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); } -void ScaLBL_ColorModel::Run(){ +void ScaLBL_GreyscaleColorModel::Run(){ int nprocs=nprocx*nprocy*nprocz; const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -529,23 +847,23 @@ void ScaLBL_ColorModel::Run(){ double log_krA_target = 1.0; double log_krA = 0.0; double slope_krA_volume = 0.0; - if (color_db->keyExists( "vol_A_previous" )){ - volA_prev = color_db->getScalar( "vol_A_previous" ); + if (greyscaleColor_db->keyExists( "vol_A_previous" )){ + volA_prev = greyscaleColor_db->getScalar( "vol_A_previous" ); } - if (color_db->keyExists( "log_krA_previous" )){ - log_krA_prev = color_db->getScalar( "log_krA_previous" ); + if (greyscaleColor_db->keyExists( "log_krA_previous" )){ + log_krA_prev = greyscaleColor_db->getScalar( "log_krA_previous" ); } - if (color_db->keyExists( "krA_morph_factor" )){ - KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); + if (greyscaleColor_db->keyExists( "krA_morph_factor" )){ + KRA_MORPH_FACTOR = greyscaleColor_db->getScalar( "krA_morph_factor" ); } /* defaults for simulation protocols */ - auto protocol = color_db->getWithDefault( "protocol", "none" ); + auto protocol = greyscaleColor_db->getWithDefault( "protocol", "none" ); if (protocol == "image sequence"){ // Get the list of images USE_DIRECT = true; - ImageList = color_db->getVector( "image_sequence"); - IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + ImageList = greyscaleColor_db->getVector( "image_sequence"); + IMAGE_INDEX = greyscaleColor_db->getWithDefault( "image_index", 0 ); IMAGE_COUNT = ImageList.size(); morph_interval = 10000; USE_MORPH = true; @@ -566,28 +884,28 @@ void ScaLBL_ColorModel::Run(){ USE_MORPH = true; } - if (color_db->keyExists( "residual_endpoint_threshold" )){ - RESIDUAL_ENDPOINT_THRESHOLD = color_db->getScalar( "residual_endpoint_threshold" ); + if (greyscaleColor_db->keyExists( "residual_endpoint_threshold" )){ + RESIDUAL_ENDPOINT_THRESHOLD = greyscaleColor_db->getScalar( "residual_endpoint_threshold" ); } NULL_USE( RESIDUAL_ENDPOINT_THRESHOLD ); - if (color_db->keyExists( "noise_threshold" )){ - NOISE_THRESHOLD = color_db->getScalar( "noise_threshold" ); + if (greyscaleColor_db->keyExists( "noise_threshold" )){ + NOISE_THRESHOLD = greyscaleColor_db->getScalar( "noise_threshold" ); USE_BUMP_RATE = true; } - if (color_db->keyExists( "bump_rate" )){ - BUMP_RATE = color_db->getScalar( "bump_rate" ); + if (greyscaleColor_db->keyExists( "bump_rate" )){ + BUMP_RATE = greyscaleColor_db->getScalar( "bump_rate" ); USE_BUMP_RATE = true; } - if (color_db->keyExists( "capillary_number" )){ - capillary_number = color_db->getScalar( "capillary_number" ); + if (greyscaleColor_db->keyExists( "capillary_number" )){ + capillary_number = greyscaleColor_db->getScalar( "capillary_number" ); SET_CAPILLARY_NUMBER=true; //RESCALE_FORCE_MAX = 1; } // if (analysis_db->keyExists( "rescale_force_count" )){ // RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); // } - if (color_db->keyExists( "timestep" )){ - timestep = color_db->getScalar( "timestep" ); + if (greyscaleColor_db->keyExists( "timestep" )){ + timestep = greyscaleColor_db->getScalar( "timestep" ); } if (BoundaryCondition != 0 && SET_CAPILLARY_NUMBER==true){ if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 \n"); @@ -699,9 +1017,15 @@ void ScaLBL_ColorModel::Run(){ } // Halo exchange for phase field ScaLBL_Comm_Regular->SendHalo(Phi); - - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (greyMode==true){ + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + else{ + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } ScaLBL_Comm_Regular->RecvHalo(Phi); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -714,8 +1038,15 @@ void ScaLBL_ColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + if (greyMode==true){ + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + } + else{ + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + } ScaLBL_DeviceBarrier(); comm.barrier(); @@ -736,8 +1067,15 @@ void ScaLBL_ColorModel::Run(){ ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } ScaLBL_Comm_Regular->SendHalo(Phi); - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (greyMode==true){ + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + else{ + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } ScaLBL_Comm_Regular->RecvHalo(Phi); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -750,8 +1088,15 @@ void ScaLBL_ColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + if (greyMode==true){ + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + } + else{ + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + } ScaLBL_DeviceBarrier(); comm.barrier(); //************************************************************************ @@ -810,7 +1155,7 @@ void ScaLBL_ColorModel::Run(){ } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); + greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } if ( morph_timesteps > morph_interval ){ @@ -919,13 +1264,13 @@ void ScaLBL_ColorModel::Run(){ Fy *= BUMP_RATE; Fz *= BUMP_RATE; capillary_number *= BUMP_RATE; - color_db->putScalar("capillary_number",capillary_number); - current_db->putDatabase("Color", color_db); + greyscaleColor_db->putScalar("capillary_number",capillary_number); + current_db->putDatabase("Color", greyscaleColor_db); MORPH_ADAPT = false; // re-run current point if below noise threshold } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); + greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } CURRENT_STEADY_TIMESTEPS = 0; @@ -949,8 +1294,8 @@ void ScaLBL_ColorModel::Run(){ if (IMAGE_INDEX < IMAGE_COUNT){ std::string next_image = ImageList[IMAGE_INDEX]; if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); - color_db->putScalar("image_index",IMAGE_INDEX); - ImageInit(next_image); + greyscaleColor_db->putScalar("image_index",IMAGE_INDEX); + //ImageInit(next_image); } else{ if (rank==0) printf("Finished simulating image sequence \n"); @@ -1031,53 +1376,54 @@ void ScaLBL_ColorModel::Run(){ // ************************************************************************ } -double ScaLBL_ColorModel::ImageInit(std::string Filename){ - - if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); - Mask->Decomp(Filename); - for (int i=0; iid[i]; // save what was read - for (int i=0; iid[i] = Mask->id[i]; // save what was read +//double ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ +// +// if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); +// Mask->Decomp(Filename); +// for (int i=0; iid[i]; // save what was read +// for (int i=0; iid[i] = Mask->id[i]; // save what was read +// +// //double *PhaseLabel; +// //PhaseLabel = new double[Nx*Ny*Nz]; +// //AssignComponentLabels(PhaseLabel); +// AssignComponentLabels(); +// +// double Count = 0.0; +// double PoreCount = 0.0; +// for (int k=1; kComm.sumReduce( Count ); +// PoreCount = Dm->Comm.sumReduce( PoreCount ); +// +// if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); +// ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); +// comm.barrier(); +// +// ScaLBL_D3Q19_Init(fq, Np); +// ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); +// ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// comm.barrier(); +// +// ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); +// +// double saturation = Count/PoreCount; +// return saturation; +// +//} - double *PhaseLabel; - PhaseLabel = new double[Nx*Ny*Nz]; - AssignComponentLabels(PhaseLabel); - - double Count = 0.0; - double PoreCount = 0.0; - for (int k=1; kComm.sumReduce( Count ); - PoreCount = Dm->Comm.sumReduce( PoreCount ); - - if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); - ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); - comm.barrier(); - - ScaLBL_D3Q19_Init(fq, Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - comm.barrier(); - - ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); - - double saturation = Count/PoreCount; - return saturation; - -} - -double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ +double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_change){ int nx = Nx; int ny = Ny; @@ -1220,7 +1566,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ return(volume_change); } -double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ +double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ srand(time(NULL)); double mass_loss =0.f; double count =0.f; @@ -1307,7 +1653,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ return(mass_loss); } -//double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ +//double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ // srand(time(NULL)); // double mass_loss =0.f; // double count =0.f; @@ -1401,7 +1747,7 @@ double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ // return(mass_loss); //} -double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ +double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double target_delta_volume){ const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); double vF = 0.f; @@ -1585,7 +1931,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta return delta_volume; } -void ScaLBL_ColorModel::WriteDebug(){ +void ScaLBL_GreyscaleColorModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); //ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); @@ -1639,6 +1985,41 @@ void ScaLBL_ColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); + ScaLBL_Comm->RegularLayout(Map,&Porosity_dvc[0],PhaseField); + FILE *POROS_FILE; + sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); + POROS_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,POROS_FILE); + fclose(POROS_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Permeability_dvc[0],PhaseField); + FILE *PERM_FILE; + sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); + PERM_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PERM_FILE); + fclose(PERM_FILE); + + ScaLBL_Comm->RegularLayout(Map,&GreySolidGrad[0],PhaseField); + FILE *GreySG_X_FILE; + sprintf(LocalRankFilename,"GreySolidGrad_X.%05i.raw",rank); + GreySG_X_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,GreySG_X_FILE); + fclose(GreySG_X_FILE); + + ScaLBL_Comm->RegularLayout(Map,&GreySolidGrad[Np],PhaseField); + FILE *GreySG_Y_FILE; + sprintf(LocalRankFilename,"GreySolidGrad_Y.%05i.raw",rank); + GreySG_Y_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,GreySG_Y_FILE); + fclose(GreySG_Y_FILE); + + ScaLBL_Comm->RegularLayout(Map,&GreySolidGrad[2*Np],PhaseField); + FILE *GreySG_Z_FILE; + sprintf(LocalRankFilename,"GreySolidGrad_Z.%05i.raw",rank); + GreySG_Z_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,GreySG_Z_FILE); + fclose(GreySG_Z_FILE); + // ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); // FILE *CGX_FILE; // sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index c52f04c3..64e3f1e3 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -16,10 +16,10 @@ Implementation of color lattice boltzmann model #include "ProfilerApp.h" #include "threadpool/thread_pool.h" -class ScaLBL_ColorModel{ +class ScaLBL_GreyscaleColorModel{ public: - ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM); - ~ScaLBL_ColorModel(); + ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM); + ~ScaLBL_GreyscaleColorModel(); // functions in they should be run void ReadParams(string filename); @@ -36,8 +36,11 @@ public: int timestep,timestepMax; int BoundaryCondition; double tauA,tauB,rhoA,rhoB,alpha,beta; + double tauA_eff,tauB_eff; double Fx,Fy,Fz,flux; double din,dout,inletA,inletB,outletA,outletB; + double GreyPorosity; + bool greyMode;//run greyColor model if true int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -53,7 +56,7 @@ public: // input database std::shared_ptr db; std::shared_ptr domain_db; - std::shared_ptr color_db; + std::shared_ptr greyscaleColor_db; std::shared_ptr analysis_db; std::shared_ptr vis_db; @@ -63,9 +66,12 @@ public: int *dvcMap; double *fq, *Aq, *Bq; double *Den, *Phi; - double *ColorGrad; + double *GreySolidGrad; + //double *ColorGrad; double *Velocity; double *Pressure; + double *Porosity_dvc; + double *Permeability_dvc; private: Utilities::MPI comm; @@ -79,7 +85,9 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); - void AssignComponentLabels(double *phase); + void AssignComponentLabels(); + void AssignGreySolidLabels(); + void AssignGreyPoroPermLabels(); double ImageInit(std::string filename); double MorphInit(const double beta, const double morph_delta); double SeedPhaseField(const double seed_water_in_oil); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 362d020e..567b43c3 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,6 +6,7 @@ ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscaleFE_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscaleSC_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_greyscaleColor_simulator.cpp b/tests/lbpm_greyscaleColor_simulator.cpp new file mode 100644 index 00000000..3f9fdd19 --- /dev/null +++ b/tests/lbpm_greyscaleColor_simulator.cpp @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI.h" +#include "models/GreyscaleColorModel.h" +//#define WRITE_SURFACES + +using namespace std; + + +int main(int argc, char **argv) +{ + //***************************************** + // ***** MPI STUFF **************** + //***************************************** + // Initialize MPI + int rank,nprocs; + MPI_Init(&argc,&argv); + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { + // parallel domain size (# of sub-domains) + int nprocx,nprocy,nprocz; + int iproc,jproc,kproc; + + if (rank == 0){ + printf("****************************************\n"); + printf("Running Greyscale Two-Phase Calculation \n"); + printf("****************************************\n"); + } + // Initialize compute device + int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm); + auto filename = argv[1]; + GreyscaleColor.ReadParams(filename); + GreyscaleColor.SetDomain(); // this reads in the domain + GreyscaleColor.ReadInput(); + GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleColor.Run(); + GreyscaleColor.WriteDebug(); + } + // **************************************************** + MPI_Barrier(comm); + MPI_Finalize(); + // **************************************************** +} From 5fae57157854561753c7290883052d6690e460b8 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 18 Jun 2020 19:56:38 -0400 Subject: [PATCH 160/270] fix connected pc --- models/ColorModel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 2784e28a..583ca230 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -854,8 +854,8 @@ void ScaLBL_ColorModel::Run(){ double pB = Averages->gwb.p; double pAc = Averages->gnc.p; double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*5.796*alpha); - double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + double pAB = (pA-pB)/(h*6.0*alpha); + double pAB_connected = (pAc-pBc)/(h*6.0*alpha); // connected contribution double Vol_nc = Averages->gnc.V/Dm->Volume; double Vol_wc = Averages->gwc.V/Dm->Volume; From cc858ea87b9b777092af5e3e5e31ca9ce16d8846 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 19 Jun 2020 21:36:28 -0400 Subject: [PATCH 161/270] Revert "fix bug in Euler characteristic at domain boundary" This reverts commit 2e47e9c306ef361b841c5b31bbfb78083324fc79. --- analysis/Minkowski.cpp | 6 ++-- models/ColorModel.cpp | 74 +++++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index e98cfdc7..faac6142 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=0; k 0.01*volume_initial) - target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); + if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); + double target_delta_volume_incremental = target_delta_volume; + if (fabs(target_delta_volume) > 0.01*volume_initial) + target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); - for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + //phase(i,j,k) = -1.0; + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } } } - } - - CalcDist(phase_distance,phase_id,*Dm); // re-calculate distance - - // 5. Update phase indicator field based on new distnace - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - //phase(i,j,k) = -1.0; - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } - } - } - fillDouble.fill(phase); + fillDouble.fill(phase); //} count = 0.f; From 3ab182daf795b8fdcdbcb6088d3b134813c379f2 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 19 Jun 2020 21:39:54 -0400 Subject: [PATCH 162/270] fixed half edge rule for Euler characteristic --- analysis/Minkowski.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index faac6142..43ab6ee8 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -88,7 +88,7 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) // printf(" (%i,%i,%i) RP(%i,%i)={%f,%f,%f} {%f,%f,%f} a=%f l=%f \n",i,j,k,e3,object.halfedge.twin(e3),P3.x,P3.y,P3.z,P1.x,P1.y,P1.z,a3,s3); //} // Euler characteristic (half edge rule: one face - 0.5*(three edges)) - Xi -= 0.5; + Xi -= 1.5; } // Euler characteristic -- each vertex shared by four cubes Xi += 0.25*double(object.VertexCount); From 435869740b87ef53c2e7e0a76f9a23d8fb44a293 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 22 Jun 2020 12:44:50 -0400 Subject: [PATCH 163/270] debug scalar MF --- analysis/Minkowski.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 43ab6ee8..9652db35 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=1; k Date: Mon, 22 Jun 2020 13:39:51 -0400 Subject: [PATCH 164/270] GPU only, make greyscaleColor equivalent single-phase model available --- common/ScaLBL.h | 5 + gpu/Greyscale.cu | 1070 ++++++++++++++++++++++++++++ models/GreyscaleModel.cpp | 21 +- tests/lbpm_greyscale_simulator.cpp | 2 +- 4 files changed, 1096 insertions(+), 2 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a74c54a2..de91e790 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -72,6 +72,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); + +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, + double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); // GREYSCALE FREE-ENERGY MODEL (Two-component) extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 19b3ccec..63b3ccc8 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -1590,6 +1590,1056 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double } } +__global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double rho0, double *Pressure){ + + int n, nread; + int nr1,nr2,nr3,nr4,nr5,nr6; + int nr7,nr8,nr9,nr10; + int nr11,nr12,nr13,nr14; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double rho,jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s even part of dist) + //fq = dist[nread]; // reading the f2 data into register fq + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + //nread = neighborList[n+2*Np]; // neighbor 4 + //fq = dist[nread]; + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + //nread = neighborList[n+3*Np]; // neighbor 3 + //fq = dist[nread]; + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + //nread = neighborList[n+4*Np]; + //fq = dist[nread]; + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + //nread = neighborList[n+5*Np]; + //fq = dist[nread]; + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + //nread = neighborList[n+6*Np]; + //fq = dist[nread]; + nr7 = neighborList[n+6*Np]; + fq = dist[nr7]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + //nread = neighborList[n+7*Np]; + //fq = dist[nread]; + nr8 = neighborList[n+7*Np]; + fq = dist[nr8]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + //nread = neighborList[n+8*Np]; + //fq = dist[nread]; + nr9 = neighborList[n+8*Np]; + fq = dist[nr9]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + //nread = neighborList[n+9*Np]; + //fq = dist[nread]; + nr10 = neighborList[n+9*Np]; + fq = dist[nr10]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + //nread = neighborList[n+10*Np]; + //fq = dist[nread]; + nr11 = neighborList[n+10*Np]; + fq = dist[nr11]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + //nread = neighborList[n+11*Np]; + //fq = dist[nread]; + nr12 = neighborList[n+11*Np]; + fq = dist[nr12]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + //nread = neighborList[n+12*Np]; + //fq = dist[nread]; + nr13 = neighborList[n+12*Np]; + fq = dist[nr13]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + //nread = neighborList[n+13*Np]; + //fq = dist[nread]; + nr14 = neighborList[n+13*Np]; + fq = dist[nr14]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*porosity*Gx; + vy = jy/rho0+0.5*porosity*Gy; + vz = jz/rho0+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=rho0*Gx; + Fy=rho0*Gy; + Fz=rho0*Gz; + } + + //Calculate pressure for MRT model + pressure=rho/3.f; + + //-------------------- MRT collison where body force has NO higher-order terms -------------// + m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) - m9); + m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) - m11); + m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) - m13); + m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) - m14); + m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + //nread = neighborList[n+7*Np]; + dist[nr8] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + //nread = neighborList[n+6*Np]; + dist[nr7] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + //nread = neighborList[n+9*Np]; + dist[nr10] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + //nread = neighborList[n+8*Np]; + dist[nr9] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + //nread = neighborList[n+11*Np]; + dist[nr12] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + //nread = neighborList[n+10*Np]; + dist[nr11]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + //nread = neighborList[n+13*Np]; + dist[nr14] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + //nread = neighborList[n+12*Np]; + dist[nr13] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz, + double *Poros,double *Perm, double *Velocity,double rho0, double *Pressure){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double rho,jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(neighborList,dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,rho0,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Greyscale_MRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,double *Poros,double *Perm, double *Velocity,double rho0,double *Pressure){ + + dvc_ScaLBL_D3Q19_AAeven_Greyscale_MRT<<>>(dist,start,finish,Np,rlx,rlx_eff,Fx,Fy,Fz,Poros,Perm,Velocity,rho0,Pressure); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Greyscale_MRT: %s \n",cudaGetErrorString(err)); + } +} + extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); cudaError_t err = cudaGetLastError(); diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 08cf09b5..9ca98cd5 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -44,7 +44,7 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ din=dout=1.0; flux=0.0; dp = 10.0; //unit of 'dp': voxel - CollisionType = 1; //1: IMRT; 2: BGK + CollisionType = 1; //1: IMRT; 2: BGK; 3: MRT // ---------------------- Greyscale Model parameters -----------------------// if (greyscale_db->keyExists( "timestepMax" )){ @@ -84,6 +84,9 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ if (collision == "BGK"){ CollisionType=2; } + else if (collision == "MRT"){ + CollisionType=3; + } // ------------------------------------------------------------------------// //------------------------ Other Domain parameters ------------------------// @@ -360,6 +363,10 @@ void ScaLBL_GreyscaleModel::Initialize(){ ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf("Collision model: BGK.\n"); } + else if (CollisionType==3){ + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf("Collision model: MRT.\n"); + } else{ if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den); @@ -441,6 +448,9 @@ void ScaLBL_GreyscaleModel::Run(){ case 2: ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; + case 3: + ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; default: ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; @@ -459,6 +469,9 @@ void ScaLBL_GreyscaleModel::Run(){ case 2: ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; + case 3: + ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; default: ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; @@ -475,6 +488,9 @@ void ScaLBL_GreyscaleModel::Run(){ case 2: ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; + case 3: + ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; default: ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; @@ -493,6 +509,9 @@ void ScaLBL_GreyscaleModel::Run(){ case 2: ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc); break; + case 3: + ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); + break; default: ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc); break; diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index a54b6fc4..de815906 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -54,7 +54,7 @@ int main(int argc, char **argv) Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables Greyscale.Initialize(); // initializing the model will set initial conditions for variables Greyscale.Run(); - //Greyscale.VelocityField(); + Greyscale.VelocityField(); //Greyscale.WriteDebug(); } // **************************************************** From 534410bb2970783ab233bf543ec1e2ce26212516 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 24 Jun 2020 13:48:53 -0400 Subject: [PATCH 165/270] add Minkowski to TestTorus --- tests/TestTorus.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index 2d486774..d3cffa05 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -63,6 +63,8 @@ int main(int argc, char **argv) auto Dm = std::make_shared(db,comm); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); auto Averages = std::make_shared(Dm); + auto Object = std::make_shared(Dm); + Nx += 2; Ny += 2; Nz += 2; @@ -124,9 +126,11 @@ int main(int argc, char **argv) //Averages->Phase(i,j,k) = - Averages->Phase(i,j,k); if (Averages->Phase(i,j,k) > 0.0){ Dm->id[n] = 2; + Object->id(i,j,k) = 1; } else{ Dm->id[n] = 1; + Object->id(i,j,k) = 0; } Averages->SDn(i,j,k) = Averages->Phase(i,j,k); Averages->Phase(i,j,k) = Averages->SDn(i,j,k); @@ -163,6 +167,17 @@ int main(int argc, char **argv) Averages->Reduce(); Averages->PrintAll(int(5)); // Averages->Reduce(); + + Object->MeasureObject(); + double Vi = Object.V(); + double Ai = Object.A(); + double Hi = Object.H(); + double Xi = Object.X(); + Vi=sumReduce( Dm->Comm, Vi); + Ai=sumReduce( Dm->Comm, Ai); + Hi=sumReduce( Dm->Comm, Hi); + Xi=sumReduce( Dm->Comm, Xi); + printf("Vi=%f, Ai=%f, Hi=%f, Xi=%f \n", Vi,Ai,Hi,Xi); } // Limit scope so variables that contain communicators will free before MPI_Finialize MPI_Barrier(comm); From 1a5b46156dc1569024870eadf916bb783a08cc3e Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 24 Jun 2020 14:01:10 -0400 Subject: [PATCH 166/270] fix Torus --- analysis/Minkowski.cpp | 2 +- tests/TestTorus.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 9652db35..e98cfdc7 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -88,7 +88,7 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) // printf(" (%i,%i,%i) RP(%i,%i)={%f,%f,%f} {%f,%f,%f} a=%f l=%f \n",i,j,k,e3,object.halfedge.twin(e3),P3.x,P3.y,P3.z,P1.x,P1.y,P1.z,a3,s3); //} // Euler characteristic (half edge rule: one face - 0.5*(three edges)) - Xi -= 1.5; + Xi -= 0.5; } // Euler characteristic -- each vertex shared by four cubes Xi += 0.25*double(object.VertexCount); diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index d3cffa05..0cad888a 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -169,10 +169,10 @@ int main(int argc, char **argv) // Averages->Reduce(); Object->MeasureObject(); - double Vi = Object.V(); - double Ai = Object.A(); - double Hi = Object.H(); - double Xi = Object.X(); + double Vi = Object->V(); + double Ai = Object->A(); + double Hi = Object->H(); + double Xi = Object->X(); Vi=sumReduce( Dm->Comm, Vi); Ai=sumReduce( Dm->Comm, Ai); Hi=sumReduce( Dm->Comm, Hi); From 9a599b504f42ebce69d57be6eaa8780d1c7ac104 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 24 Jun 2020 21:45:08 -0400 Subject: [PATCH 167/270] fix typo --- common/Domain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 1274d80d..ab57ea4e 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -224,7 +224,7 @@ void Domain::initialize( std::shared_ptr db ) if (rank_info.kz < nproc[2]-1) outlet_layers_z = 0; // Fill remaining variables N = Nx*Ny*Nz; - Volume = nx*ny*nx*nproc[0]*nproc[1]*nproc[2]*1.0; + Volume = nx*ny*nz*nproc[0]*nproc[1]*nproc[2]*1.0; if (myrank==0) printf("voxel length = %f micron \n", voxel_length); From 7aad36e37a3974c134f0fb68dee7275fcdbbe758 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 24 Jun 2020 22:07:21 -0400 Subject: [PATCH 168/270] CPU also available, make greyscaleColor equivalent single-phase model available --- cpu/Greyscale.cpp | 1033 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1033 insertions(+) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index affe6c2b..a2395e61 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1566,6 +1566,1039 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dis } +extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz,double *Poros,double *Perm, double *Velocity,double rho0,double *Pressure){ + + int n, nread; + int nr1,nr2,nr3,nr4,nr5,nr6; + int nr7,nr8,nr9,nr10; + int nr11,nr12,nr13,nr14; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double rho,jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + for (int n=start; n even part of dist) + //fq = dist[nread]; // reading the f2 data into register fq + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nr2]; // reading the f2 data into register fq + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + //nread = neighborList[n+2*Np]; // neighbor 4 + //fq = dist[nread]; + nr3 = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nr3]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + //nread = neighborList[n+3*Np]; // neighbor 3 + //fq = dist[nread]; + nr4 = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nr4]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + //nread = neighborList[n+4*Np]; + //fq = dist[nread]; + nr5 = neighborList[n+4*Np]; + fq = dist[nr5]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + //nread = neighborList[n+5*Np]; + //fq = dist[nread]; + nr6 = neighborList[n+5*Np]; + fq = dist[nr6]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + //nread = neighborList[n+6*Np]; + //fq = dist[nread]; + nr7 = neighborList[n+6*Np]; + fq = dist[nr7]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + //nread = neighborList[n+7*Np]; + //fq = dist[nread]; + nr8 = neighborList[n+7*Np]; + fq = dist[nr8]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + //nread = neighborList[n+8*Np]; + //fq = dist[nread]; + nr9 = neighborList[n+8*Np]; + fq = dist[nr9]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + //nread = neighborList[n+9*Np]; + //fq = dist[nread]; + nr10 = neighborList[n+9*Np]; + fq = dist[nr10]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + //nread = neighborList[n+10*Np]; + //fq = dist[nread]; + nr11 = neighborList[n+10*Np]; + fq = dist[nr11]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + //nread = neighborList[n+11*Np]; + //fq = dist[nread]; + nr12 = neighborList[n+11*Np]; + fq = dist[nr12]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + //nread = neighborList[n+12*Np]; + //fq = dist[nread]; + nr13 = neighborList[n+12*Np]; + fq = dist[nr13]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + //nread = neighborList[n+13*Np]; + //fq = dist[nread]; + nr14 = neighborList[n+13*Np]; + fq = dist[nr14]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + //---------------------------------------------------------------------// + + porosity = Poros[n]; + perm = Perm[n]; + + c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); + if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes + GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); + c1 = porosity*0.5*GeoFun/sqrt(perm); + if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes + + vx = jx/rho0+0.5*porosity*Gx; + vy = jy/rho0+0.5*porosity*Gy; + vz = jz/rho0+0.5*porosity*Gz; + v_mag=sqrt(vx*vx+vy*vy+vz*vz); + ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); + uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); + uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); + u_mag=sqrt(ux*ux+uy*uy+uz*uz); + + //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium + Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); + Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); + Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); + if (porosity==1.0){ + Fx=rho0*Gx; + Fy=rho0*Gy; + Fz=rho0*Gz; + } + + //Calculate pressure for MRT model + pressure=rho/3.f; + + //-------------------- MRT collison where body force has NO higher-order terms -------------// + m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity) - m2); + jx = jx + Fx; + m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + jy = jy + Fy; + m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + jz = jz + Fz; + m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) - m9); + m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); + m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) - m11); + m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); + m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) - m13); + m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) - m14); + m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + + + //.................inverse transformation...................................................... + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); + //nread = neighborList[n+Np]; + dist[nr2] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); + //nread = neighborList[n]; + dist[nr1] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+3*Np]; + dist[nr4] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); + //nread = neighborList[n+2*Np]; + dist[nr3] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+5*Np]; + dist[nr6] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); + //nread = neighborList[n+4*Np]; + dist[nr5] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); + //nread = neighborList[n+7*Np]; + dist[nr8] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16); + //nread = neighborList[n+6*Np]; + dist[nr7] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); + //nread = neighborList[n+9*Np]; + dist[nr10] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ + mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); + //nread = neighborList[n+8*Np]; + dist[nr9] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16); + //nread = neighborList[n+11*Np]; + dist[nr12] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ + mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); + //nread = neighborList[n+10*Np]; + dist[nr11]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18); + //nread = neighborList[n+13*Np]; + dist[nr14] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18); + //nread = neighborList[n+12*Np]; + dist[nr13] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + //........................................................................ + + //Update velocity on device + Velocity[0*Np+n] = ux; + Velocity[1*Np+n] = uy; + Velocity[2*Np+n] = uz; + //Update pressure on device + Pressure[n] = pressure; + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Gx, double Gy, double Gz,double *Poros,double *Perm, double *Velocity,double rho0,double *Pressure){ + + int n; + double vx,vy,vz,v_mag; + double ux,uy,uz,u_mag; + double pressure;//defined for this incompressible model + // conserved momemnts + double rho,jx,jy,jz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + double fq; + //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double GeoFun;//geometric function from Guo's PRE 66, 036304 (2002) + double porosity; + double perm;//voxel permeability + double c0, c1; //Guo's model parameters + double mu_eff = (1.0/rlx_eff-0.5)/3.0;//kinematic viscosity + double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) + double rlx_setA = rlx; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + + for (int n=start; n Date: Wed, 24 Jun 2020 23:33:36 -0400 Subject: [PATCH 169/270] fix volume bug --- common/Domain.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 3dec0128..eadc4592 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -232,7 +232,7 @@ void Domain::initialize( std::shared_ptr db ) if (rank_info.kz < nproc[2]-1) outlet_layers_z = 0; // Fill remaining variables N = Nx*Ny*Nz; - Volume = nx*ny*nx*nproc[0]*nproc[1]*nproc[2]*1.0; + Volume = nx*ny*nz*nproc[0]*nproc[1]*nproc[2]*1.0; if (myrank==0) printf("voxel length = %f micron \n", voxel_length); @@ -266,8 +266,8 @@ void Domain::Decomp( const std::string& Filename ) outlet_layers_x = 0; outlet_layers_y = 0; outlet_layers_z = 0; - inlet_layers_phase=1; - outlet_layers_phase=2; + inlet_layers_phase=1; + outlet_layers_phase=2; checkerSize = 32; // Read domain parameters From 67e8f94574a6089a02b4645c6edabc33dbe1be3e Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Fri, 26 Jun 2020 10:39:45 -0400 Subject: [PATCH 170/270] update minkowski --- analysis/Minkowski.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index e98cfdc7..faac6142 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=0; k Date: Fri, 26 Jun 2020 12:48:32 -0400 Subject: [PATCH 171/270] add stl writer to dcel --- analysis/dcel.cpp | 19 +++++++++++++++++++ analysis/dcel.h | 1 + 2 files changed, 20 insertions(+) diff --git a/analysis/dcel.cpp b/analysis/dcel.cpp index ca21c0e6..9ca8921c 100644 --- a/analysis/dcel.cpp +++ b/analysis/dcel.cpp @@ -15,6 +15,25 @@ int DECL::Face(int index){ return FaceData[index]; } +void DECL::Write(){ + int e1,e2,e3; + FILE *TRIANGLES; + TRIANGLES = fopen("triangles.stl","w"); + fprintf("solid \n"); + for (int idx=0; idx Date: Fri, 26 Jun 2020 13:09:30 -0400 Subject: [PATCH 172/270] fix dcel write --- analysis/dcel.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/analysis/dcel.cpp b/analysis/dcel.cpp index 9ca8921c..01e7b571 100644 --- a/analysis/dcel.cpp +++ b/analysis/dcel.cpp @@ -19,17 +19,17 @@ void DECL::Write(){ int e1,e2,e3; FILE *TRIANGLES; TRIANGLES = fopen("triangles.stl","w"); - fprintf("solid \n"); + fprintf(TRIANGLES,"solid \n"); for (int idx=0; idx Date: Fri, 26 Jun 2020 15:53:23 -0400 Subject: [PATCH 173/270] stl writer for dcel --- analysis/dcel.cpp | 270 ++++++---------------------------------------- analysis/dcel.h | 7 ++ 2 files changed, 38 insertions(+), 239 deletions(-) diff --git a/analysis/dcel.cpp b/analysis/dcel.cpp index 01e7b571..71f69951 100644 --- a/analysis/dcel.cpp +++ b/analysis/dcel.cpp @@ -1,7 +1,5 @@ #include "analysis/dcel.h" - - DECL::DECL(){ } @@ -369,243 +367,37 @@ double DECL::EdgeAngle(int edge) return angle; } -void Isosurface(DoubleArray &A, const double &v) +void iso_surface(const Array&Field, const double isovalue) { - NULL_USE( v ); - - Point P,Q; - Point PlaceHolder; - Point C0,C1,C2,C3,C4,C5,C6,C7; - - int TriangleCount; - int VertexCount; - int CubeIndex; - - Point VertexList[12]; - Point NewVertexList[12]; - int LocalRemap[12]; - - Point cellvertices[20]; - std::array,20> Triangles; - Triangles.fill( { 0 } ); - - // Values from array 'A' at the cube corners - double CubeValues[8]; - - int Nx = A.size(0); - int Ny = A.size(1); - int Nz = A.size(2); - - // Points corresponding to cube corners - C0.x = 0.0; C0.y = 0.0; C0.z = 0.0; - C1.x = 1.0; C1.y = 0.0; C1.z = 0.0; - C2.x = 1.0; C2.y = 1.0; C2.z = 0.0; - C3.x = 0.0; C3.y = 1.0; C3.z = 0.0; - C4.x = 0.0; C4.y = 0.0; C4.z = 1.0; - C5.x = 1.0; C5.y = 0.0; C5.z = 1.0; - C6.x = 1.0; C6.y = 1.0; C6.z = 1.0; - C7.x = 0.0; C7.y = 1.0; C7.z = 1.0; - - std::vector> HalfEdge; - for (int k=1; kV2 - HalfEdge[idx_edge][0] = V1; // first vertex - HalfEdge[idx_edge][1] = V2; // second vertex - HalfEdge[idx_edge][2] = idx; // triangle - HalfEdge[idx_edge][3] = -1; // twin - HalfEdge[idx_edge][4] = idx_edge+2; // previous edge - HalfEdge[idx_edge][5] = idx_edge+1; // next edge - idx_edge++; - // second edge: V2->V3 - HalfEdge[idx_edge][0] = V2; // first vertex - HalfEdge[idx_edge][1] = V3; // second vertex - HalfEdge[idx_edge][2] = idx; // triangle - HalfEdge[idx_edge][3] = -1; // twin - HalfEdge[idx_edge][4] = idx_edge-1; // previous edge - HalfEdge[idx_edge][5] = idx_edge+1; // next edge - idx_edge++; - // third edge: V3->V1 - HalfEdge[idx_edge][0] = V3; // first vertex - HalfEdge[idx_edge][1] = V1; // second vertex - HalfEdge[idx_edge][2] = idx; // triangle - HalfEdge[idx_edge][3] = -1; // twin - HalfEdge[idx_edge][4] = idx_edge-1; // previous edge - HalfEdge[idx_edge][5] = idx_edge-2; // next edge - idx_edge++; - } - int EdgeCount=idx_edge; - for (int idx=0; idx #include "analysis/pmmc.h" @@ -79,3 +82,7 @@ public: private: std::vector FaceData; }; + +void iso_surface(const Array&Field, const double isovalue); + +#endif From d89dcf2648d0b9e23d786bb6d3f8cc0d380111f3 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 27 Jun 2020 07:28:48 -0400 Subject: [PATCH 174/270] Eikonal solver in distance --- analysis/distance.cpp | 141 ++++++++++++++++++++++++++++++++++++++++++ analysis/distance.h | 11 ++++ 2 files changed, 152 insertions(+) diff --git a/analysis/distance.cpp b/analysis/distance.cpp index e297b435..78c344d2 100644 --- a/analysis/distance.cpp +++ b/analysis/distance.cpp @@ -182,6 +182,147 @@ void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, } } +double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ + + /* + * This routine converts the data in the Distance array to a signed distance + * by solving the equation df/dt = sign(1-|grad f|), where Distance provides + * the values of f on the mesh associated with domain Dm + * It has been tested with segmented data initialized to values [-1,1] + * and will converge toward the signed distance to the surface bounding the associated phases + * + * Reference: + * Min C (2010) On reinitializing level set functions, Journal of Computational Physics229 + */ + + int i,j,k; + double dt=0.1; + double Dx,Dy,Dz; + double Dxp,Dxm,Dyp,Dym,Dzp,Dzm; + double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm; + double sign,norm; + double LocalVar,GlobalVar,LocalMax,GlobalMax; + + int xdim,ydim,zdim; + xdim=Dm.Nx-2; + ydim=Dm.Ny-2; + zdim=Dm.Nz-2; + fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + + // Arrays to store the second derivatives + DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz); + DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz); + DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz); + + int count = 0; + while (count < timesteps){ + + // Communicate the halo of values + fillData.fill(Distance); + + // Compute second order derivatives + for (k=1;k 0.f) Dx = Dxp*Dxp; + elseDx = Dxm*Dxm; + + if (Dyp + Dym > 0.f) Dy = Dyp*Dyp; + elseDy = Dym*Dym; + + if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp; + elseDz = Dzm*Dzm; + } + else{ + + if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp; + elseDx = Dxm*Dxm; + + if (Dyp + Dym < 0.f) Dy = Dyp*Dyp; + elseDy = Dym*Dym; + + if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp; + elseDz = Dzm*Dzm; + } + + //Dx = max(Dxp*Dxp,Dxm*Dxm); + //Dy = max(Dyp*Dyp,Dym*Dym); + //Dz = max(Dzp*Dzp,Dzm*Dzm); + + norm=sqrt(Dx + Dy + Dz); + if (norm > 1.0) norm=1.0; + + Distance(i,j,k) += dt*sign*(1.0 - norm); + LocalVar += dt*sign*(1.0 - norm); + + if (fabs(dt*sign*(1.0 - norm)) > LocalMax) + LocalMax = fabs(dt*sign*(1.0 - norm)); + } + } + } + + MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); + GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz; + count++; + + if (count%50 == 0 && Dm.rank==0 ) + printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); + + if (fabs(GlobalMax) < 1e-5){ + if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n"); + count=timesteps; + } + } + return GlobalVar; +} // Explicit instantiations template void CalcDist( Array&, const Array&, const Domain&, const std::array&, const std::array& ); diff --git a/analysis/distance.h b/analysis/distance.h index b3fc870e..5ca00c83 100644 --- a/analysis/distance.h +++ b/analysis/distance.h @@ -40,4 +40,15 @@ void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic = {true,true,true}, const std::array& dx = {1,1,1} ); + +/*! + * @brief Calculate the distance based on solution of Eikonal equation + * @details This routine calculates the signed distance to the nearest domain surface. + * @param[out] Distance Distance function + * @param[in] ID Domain id + * @param[in] Dm Domain information + * @param[in] timesteps number of timesteps to run for Eikonal solver + */ +double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps); + #endif From 7cdfd4006ab122396283e4819991c2a989a7c635 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 27 Jun 2020 07:30:46 -0400 Subject: [PATCH 175/270] Eikonal solver in distance --- analysis/distance.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/analysis/distance.cpp b/analysis/distance.cpp index 78c344d2..3b7641d6 100644 --- a/analysis/distance.cpp +++ b/analysis/distance.cpp @@ -272,24 +272,24 @@ double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ // Compute upwind derivatives for Godunov Hamiltonian if (sign < 0.0){ if (Dxp + Dxm > 0.f) Dx = Dxp*Dxp; - elseDx = Dxm*Dxm; + else Dx = Dxm*Dxm; if (Dyp + Dym > 0.f) Dy = Dyp*Dyp; - elseDy = Dym*Dym; + else Dy = Dym*Dym; if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp; - elseDz = Dzm*Dzm; + else Dz = Dzm*Dzm; } else{ if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp; - elseDx = Dxm*Dxm; + else Dx = Dxm*Dxm; if (Dyp + Dym < 0.f) Dy = Dyp*Dyp; - elseDy = Dym*Dym; + else Dy = Dym*Dym; if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp; - elseDz = Dzm*Dzm; + else Dz = Dzm*Dzm; } //Dx = max(Dxp*Dxp,Dxm*Dxm); From 3d58822fef7bd2d1a3ca58dc534e45de13c42091 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 27 Jun 2020 07:43:26 -0400 Subject: [PATCH 176/270] Eikonal solver in distance --- analysis/distance.cpp | 11 ++++++----- analysis/distance.h | 13 ++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/analysis/distance.cpp b/analysis/distance.cpp index 3b7641d6..be4cbac6 100644 --- a/analysis/distance.cpp +++ b/analysis/distance.cpp @@ -182,7 +182,7 @@ void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, } } -double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ +double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps, const std::array& periodic){ /* * This routine converts the data in the Distance array to a signed distance @@ -207,7 +207,8 @@ double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ xdim=Dm.Nx-2; ydim=Dm.Ny-2; zdim=Dm.Nz-2; - fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + //fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + fillHalo fillData( Dm.Comm, Dm.rank_info, {xdim, ydim, zdim}, {1,1,1}, 50, 1, {true,true,true}, periodic ); // Arrays to store the second derivatives DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz); @@ -310,14 +311,14 @@ double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); - GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz; + GlobalVar /= Dm.Volume; count++; - if (count%50 == 0 && Dm.rank==0 ) + if (count%50 == 0 && Dm.rank()==0 ) printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); if (fabs(GlobalMax) < 1e-5){ - if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n"); + if (Dm.rank()==0) printf("Exiting with max tolerance of 1e-5 \n"); count=timesteps; } } diff --git a/analysis/distance.h b/analysis/distance.h index 5ca00c83..b382a49f 100644 --- a/analysis/distance.h +++ b/analysis/distance.h @@ -16,6 +16,16 @@ struct Vec { }; inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; } +inline double minmod(double &a, double &b){ + + double value; + + value = a; + if ( a*b < 0.0) value=0.0; + else if (fabs(a) > fabs(b)) value = b; + + return value; +} /*! * @brief Calculate the distance using a simple method @@ -48,7 +58,8 @@ void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, * @param[in] ID Domain id * @param[in] Dm Domain information * @param[in] timesteps number of timesteps to run for Eikonal solver + * @param[in] periodic Directions that are periodic */ -double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps); +double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps, const std::array& periodic); #endif From a16d82bac0c5d42afa1138fbfe8bf22ceeb47511 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 27 Jun 2020 07:50:03 -0400 Subject: [PATCH 177/270] Eikonal solver in distance --- analysis/distance.cpp | 4 ++-- analysis/distance.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/distance.cpp b/analysis/distance.cpp index be4cbac6..fd48f7c7 100644 --- a/analysis/distance.cpp +++ b/analysis/distance.cpp @@ -182,7 +182,7 @@ void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, } } -double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps, const std::array& periodic){ +double Eikonal(DoubleArray &Distance, const Array &ID, Domain &Dm, int timesteps, const std::array& periodic){ /* * This routine converts the data in the Distance array to a signed distance @@ -244,7 +244,7 @@ double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps, const int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i; sign = -1; - if (ID[n] == 1) sign = 1; + if (ID(i,j,k) == 1) sign = 1; // local second derivative terms Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k)); diff --git a/analysis/distance.h b/analysis/distance.h index b382a49f..d6c2740c 100644 --- a/analysis/distance.h +++ b/analysis/distance.h @@ -60,6 +60,6 @@ void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, * @param[in] timesteps number of timesteps to run for Eikonal solver * @param[in] periodic Directions that are periodic */ -double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps, const std::array& periodic); +double Eikonal(DoubleArray &Distance, const Array &ID, Domain &Dm, int timesteps, const std::array& periodic); #endif From 3dd7ba5936c60676e438c67e8f1bae4bbd2430d7 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Sat, 27 Jun 2020 07:56:53 -0400 Subject: [PATCH 178/270] refine dist in Minkowski (fix isosurface bugs) --- analysis/Minkowski.cpp | 1 + analysis/dcel.cpp | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index faac6142..476f208c 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -137,6 +137,7 @@ void Minkowski::MeasureObject(){ } } CalcDist(distance,id,*Dm); + Eikonal(distance, id, *Dm, 10, {true, true, true}); ComputeScalar(distance,0.0); } diff --git a/analysis/dcel.cpp b/analysis/dcel.cpp index 71f69951..91102acf 100644 --- a/analysis/dcel.cpp +++ b/analysis/dcel.cpp @@ -388,16 +388,22 @@ void iso_surface(const Array&Field, const double isovalue) auto P1 = object.vertex.coords(object.halfedge.v1(e1)); auto P2 = object.vertex.coords(object.halfedge.v1(e2)); auto P3 = object.vertex.coords(object.halfedge.v1(e3)); - P1.x += 1.0*i; P1.y += 1.0*j; P1.z +=1.0*k; - P2.x += 1.0*i; P2.y += 1.0*j; P2.z +=1.0*k; - P3.x += 1.0*i; P3.y += 1.0*j; P3.z +=1.0*k; - fprintf(TRIANGLES,"vertex %f %f %f\n",P1.x,P1.y,P1.z); - fprintf(TRIANGLES,"vertex %f %f %f\n",P2.x,P2.y,P2.z); - fprintf(TRIANGLES,"vertex %f %f %f\n",P3.x,P3.y,P3.z); + auto Normal = object.TriNormal(e1); + // P1.x += 1.0*i; P1.y += 1.0*j; P1.z +=1.0*k; + //P2.x += 1.0*i; P2.y += 1.0*j; P2.z +=1.0*k; + //P3.x += 1.0*i; P3.y += 1.0*j; P3.z +=1.0*k; + fprintf(TRIANGLES,"facet normal %f %f %f\n",Normal.x,Normal.y,Normal.z); + fprintf(TRIANGLES," outer loop\n"); + fprintf(TRIANGLES," vertex %f %f %f\n",P1.x,P1.y,P1.z); + fprintf(TRIANGLES," vertex %f %f %f\n",P2.x,P2.y,P2.z); + fprintf(TRIANGLES," vertex %f %f %f\n",P3.x,P3.y,P3.z); + fprintf(TRIANGLES," endloop\n"); + fprintf(TRIANGLES,"endfacet\n"); } } } } + fprintf(TRIANGLES,"endsolid isosurface\n"); fclose(TRIANGLES); } From 07ea37d2f246b2d0fbabe6321e3716af3d229aba Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 27 Jun 2020 15:16:13 -0400 Subject: [PATCH 179/270] at device test --- tests/CMakeLists.txt | 1 + tests/TestSetDevice.cpp | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/TestSetDevice.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8b14a9dc..7e703c9d 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -70,6 +70,7 @@ ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) ADD_LBPM_TEST_1_2_4( testCommunication ) ADD_LBPM_TEST( TestWriter ) ADD_LBPM_TEST( TestDatabase ) +ADD_LBPM_TEST( TestSetDevice ) ADD_LBPM_PROVISIONAL_TEST( TestMicroCTReader ) IF ( USE_NETCDF ) ADD_LBPM_TEST_PARALLEL( TestNetcdf 8 ) diff --git a/tests/TestSetDevice.cpp b/tests/TestSetDevice.cpp new file mode 100644 index 00000000..79627b88 --- /dev/null +++ b/tests/TestSetDevice.cpp @@ -0,0 +1,37 @@ +#include +#include "common/MPI_Helpers.h" +#include "common/Utilities.h" +#include "common/ScaLBL.h" + +int main (int argc, char **argv) +{ + MPI_Init(&argc,&argv); + int rank = MPI_WORLD_RANK(); + int nprocs = MPI_WORLD_SIZE(); + + for (int i=0; i Date: Sat, 27 Jun 2020 20:54:00 -0400 Subject: [PATCH 180/270] update cubes --- analysis/Minkowski.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 476f208c..c87ce22b 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=1; k Date: Mon, 29 Jun 2020 11:07:55 -0400 Subject: [PATCH 181/270] use internal / external edge count in Euler characteristic --- analysis/Minkowski.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index c87ce22b..16a1a6c7 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -58,9 +58,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //int Nx = Field.size(0); //int Ny = Field.size(1); //int Nz = Field.size(2); - for (int k=0; k 0) EulerChar = (0.25*nvert - nside_intern - 0.5*nside_extern + nface); */ + Xi += 0.25*double(object.VertexCount) - nside_intern - 0.5*nside_extern + double(object.TriangleCount); } } } From 736870f4b59c0fce414893b7a6e1181d9c5d6eaa Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 29 Jun 2020 13:46:24 -0400 Subject: [PATCH 182/270] still debugging Euler characteristic bug --- analysis/Minkowski.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 16a1a6c7..c4600343 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -87,13 +87,23 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) // printf(" (%i,%i,%i) QR(%i,%i)={%f,%f,%f} {%f,%f,%f} a=%f l=%f \n",i,j,k,e2,object.halfedge.twin(e2),P2.x,P2.y,P2.z,P3.x,P3.y,P3.z,a2,s2); // printf(" (%i,%i,%i) RP(%i,%i)={%f,%f,%f} {%f,%f,%f} a=%f l=%f \n",i,j,k,e3,object.halfedge.twin(e3),P3.x,P3.y,P3.z,P1.x,P1.y,P1.z,a3,s3); //} + // Euler characteristic (half edge rule: one face - 0.5*(three edges)) + Xi -= 0.5; } // Euler characteristic -- each vertex shared by four cubes - double nside_extern = double(object.VertexCount); - double nside_intern = double(object.VertexCount)-3.0; - /*EulerChar=0.0; + //Xi += 0.25*double(object.VertexCount); + // check if vertices are at corners + for (int idx=0; idx 0) EulerChar = (0.25*nvert - nside_intern - 0.5*nside_extern + nface); */ - Xi += 0.25*double(object.VertexCount) - nside_intern - 0.5*nside_extern + double(object.TriangleCount); } } } From 4d2174cedb13ccd6ae24c3f383e57139c2e31512 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 29 Jun 2020 14:18:05 -0400 Subject: [PATCH 183/270] update Euler characteristic --- analysis/Minkowski.cpp | 6 +++--- tests/TestSetDevice.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index c4600343..5b9826e6 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -95,10 +95,10 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) // check if vertices are at corners for (int idx=0; idx Date: Wed, 8 Jul 2020 22:24:37 -0400 Subject: [PATCH 184/270] 1. add color grad for debug;2. fix bug for calculating greyscale solid gradient --- common/ScaLBL.h | 4 +-- gpu/GreyscaleColor.cu | 58 +++++++++++++++++++++++----------- models/GreyscaleColorModel.cpp | 56 ++++++++++++++++---------------- models/GreyscaleColorModel.h | 2 +- 4 files changed, 71 insertions(+), 49 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index de91e790..0170461a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -167,12 +167,12 @@ extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *l //extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np); extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index ea7b5ce1..1c0a6087 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -4,7 +4,7 @@ #define NTHREADS 256 __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ @@ -33,6 +33,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma double tau_eff; double mu_eff;//kinematic viscosity double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double nx_phase,ny_phase,nz_phase,C_phase; double Fx,Fy,Fz; const double mrt_V1=0.05263157894736842; @@ -134,13 +135,23 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma nn = ijk-strideZ+strideY; // neighbor index (get convention) m18 = Phi[nn]; // get neighbor for phi - 18 //............Compute the Color Gradient................................... - nx = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + ColorGrad[n]=nx_phase; + ColorGrad[n+Np]=ny_phase; + ColorGrad[n+2*Np]=nz_phase; + C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); + //correct the normal color gradient by considering the effect of grey solid - nx += (1.0-porosity)*nx_gs; - ny += (1.0-porosity)*ny_gs; - nz += (1.0-porosity)*nz_gs; + nx = nx_phase + (1.0-porosity)*nx_gs; + ny = ny_phase + (1.0-porosity)*ny_gs; + nz = nz_phase + (1.0-porosity)*nz_gs; + if (C_phase==0.0){ + nx = nx_phase; + ny = ny_phase; + nz = nz_phase; + } //...........Normalize the Color Gradient................................. C = sqrt(nx*nx+ny*ny+nz*nz); @@ -717,7 +728,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma } __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ int ijk,nn,n; @@ -741,6 +752,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double tau_eff; double mu_eff;//kinematic viscosity double nx_gs,ny_gs,nz_gs;//grey-solid color gradient + double nx_phase,ny_phase,nz_phase,C_phase; double Fx,Fy,Fz; const double mrt_V1=0.05263157894736842; @@ -843,13 +855,23 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, nn = ijk-strideZ+strideY; // neighbor index (get convention) m18 = Phi[nn]; // get neighbor for phi - 18 //............Compute the Color Gradient................................... - nx = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + ColorGrad[n]=nx_phase; + ColorGrad[n+Np]=ny_phase; + ColorGrad[n+2*Np]=nz_phase; + C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); + //correct the normal color gradient by considering the effect of grey solid - nx += (1.0-porosity)*nx_gs; - ny += (1.0-porosity)*ny_gs; - nz += (1.0-porosity)*nz_gs; + nx = nx_phase + (1.0-porosity)*nx_gs; + ny = ny_phase + (1.0-porosity)*ny_gs; + nz = nz_phase + (1.0-porosity)*nz_gs; + if (C_phase==0.0){ + nx = nx_phase; + ny = ny_phase; + nz = nz_phase; + } //...........Normalize the Color Gradient................................. C = sqrt(nx*nx+ny*ny+nz*nz); @@ -1392,14 +1414,14 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, //} extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den,ColorGrad, Phi, GreySolidGrad, Poros, Perm, Vel, rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1410,14 +1432,14 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl } extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den,ColorGrad, Phi, GreySolidGrad, Poros, Perm,Vel, rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff,alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index c7cabe04..dcbab77e 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -119,9 +119,9 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ if (greyscaleColor_db->keyExists( "flux" )){ flux = greyscaleColor_db->getScalar( "flux" ); } - if (greyscaleColor_db->keyExists("GreySolidLabels")|| - greyscaleColor_db->keyExists("GreySolidAffinity")|| - greyscaleColor_db->keyExists("PorosityList")|| + if (greyscaleColor_db->keyExists("GreySolidLabels")&& + greyscaleColor_db->keyExists("GreySolidAffinity")&& + greyscaleColor_db->keyExists("PorosityList")&& greyscaleColor_db->keyExists("PermeabilityList")){ greyMode = true; } @@ -671,7 +671,7 @@ void ScaLBL_GreyscaleColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Porosity_dvc, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Permeability_dvc, sizeof(double)*Np); } - //ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); @@ -1018,7 +1018,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Halo exchange for phase field ScaLBL_Comm_Regular->SendHalo(Phi); if (greyMode==true){ - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } @@ -1039,7 +1039,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } @@ -1068,7 +1068,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ } ScaLBL_Comm_Regular->SendHalo(Phi); if (greyMode==true){ - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } @@ -1089,7 +1089,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } @@ -2020,25 +2020,25 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,GreySG_Z_FILE); fclose(GreySG_Z_FILE); -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); -// FILE *CGX_FILE; -// sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); -// CGX_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGX_FILE); -// fclose(CGX_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); -// FILE *CGY_FILE; -// sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); -// CGY_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGY_FILE); -// fclose(CGY_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); -// FILE *CGZ_FILE; -// sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); -// CGZ_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGZ_FILE); -// fclose(CGZ_FILE); + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); + FILE *CGX_FILE; + sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); + CGX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGX_FILE); + fclose(CGX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); + FILE *CGY_FILE; + sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); + CGY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGY_FILE); + fclose(CGY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); + FILE *CGZ_FILE; + sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); + CGZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGZ_FILE); + fclose(CGZ_FILE); } diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 64e3f1e3..b3a894bd 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -67,7 +67,7 @@ public: double *fq, *Aq, *Bq; double *Den, *Phi; double *GreySolidGrad; - //double *ColorGrad; + double *ColorGrad; double *Velocity; double *Pressure; double *Porosity_dvc; From a1f337591124aa2dde67cd259c62505d9feba774 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 22 Jul 2020 11:57:51 -0400 Subject: [PATCH 185/270] added mean filter --- analysis/filters.cpp | 31 +++++++++++++++++++++++++++++++ analysis/filters.h | 8 +++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/analysis/filters.cpp b/analysis/filters.cpp index c5858560..8c532712 100644 --- a/analysis/filters.cpp +++ b/analysis/filters.cpp @@ -2,6 +2,37 @@ #include "math.h" #include "ProfilerApp.h" +void Mean3D( const Array &Input, Array &Output ) +{ + PROFILE_START("Mean3D"); + // Perform a 3D Mean filter on Input array + int i,j,k,ii,jj,kk; + int imin,jmin,kmin,imax,jmax,kmax; + + float *List; + List=new float[27]; + + int Nx = int(Input.size(0)); + int Ny = int(Input.size(1)); + int Nz = int(Input.size(2)); + + for (k=1; k &Input, Array &Output ) { diff --git a/analysis/filters.h b/analysis/filters.h index 131b2b9f..250ba23a 100644 --- a/analysis/filters.h +++ b/analysis/filters.h @@ -4,6 +4,13 @@ #include "common/Array.h" +/*! + * @brief Filter image + * @details This routine performs a mean filter + * @param[in] Input Input image + * @param[out] Output Output image + */ +void Mean3D( const Array &Input, Array &Output ) /*! * @brief Filter image @@ -13,7 +20,6 @@ */ void Med3D( const Array &Input, Array &Output ); - /*! * @brief Filter image * @details This routine performs a non-linear local means filter From c9640c46ba54699bba7d4af39a235719bb496b4e Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 22 Jul 2020 13:14:07 -0400 Subject: [PATCH 186/270] fix smooth distance --- analysis/Minkowski.cpp | 7 ++++--- analysis/Minkowski.h | 1 + analysis/filters.cpp | 8 ++------ analysis/filters.h | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 5b9826e6..7707286f 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -140,7 +140,7 @@ void Minkowski::MeasureObject(){ * 0 - labels the object * 1 - labels the rest of the */ - + DoubleArray smooth_distance(Nx,Ny,Nz); for (int k=0; k &Input, Array &Output ) +void Mean3D( const Array &Input, Array &Output ) { PROFILE_START("Mean3D"); // Perform a 3D Mean filter on Input array - int i,j,k,ii,jj,kk; - int imin,jmin,kmin,imax,jmax,kmax; - - float *List; - List=new float[27]; + int i,j,k; int Nx = int(Input.size(0)); int Ny = int(Input.size(1)); diff --git a/analysis/filters.h b/analysis/filters.h index 250ba23a..404fb265 100644 --- a/analysis/filters.h +++ b/analysis/filters.h @@ -10,7 +10,7 @@ * @param[in] Input Input image * @param[out] Output Output image */ -void Mean3D( const Array &Input, Array &Output ) +void Mean3D( const Array &Input, Array &Output ); /*! * @brief Filter image From dd4903155213e5ccc119dda1268ea29eaec3b9fe Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Wed, 22 Jul 2020 20:31:09 -0400 Subject: [PATCH 187/270] fix smooth distance --- analysis/Minkowski.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 7707286f..c45f2a58 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -140,7 +140,7 @@ void Minkowski::MeasureObject(){ * 0 - labels the object * 1 - labels the rest of the */ - DoubleArray smooth_distance(Nx,Ny,Nz); + //DoubleArray smooth_distance(Nx,Ny,Nz); for (int k=0; k Date: Wed, 22 Jul 2020 20:34:43 -0400 Subject: [PATCH 188/270] fix smooth distance --- analysis/Minkowski.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index c45f2a58..9e6e0f43 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -151,7 +151,7 @@ void Minkowski::MeasureObject(){ CalcDist(distance,id,*Dm); //Mean3D(distance,smooth_distance); Eikonal(distance, id, *Dm, 20, {true, true, true}); - ComputeScalar(smooth_distance,0.0); + ComputeScalar(distance,0.0); } From 80c7afc27c1c34f53a66ab0ad94f5d589b288ab9 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 23 Jul 2020 10:44:43 -0400 Subject: [PATCH 189/270] save the work; make several greynode wettability models available; need more validation and tests --- common/ScaLBL.h | 14 +- gpu/GreyscaleColor.cu | 1490 +++++++++++++++++++++++++++++++- models/GreyscaleColorModel.cpp | 779 +++++++++++++---- models/GreyscaleColorModel.h | 5 +- 4 files changed, 2082 insertions(+), 206 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 0170461a..2c674fb6 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -166,13 +166,23 @@ extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *l // GREYSCALE COLOR MODEL (Two-component) //extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np); +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, +// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, +// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, +// double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, +// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); + extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 1c0a6087..9ac4ce00 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -1,10 +1,12 @@ #include +#include #define NBLOCKS 1024 #define NTHREADS 256 +//Model-1 __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ @@ -57,6 +59,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma // read the component number densities nA = Den[n]; nB = Den[Np + n]; + porosity = Poros[n]; perm = Perm[n]; nx_gs = GreySolidGrad[n+0*Np]; @@ -138,9 +141,6 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); - ColorGrad[n]=nx_phase; - ColorGrad[n+Np]=ny_phase; - ColorGrad[n+2*Np]=nz_phase; C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); //correct the normal color gradient by considering the effect of grey solid @@ -727,8 +727,9 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma } } +//Model-1 __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ int ijk,nn,n; @@ -858,9 +859,6 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); - ColorGrad[n]=nx_phase; - ColorGrad[n+Np]=ny_phase; - ColorGrad[n+2*Np]=nz_phase; C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); //correct the normal color gradient by considering the effect of grey solid @@ -1369,6 +1367,1435 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, } } +////Model-2&3 +//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, +// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, +// double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ +// +// int n,nn,ijk,nread; +// int nr1,nr2,nr3,nr4,nr5,nr6; +// int nr7,nr8,nr9,nr10; +// int nr11,nr12,nr13,nr14; +// //int nr15,nr16,nr17,nr18; +// double fq; +// // conserved momemnts +// double rho,jx,jy,jz; +// double vx,vy,vz,v_mag; +// double ux,uy,uz,u_mag; +// // non-conserved moments +// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; +// double m3,m5,m7; +// double t1,t2,t4,t6,t8,t9,t10,t11,t12,t13,t14,t15,t16,t17,t18; +// double t3,t5,t7; +// double nA,nB; // number density +// double a1,b1,a2,b2,nAB,delta; +// double C,nx,ny,nz; //color gradient magnitude and direction +// double phi,tau,rho0,rlx_setA,rlx_setB; +// +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double c0, c1; //Guo's model parameters +// double tau_eff; +// double mu_eff;//kinematic viscosity +// double nx_phase,ny_phase,nz_phase,C_phase; +// double Fx,Fy,Fz; +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s1.0) t1 =((t1>0.0)-(t1<0.0))*(1.0-fabs(t1))+t1; +// //........................................................................ +// nn = ijk+1; // neighbor index (get convention) +// m2 = Phi[nn]; // get neighbor for phi - 2 +// t2 = m2+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t2)>1.0) t2 =((t2>0.0)-(t2<0.0))*(1.0-fabs(t2))+t2; +// //........................................................................ +// nn = ijk-strideY; // neighbor index (get convention) +// m3 = Phi[nn]; // get neighbor for phi - 3 +// t3 = m3+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t3)>1.0) t3 =((t3>0.0)-(t3<0.0))*(1.0-fabs(t3))+t3; +// //........................................................................ +// nn = ijk+strideY; // neighbor index (get convention) +// m4 = Phi[nn]; // get neighbor for phi - 4 +// t4 = m4+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t4)>1.0) t4 =((t4>0.0)-(t4<0.0))*(1.0-fabs(t4))+t4; +// //........................................................................ +// nn = ijk-strideZ; // neighbor index (get convention) +// m5 = Phi[nn]; // get neighbor for phi - 5 +// t5 = m5+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t5)>1.0) t5 =((t5>0.0)-(t5<0.0))*(1.0-fabs(t5))+t5; +// //........................................................................ +// nn = ijk+strideZ; // neighbor index (get convention) +// m6 = Phi[nn]; // get neighbor for phi - 6 +// t6 = m6+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t6)>1.0) t6 =((t6>0.0)-(t6<0.0))*(1.0-fabs(t6))+t6; +// //........................................................................ +// nn = ijk-strideY-1; // neighbor index (get convention) +// m7 = Phi[nn]; // get neighbor for phi - 7 +// t7 = m7+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t7)>1.0) t7 =((t7>0.0)-(t7<0.0))*(1.0-fabs(t7))+t7; +// //........................................................................ +// nn = ijk+strideY+1; // neighbor index (get convention) +// m8 = Phi[nn]; // get neighbor for phi - 8 +// t8 = m8+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t8)>1.0) t8 =((t8>0.0)-(t8<0.0))*(1.0-fabs(t8))+t8; +// //........................................................................ +// nn = ijk+strideY-1; // neighbor index (get convention) +// m9 = Phi[nn]; // get neighbor for phi - 9 +// t9 = m9+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t9)>1.0) t9 =((t9>0.0)-(t9<0.0))*(1.0-fabs(t9))+t9; +// //........................................................................ +// nn = ijk-strideY+1; // neighbor index (get convention) +// m10 = Phi[nn]; // get neighbor for phi - 10 +// t10 = m10+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t10)>1.0) t10 =((t10>0.0)-(t10<0.0))*(1.0-fabs(t10))+t10; +// //........................................................................ +// nn = ijk-strideZ-1; // neighbor index (get convention) +// m11 = Phi[nn]; // get neighbor for phi - 11 +// t11 = m11+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t11)>1.0) t11 =((t11>0.0)-(t11<0.0))*(1.0-fabs(t11))+t11; +// //........................................................................ +// nn = ijk+strideZ+1; // neighbor index (get convention) +// m12 = Phi[nn]; // get neighbor for phi - 12 +// t12 = m12+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t12)>1.0) t12 =((t12>0.0)-(t12<0.0))*(1.0-fabs(t12))+t12; +// //........................................................................ +// nn = ijk+strideZ-1; // neighbor index (get convention) +// m13 = Phi[nn]; // get neighbor for phi - 13 +// t13 = m13+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t13)>1.0) t13 =((t13>0.0)-(t13<0.0))*(1.0-fabs(t13))+t13; +// //........................................................................ +// nn = ijk-strideZ+1; // neighbor index (get convention) +// m14 = Phi[nn]; // get neighbor for phi - 14 +// t14 = m14+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t14)>1.0) t14 =((t14>0.0)-(t14<0.0))*(1.0-fabs(t14))+t14; +// //........................................................................ +// nn = ijk-strideZ-strideY; // neighbor index (get convention) +// m15 = Phi[nn]; // get neighbor for phi - 15 +// t15 = m15+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t15)>1.0) t15 =((t15>0.0)-(t15<0.0))*(1.0-fabs(t15))+t15; +// //........................................................................ +// nn = ijk+strideZ+strideY; // neighbor index (get convention) +// m16 = Phi[nn]; // get neighbor for phi - 16 +// t16 = m16+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t16)>1.0) t16 =((t16>0.0)-(t16<0.0))*(1.0-fabs(t16))+t16; +// //........................................................................ +// nn = ijk+strideZ-strideY; // neighbor index (get convention) +// m17 = Phi[nn]; // get neighbor for phi - 17 +// t17 = m17+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t17)>1.0) t17 =((t17>0.0)-(t17<0.0))*(1.0-fabs(t17))+t17; +// //........................................................................ +// nn = ijk-strideZ+strideY; // neighbor index (get convention) +// m18 = Phi[nn]; // get neighbor for phi - 18 +// t18 = m18+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t18)>1.0) t18 =((t18>0.0)-(t18<0.0))*(1.0-fabs(t18))+t18; +// //............Compute the Color Gradient................................... +// nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); +// ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); +// nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); +// C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); +// //correct the normal color gradient by considering the effect of grey solid +// nx = -(t1-t2+0.5*(t7-t8+t9-t10+t11-t12+t13-t14)); +// ny = -(t3-t4+0.5*(t7-t8-t9+t10+t15-t16+t17-t18)); +// nz = -(t5-t6+0.5*(t11-t12-t13+t14+t15-t16-t17+t18)); +// +// if (C_phase==0.0){//i.e. if in a bulk phase, there is no need for grey-solid correction +// nx = nx_phase; +// ny = ny_phase; +// nz = nz_phase; +// } +// +// //...........Normalize the Color Gradient................................. +// C = sqrt(nx*nx+ny*ny+nz*nz); +// double ColorMag = C; +// if (C==0.0) ColorMag=1.0; +// nx = nx/ColorMag; +// ny = ny/ColorMag; +// nz = nz/ColorMag; +// +// // q=0 +// fq = dist[n]; +// rho = fq; +// m1 = -30.0*fq; +// m2 = 12.0*fq; +// +// // q=1 +// //nread = neighborList[n]; // neighbor 2 +// //fq = dist[nread]; // reading the f1 data into register fq +// nr1 = neighborList[n]; +// fq = dist[nr1]; // reading the f1 data into register fq +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jx = fq; +// m4 = -4.0*fq; +// m9 = 2.0*fq; +// m10 = -4.0*fq; +// +// // f2 = dist[10*Np+n]; +// //nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) +// //fq = dist[nread]; // reading the f2 data into register fq +// nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) +// fq = dist[nr2]; // reading the f2 data into register fq +// rho += fq; +// m1 -= 11.0*(fq); +// m2 -= 4.0*(fq); +// jx -= fq; +// m4 += 4.0*(fq); +// m9 += 2.0*(fq); +// m10 -= 4.0*(fq); +// +// // q=3 +// //nread = neighborList[n+2*Np]; // neighbor 4 +// //fq = dist[nread]; +// nr3 = neighborList[n+2*Np]; // neighbor 4 +// fq = dist[nr3]; +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy = fq; +// m6 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 = fq; +// m12 = -2.0*fq; +// +// // q = 4 +// //nread = neighborList[n+3*Np]; // neighbor 3 +// //fq = dist[nread]; +// nr4 = neighborList[n+3*Np]; // neighbor 3 +// fq = dist[nr4]; +// rho+= fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy -= fq; +// m6 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 += fq; +// m12 -= 2.0*fq; +// +// // q=5 +// //nread = neighborList[n+4*Np]; +// //fq = dist[nread]; +// nr5 = neighborList[n+4*Np]; +// fq = dist[nr5]; +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz = fq; +// m8 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// +// // q = 6 +// //nread = neighborList[n+5*Np]; +// //fq = dist[nread]; +// nr6 = neighborList[n+5*Np]; +// fq = dist[nr6]; +// rho+= fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz -= fq; +// m8 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q=7 +// //nread = neighborList[n+6*Np]; +// //fq = dist[nread]; +// nr7 = neighborList[n+6*Np]; +// fq = dist[nr7]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 = fq; +// m16 = fq; +// m17 = -fq; +// +// // q = 8 +// //nread = neighborList[n+7*Np]; +// //fq = dist[nread]; +// nr8 = neighborList[n+7*Np]; +// fq = dist[nr8]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 += fq; +// m16 -= fq; +// m17 += fq; +// +// // q=9 +// //nread = neighborList[n+8*Np]; +// //fq = dist[nread]; +// nr9 = neighborList[n+8*Np]; +// fq = dist[nr9]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 += fq; +// m17 += fq; +// +// // q = 10 +// //nread = neighborList[n+9*Np]; +// //fq = dist[nread]; +// nr10 = neighborList[n+9*Np]; +// fq = dist[nr10]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 -= fq; +// m17 -= fq; +// +// // q=11 +// //nread = neighborList[n+10*Np]; +// //fq = dist[nread]; +// nr11 = neighborList[n+10*Np]; +// fq = dist[nr11]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 = fq; +// m16 -= fq; +// m18 = fq; +// +// // q=12 +// //nread = neighborList[n+11*Np]; +// //fq = dist[nread]; +// nr12 = neighborList[n+11*Np]; +// fq = dist[nr12]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 += fq; +// m16 += fq; +// m18 -= fq; +// +// // q=13 +// //nread = neighborList[n+12*Np]; +// //fq = dist[nread]; +// nr13 = neighborList[n+12*Np]; +// fq = dist[nr13]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 -= fq; +// m18 -= fq; +// +// // q=14 +// //nread = neighborList[n+13*Np]; +// //fq = dist[nread]; +// nr14 = neighborList[n+13*Np]; +// fq = dist[nr14]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 += fq; +// m18 += fq; +// +// // q=15 +// nread = neighborList[n+14*Np]; +// fq = dist[nread]; +// //fq = dist[17*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 = fq; +// m17 += fq; +// m18 -= fq; +// +// // q=16 +// nread = neighborList[n+15*Np]; +// fq = dist[nread]; +// //fq = dist[8*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 += fq; +// m17 -= fq; +// m18 += fq; +// +// // q=17 +// //fq = dist[18*Np+n]; +// nread = neighborList[n+16*Np]; +// fq = dist[nread]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 += fq; +// m18 += fq; +// +// // q=18 +// nread = neighborList[n+17*Np]; +// fq = dist[nread]; +// //fq = dist[9*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 -= fq; +// m18 -= fq; +// +// // Compute greyscale related parameters +// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(perm); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jx/rho0+0.5*(porosity*Gx); +// vy = jy/rho0+0.5*(porosity*Gy); +// vz = jz/rho0+0.5*(porosity*Gz); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux*ux+uy*uy+uz*uz); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); +// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); +// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); +// if (porosity==1.0){ +// Fx=rho0*(Gx); +// Fy=rho0*(Gy); +// Fz=rho0*(Gz); +// } +// +// // write the velocity +// Velocity[n] = ux; +// Velocity[Np+n] = uy; +// Velocity[2*Np+n] = uz; +// +// //........................................................................ +// //..............carry out relaxation process.............................. +// //..........Toelke, Fruediger et. al. 2006................................ +// if (C == 0.0) nx = ny = nz = 0.0; +// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); +// m10 = m10 + rlx_setA*( - m10); +// //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); +// m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); +// m12 = m12 + rlx_setA*( - m12); +// //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); +// m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); +// m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); +// m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; +// dist[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); +// //nread = neighborList[n+Np]; +// dist[nr2] = fq; +// +// // q=2 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); +// //nread = neighborList[n]; +// dist[nr1] = fq; +// +// // q = 3 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// //nread = neighborList[n+3*Np]; +// dist[nr4] = fq; +// +// // q = 4 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// //nread = neighborList[n+2*Np]; +// dist[nr3] = fq; +// +// // q = 5 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// //nread = neighborList[n+5*Np]; +// dist[nr6] = fq; +// +// // q = 6 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// //nread = neighborList[n+4*Np]; +// dist[nr5] = fq; +// +// // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); +// //nread = neighborList[n+7*Np]; +// dist[nr8] = fq; +// +// // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16); +// //nread = neighborList[n+6*Np]; +// dist[nr7] = fq; +// +// // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); +// //nread = neighborList[n+9*Np]; +// dist[nr10] = fq; +// +// // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); +// //nread = neighborList[n+8*Np]; +// dist[nr9] = fq; +// +// // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16); +// //nread = neighborList[n+11*Np]; +// dist[nr12] = fq; +// +// // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ +// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); +// //nread = neighborList[n+10*Np]; +// dist[nr11]= fq; +// +// // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18); +// //nread = neighborList[n+13*Np]; +// dist[nr14] = fq; +// +// // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18); +// //nread = neighborList[n+12*Np]; +// dist[nr13] = fq; +// +// +// // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); +// nread = neighborList[n+15*Np]; +// dist[nread] = fq; +// +// // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); +// nread = neighborList[n+14*Np]; +// dist[nread] = fq; +// +// +// // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); +// nread = neighborList[n+17*Np]; +// dist[nread] = fq; +// +// // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); +// nread = neighborList[n+16*Np]; +// dist[nread] = fq; +// //........................................................................ +// +// // Instantiate mass transport distributions +// // Stationary value - distribution 0 +// nAB = 1.0/(nA+nB); +// Aq[n] = 0.3333333333333333*nA; +// Bq[n] = 0.3333333333333333*nB; +// +// //............................................... +// // q = 0,2,4 +// // Cq = {1,0,0}, {0,1,0}, {0,0,1} +// delta = beta*nA*nB*nAB*0.1111111111111111*nx; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; +// +// // q = 1 +// //nread = neighborList[n+Np]; +// Aq[nr2] = a1; +// Bq[nr2] = b1; +// // q=2 +// //nread = neighborList[n]; +// Aq[nr1] = a2; +// Bq[nr1] = b2; +// +// //............................................... +// // Cq = {0,1,0} +// delta = beta*nA*nB*nAB*0.1111111111111111*ny; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; +// +// // q = 3 +// //nread = neighborList[n+3*Np]; +// Aq[nr4] = a1; +// Bq[nr4] = b1; +// // q = 4 +// //nread = neighborList[n+2*Np]; +// Aq[nr3] = a2; +// Bq[nr3] = b2; +// +// //............................................... +// // q = 4 +// // Cq = {0,0,1} +// delta = beta*nA*nB*nAB*0.1111111111111111*nz; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; +// +// // q = 5 +// //nread = neighborList[n+5*Np]; +// Aq[nr6] = a1; +// Bq[nr6] = b1; +// // q = 6 +// //nread = neighborList[n+4*Np]; +// Aq[nr5] = a2; +// Bq[nr5] = b2; +// //............................................... +// } +// } +//} +// +////Model-2&3 +//__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, +// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, +// double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ +// int ijk,nn,n; +// double fq; +// // conserved momemnts +// double rho,jx,jy,jz; +// double vx,vy,vz,v_mag; +// double ux,uy,uz,u_mag; +// // non-conserved moments +// double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; +// double m3,m5,m7; +// double t1,t2,t4,t6,t8,t9,t10,t11,t12,t13,t14,t15,t16,t17,t18; +// double t3,t5,t7; +// double nA,nB; // number density +// double a1,b1,a2,b2,nAB,delta; +// double C,nx,ny,nz; //color gradient magnitude and direction +// double phi,tau,rho0,rlx_setA,rlx_setB; +// +// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) +// double porosity; +// double perm;//voxel permeability +// double c0, c1; //Guo's model parameters +// double tau_eff; +// double mu_eff;//kinematic viscosity +// double nx_phase,ny_phase,nz_phase,C_phase; +// double Fx,Fy,Fz; +// +// const double mrt_V1=0.05263157894736842; +// const double mrt_V2=0.012531328320802; +// const double mrt_V3=0.04761904761904762; +// const double mrt_V4=0.004594820384294068; +// const double mrt_V5=0.01587301587301587; +// const double mrt_V6=0.0555555555555555555555555; +// const double mrt_V7=0.02777777777777778; +// const double mrt_V8=0.08333333333333333; +// const double mrt_V9=0.003341687552213868; +// const double mrt_V10=0.003968253968253968; +// const double mrt_V11=0.01388888888888889; +// const double mrt_V12=0.04166666666666666; +// +// int S = Np/NBLOCKS/NTHREADS + 1; +// for (int s=0; s1.0) t1 =((t1>0.0)-(t1<0.0))*(1.0-fabs(t1))+t1; +// //........................................................................ +// nn = ijk+1; // neighbor index (get convention) +// m2 = Phi[nn]; // get neighbor for phi - 2 +// t2 = m2+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t2)>1.0) t2 =((t2>0.0)-(t2<0.0))*(1.0-fabs(t2))+t2; +// //........................................................................ +// nn = ijk-strideY; // neighbor index (get convention) +// m3 = Phi[nn]; // get neighbor for phi - 3 +// t3 = m3+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t3)>1.0) t3 =((t3>0.0)-(t3<0.0))*(1.0-fabs(t3))+t3; +// //........................................................................ +// nn = ijk+strideY; // neighbor index (get convention) +// m4 = Phi[nn]; // get neighbor for phi - 4 +// t4 = m4+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t4)>1.0) t4 =((t4>0.0)-(t4<0.0))*(1.0-fabs(t4))+t4; +// //........................................................................ +// nn = ijk-strideZ; // neighbor index (get convention) +// m5 = Phi[nn]; // get neighbor for phi - 5 +// t5 = m5+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t5)>1.0) t5 =((t5>0.0)-(t5<0.0))*(1.0-fabs(t5))+t5; +// //........................................................................ +// nn = ijk+strideZ; // neighbor index (get convention) +// m6 = Phi[nn]; // get neighbor for phi - 6 +// t6 = m6+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t6)>1.0) t6 =((t6>0.0)-(t6<0.0))*(1.0-fabs(t6))+t6; +// //........................................................................ +// nn = ijk-strideY-1; // neighbor index (get convention) +// m7 = Phi[nn]; // get neighbor for phi - 7 +// t7 = m7+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t7)>1.0) t7 =((t7>0.0)-(t7<0.0))*(1.0-fabs(t7))+t7; +// //........................................................................ +// nn = ijk+strideY+1; // neighbor index (get convention) +// m8 = Phi[nn]; // get neighbor for phi - 8 +// t8 = m8+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t8)>1.0) t8 =((t8>0.0)-(t8<0.0))*(1.0-fabs(t8))+t8; +// //........................................................................ +// nn = ijk+strideY-1; // neighbor index (get convention) +// m9 = Phi[nn]; // get neighbor for phi - 9 +// t9 = m9+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t9)>1.0) t9 =((t9>0.0)-(t9<0.0))*(1.0-fabs(t9))+t9; +// //........................................................................ +// nn = ijk-strideY+1; // neighbor index (get convention) +// m10 = Phi[nn]; // get neighbor for phi - 10 +// t10 = m10+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t10)>1.0) t10 =((t10>0.0)-(t10<0.0))*(1.0-fabs(t10))+t10; +// //........................................................................ +// nn = ijk-strideZ-1; // neighbor index (get convention) +// m11 = Phi[nn]; // get neighbor for phi - 11 +// t11 = m11+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t11)>1.0) t11 =((t11>0.0)-(t11<0.0))*(1.0-fabs(t11))+t11; +// //........................................................................ +// nn = ijk+strideZ+1; // neighbor index (get convention) +// m12 = Phi[nn]; // get neighbor for phi - 12 +// t12 = m12+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t12)>1.0) t12 =((t12>0.0)-(t12<0.0))*(1.0-fabs(t12))+t12; +// //........................................................................ +// nn = ijk+strideZ-1; // neighbor index (get convention) +// m13 = Phi[nn]; // get neighbor for phi - 13 +// t13 = m13+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t13)>1.0) t13 =((t13>0.0)-(t13<0.0))*(1.0-fabs(t13))+t13; +// //........................................................................ +// nn = ijk-strideZ+1; // neighbor index (get convention) +// m14 = Phi[nn]; // get neighbor for phi - 14 +// t14 = m14+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t14)>1.0) t14 =((t14>0.0)-(t14<0.0))*(1.0-fabs(t14))+t14; +// //........................................................................ +// nn = ijk-strideZ-strideY; // neighbor index (get convention) +// m15 = Phi[nn]; // get neighbor for phi - 15 +// t15 = m15+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t15)>1.0) t15 =((t15>0.0)-(t15<0.0))*(1.0-fabs(t15))+t15; +// //........................................................................ +// nn = ijk+strideZ+strideY; // neighbor index (get convention) +// m16 = Phi[nn]; // get neighbor for phi - 16 +// t16 = m16+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t16)>1.0) t16 =((t16>0.0)-(t16<0.0))*(1.0-fabs(t16))+t16; +// //........................................................................ +// nn = ijk+strideZ-strideY; // neighbor index (get convention) +// m17 = Phi[nn]; // get neighbor for phi - 17 +// t17 = m17+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t17)>1.0) t17 =((t17>0.0)-(t17<0.0))*(1.0-fabs(t17))+t17; +// //........................................................................ +// nn = ijk-strideZ+strideY; // neighbor index (get convention) +// m18 = Phi[nn]; // get neighbor for phi - 18 +// t18 = m18+(1.0-porosity)*GreySolidGrad[nn]; +// if (fabs(t18)>1.0) t18 =((t18>0.0)-(t18<0.0))*(1.0-fabs(t18))+t18; +// //............Compute the Color Gradient................................... +// nx_phase = -(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); +// ny_phase = -(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); +// nz_phase = -(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); +// C_phase = sqrt(nx_phase*nx_phase+ny_phase*ny_phase+nz_phase*nz_phase); +// //correct the normal color gradient by considering the effect of grey solid +// nx = -(t1-t2+0.5*(t7-t8+t9-t10+t11-t12+t13-t14)); +// ny = -(t3-t4+0.5*(t7-t8-t9+t10+t15-t16+t17-t18)); +// nz = -(t5-t6+0.5*(t11-t12-t13+t14+t15-t16-t17+t18)); +// +// if (C_phase==0.0){ +// nx = nx_phase; +// ny = ny_phase; +// nz = nz_phase; +// } +// +// //...........Normalize the Color Gradient................................. +// C = sqrt(nx*nx+ny*ny+nz*nz); +// double ColorMag = C; +// if (C==0.0) ColorMag=1.0; +// nx = nx/ColorMag; +// ny = ny/ColorMag; +// nz = nz/ColorMag; +// +// // q=0 +// fq = dist[n]; +// rho = fq; +// m1 = -30.0*fq; +// m2 = 12.0*fq; +// +// // q=1 +// fq = dist[2*Np+n]; +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jx = fq; +// m4 = -4.0*fq; +// m9 = 2.0*fq; +// m10 = -4.0*fq; +// +// // f2 = dist[10*Np+n]; +// fq = dist[1*Np+n]; +// rho += fq; +// m1 -= 11.0*(fq); +// m2 -= 4.0*(fq); +// jx -= fq; +// m4 += 4.0*(fq); +// m9 += 2.0*(fq); +// m10 -= 4.0*(fq); +// +// // q=3 +// fq = dist[4*Np+n]; +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy = fq; +// m6 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 = fq; +// m12 = -2.0*fq; +// +// // q = 4 +// fq = dist[3*Np+n]; +// rho+= fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jy -= fq; +// m6 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 += fq; +// m12 -= 2.0*fq; +// +// // q=5 +// fq = dist[6*Np+n]; +// rho += fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz = fq; +// m8 = -4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q = 6 +// fq = dist[5*Np+n]; +// rho+= fq; +// m1 -= 11.0*fq; +// m2 -= 4.0*fq; +// jz -= fq; +// m8 += 4.0*fq; +// m9 -= fq; +// m10 += 2.0*fq; +// m11 -= fq; +// m12 += 2.0*fq; +// +// // q=7 +// fq = dist[8*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 = fq; +// m16 = fq; +// m17 = -fq; +// +// // q = 8 +// fq = dist[7*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 += fq; +// m16 -= fq; +// m17 += fq; +// +// // q=9 +// fq = dist[10*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jy -= fq; +// m6 -= fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 += fq; +// m17 += fq; +// +// // q = 10 +// fq = dist[9*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jy += fq; +// m6 += fq; +// m9 += fq; +// m10 += fq; +// m11 += fq; +// m12 += fq; +// m13 -= fq; +// m16 -= fq; +// m17 -= fq; +// +// // q=11 +// fq = dist[12*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 = fq; +// m16 -= fq; +// m18 = fq; +// +// // q=12 +// fq = dist[11*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 += fq; +// m16 += fq; +// m18 -= fq; +// +// // q=13 +// fq = dist[14*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx += fq; +// m4 += fq; +// jz -= fq; +// m8 -= fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 -= fq; +// m18 -= fq; +// +// // q=14 +// fq = dist[13*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jx -= fq; +// m4 -= fq; +// jz += fq; +// m8 += fq; +// m9 += fq; +// m10 += fq; +// m11 -= fq; +// m12 -= fq; +// m15 -= fq; +// m16 += fq; +// m18 += fq; +// +// // q=15 +// fq = dist[16*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 = fq; +// m17 += fq; +// m18 -= fq; +// +// // q=16 +// fq = dist[15*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 += fq; +// m17 -= fq; +// m18 += fq; +// +// // q=17 +// fq = dist[18*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy += fq; +// m6 += fq; +// jz -= fq; +// m8 -= fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 += fq; +// m18 += fq; +// +// // q=18 +// fq = dist[17*Np+n]; +// rho += fq; +// m1 += 8.0*fq; +// m2 += fq; +// jy -= fq; +// m6 -= fq; +// jz += fq; +// m8 += fq; +// m9 -= 2.0*fq; +// m10 -= 2.0*fq; +// m14 -= fq; +// m17 -= fq; +// m18 -= fq; +// +// // Compute greyscale related parameters +// c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); +// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes +// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); +// c1 = porosity*0.5*GeoFun/sqrt(perm); +// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes +// +// vx = jx/rho0+0.5*(porosity*Gx); +// vy = jy/rho0+0.5*(porosity*Gy); +// vz = jz/rho0+0.5*(porosity*Gz); +// v_mag=sqrt(vx*vx+vy*vy+vz*vz); +// ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); +// uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); +// uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); +// u_mag=sqrt(ux*ux+uy*uy+uz*uz); +// +// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium +// Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx); +// Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy); +// Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz); +// if (porosity==1.0){ +// Fx=rho0*(Gx); +// Fy=rho0*(Gy); +// Fz=rho0*(Gz); +// } +// +// // write the velocity +// Velocity[n] = ux; +// Velocity[Np+n] = uy; +// Velocity[2*Np+n] = uz; +// +// //........................................................................ +// //..............carry out relaxation process.............................. +// //..........Toelke, Fruediger et. al. 2006................................ +// if (C == 0.0) nx = ny = nz = 0.0; +// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); +// m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); +// jx = jx + Fx; +// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); +// jy = jy + Fy; +// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); +// jz = jz + Fz; +// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) +// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); +// m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9); +// m10 = m10 + rlx_setA*( - m10); +// //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); +// m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11); +// m12 = m12 + rlx_setA*( - m12); +// //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); +// m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); +// m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); +// m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); +// m16 = m16 + rlx_setB*( - m16); +// m17 = m17 + rlx_setB*( - m17); +// m18 = m18 + rlx_setB*( - m18); +// +// //.................inverse transformation...................................................... +// // q=0 +// fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; +// dist[n] = fq; +// +// // q = 1 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); +// dist[1*Np+n] = fq; +// +// // q=2 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); +// dist[2*Np+n] = fq; +// +// // q = 3 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// dist[3*Np+n] = fq; +// +// // q = 4 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); +// dist[4*Np+n] = fq; +// +// // q = 5 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// dist[5*Np+n] = fq; +// +// // q = 6 +// fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); +// dist[6*Np+n] = fq; +// +// // q = 7 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); +// dist[7*Np+n] = fq; +// +// +// // q = 8 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 +// +mrt_V12*m12+0.25*m13+0.125*(m17-m16); +// dist[8*Np+n] = fq; +// +// // q = 9 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); +// dist[9*Np+n] = fq; +// +// // q = 10 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+ +// mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); +// dist[10*Np+n] = fq; +// +// +// // q = 11 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12+0.25*m15+0.125*(m18-m16); +// dist[11*Np+n] = fq; +// +// // q = 12 +// fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+ +// mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); +// dist[12*Np+n] = fq; +// +// // q = 13 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15-0.125*(m16+m18); +// dist[13*Np+n] = fq; +// +// // q= 14 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) +// +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 +// -mrt_V12*m12-0.25*m15+0.125*(m16+m18); +// +// dist[14*Np+n] = fq; +// +// // q = 15 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); +// dist[15*Np+n] = fq; +// +// // q = 16 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) +// -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); +// dist[16*Np+n] = fq; +// +// +// // q = 17 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); +// dist[17*Np+n] = fq; +// +// // q = 18 +// fq = mrt_V1*rho+mrt_V9*m1 +// +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) +// -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); +// dist[18*Np+n] = fq; +// //........................................................................ +// +// // Instantiate mass transport distributions +// // Stationary value - distribution 0 +// nAB = 1.0/(nA+nB); +// Aq[n] = 0.3333333333333333*nA; +// Bq[n] = 0.3333333333333333*nB; +// +// //............................................... +// // q = 0,2,4 +// // Cq = {1,0,0}, {0,1,0}, {0,0,1} +// delta = beta*nA*nB*nAB*0.1111111111111111*nx; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; +// +// Aq[1*Np+n] = a1; +// Bq[1*Np+n] = b1; +// Aq[2*Np+n] = a2; +// Bq[2*Np+n] = b2; +// +// //............................................... +// // q = 2 +// // Cq = {0,1,0} +// delta = beta*nA*nB*nAB*0.1111111111111111*ny; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; +// +// Aq[3*Np+n] = a1; +// Bq[3*Np+n] = b1; +// Aq[4*Np+n] = a2; +// Bq[4*Np+n] = b2; +// //............................................... +// // q = 4 +// // Cq = {0,0,1} +// delta = beta*nA*nB*nAB*0.1111111111111111*nz; +// if (!(nA*nB*nAB>0)) delta=0; +// a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; +// b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; +// a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; +// b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; +// +// Aq[5*Np+n] = a1; +// Bq[5*Np+n] = b1; +// Aq[6*Np+n] = a2; +// Bq[6*Np+n] = b2; +// //............................................... +// +// } +// } +//} //__global__ void dvc_ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np) //{ @@ -1413,15 +2840,16 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, // } //} +//Model-1 extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den,ColorGrad, Phi, GreySolidGrad, Poros, Perm, Vel, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -1431,15 +2859,16 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl } +//Model-1 extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den,ColorGrad, Phi, GreySolidGrad, Poros, Perm,Vel, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel, rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff,alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); @@ -1449,3 +2878,40 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, //cudaProfilerStop(); } +////Model-2&3 +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, +// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, +// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ +// +// //cudaProfilerStart(); +// //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor, cudaFuncCachePreferL1); +// +// dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, +// rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleColor: %s \n",cudaGetErrorString(err)); +// } +// //cudaProfilerStop(); +// +//} +// +////Model-2&3 +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, +// double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, +// double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, +// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ +// +// //cudaProfilerStart(); +// //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor, cudaFuncCachePreferL1); +// +// dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel, +// rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff,alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); +// +// cudaError_t err = cudaGetLastError(); +// if (cudaSuccess != err){ +// printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleColor: %s \n",cudaGetErrorString(err)); +// } +// //cudaProfilerStop(); +//} diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index dcbab77e..16e5ece5 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -318,7 +318,7 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels() label_count_global[idx] = Dm->Comm.sumReduce( label_count[idx] ); if (rank==0){ - printf("Component labels: %lu \n",NLABELS); + printf("Number of component labels: %lu \n",NLABELS); for (unsigned int idx=0; idx 0 - // water-wet < 0 - // neutral = 0 - double *SolidPotential_host = new double [Nx*Ny*Nz]; - double *GreySolidGrad_host = new double [3*Np]; - - size_t NLABELS=0; - signed char VALUE=0; - double AFFINITY=0.f; - - auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); - auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); - - NLABELS=LabelList.size(); - if (NLABELS != AffinityList.size()){ - ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); - } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - // Assign the labels - - for (size_t idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component - } - } - SolidPotential_host[n] = AFFINITY; - } - } - } - - // Calculate grey-solid color-gradient - double *Dst; - Dst = new double [3*3*3]; - for (int kk=0; kk<3; kk++){ - for (int jj=0; jj<3; jj++){ - for (int ii=0; ii<3; ii++){ - int index = kk*9+jj*3+ii; - Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); - } - } - } - double w_face = 1.f; - double w_edge = 0.5; - double w_corner = 0.f; - //local - Dst[13] = 0.f; - //faces - Dst[4] = w_face; - Dst[10] = w_face; - Dst[12] = w_face; - Dst[14] = w_face; - Dst[16] = w_face; - Dst[22] = w_face; - // corners - Dst[0] = w_corner; - Dst[2] = w_corner; - Dst[6] = w_corner; - Dst[8] = w_corner; - Dst[18] = w_corner; - Dst[20] = w_corner; - Dst[24] = w_corner; - Dst[26] = w_corner; - // edges - Dst[1] = w_edge; - Dst[3] = w_edge; - Dst[5] = w_edge; - Dst[7] = w_edge; - Dst[9] = w_edge; - Dst[11] = w_edge; - Dst[15] = w_edge; - Dst[17] = w_edge; - Dst[19] = w_edge; - Dst[21] = w_edge; - Dst[23] = w_edge; - Dst[25] = w_edge; - - for (int k=1; kComm.sumReduce( label_count[idx] ); - - if (rank==0){ - printf("Grey-solid labels: %lu \n",NLABELS); - for (unsigned int idx=0; idx 0 +// // water-wet < 0 +// // neutral = 0 +// +// double *GreySolidPhi_host = new double [Nx*Ny*Nz]; +// //initialize grey solid phase field +// for (int k=0;kgetVector( "GreySolidLabels" ); +// auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); +// +// size_t NLABELS=0; +// NLABELS=LabelList.size(); +// if (NLABELS != AffinityList.size()){ +// ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); +// } +// +// double *Dst; +// Dst = new double [3*3*3]; +// for (int kk=0; kk<3; kk++){ +// for (int jj=0; jj<3; jj++){ +// for (int ii=0; ii<3; ii++){ +// int index = kk*9+jj*3+ii; +// Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); +// } +// } +// } +// double w_face = 1.f; +// double w_edge = 1.f; +// double w_corner = 0.f; +// //local +// Dst[13] = 0.f; +// //faces +// Dst[4] = w_face; +// Dst[10] = w_face; +// Dst[12] = w_face; +// Dst[14] = w_face; +// Dst[16] = w_face; +// Dst[22] = w_face; +// // corners +// Dst[0] = w_corner; +// Dst[2] = w_corner; +// Dst[6] = w_corner; +// Dst[8] = w_corner; +// Dst[18] = w_corner; +// Dst[20] = w_corner; +// Dst[24] = w_corner; +// Dst[26] = w_corner; +// // edges +// Dst[1] = w_edge; +// Dst[3] = w_edge; +// Dst[5] = w_edge; +// Dst[7] = w_edge; +// Dst[9] = w_edge; +// Dst[11] = w_edge; +// Dst[15] = w_edge; +// Dst[17] = w_edge; +// Dst[19] = w_edge; +// Dst[21] = w_edge; +// Dst[23] = w_edge; +// Dst[25] = w_edge; +// +// for (int k=1; kid[n]; +// double AFFINITY=0.f; +// // Assign the affinity from the paired list +// for (unsigned int idx=0; idx < NLABELS; idx++){ +// //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); +// if (VALUE == LabelList[idx]){ +// AFFINITY=AffinityList[idx]; +// idx = NLABELS; +// //Mask->id[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// +// if (VALUE>2){//i.e. a grey node +// double neighbor_counter = 0; +// for (int kk=0; kk<3; kk++){ +// for (int jj=0; jj<3; jj++){ +// for (int ii=0; ii<3; ii++){ +// +// int index = kk*9+jj*3+ii; +// double weight= Dst[index]; +// +// int idi=i+ii-1; +// int idj=j+jj-1; +// int idk=k+kk-1; +// +// if (idi < 0) idi=0; +// if (idj < 0) idj=0; +// if (idk < 0) idk=0; +// if (!(idi < Nx)) idi=Nx-1; +// if (!(idj < Ny)) idj=Ny-1; +// if (!(idk < Nz)) idk=Nz-1; +// +// int nn = idk*Nx*Ny + idj*Nx + idi; +// //if (Mask->id[nn] != VALUE){//Model-2:i.e. open nodes, impermeable solid nodes or any other type of greynodes +// if (Mask->id[nn] <=0){//Model-3:i.e. only impermeable solid nodes or any other type of greynodes +// neighbor_counter +=weight; +// } +// } +// } +// } +// if (neighbor_counter>0){ +// GreySolidPhi_host[n] = AFFINITY; +// } +// } +// } +// } +// } +// +// if (rank==0){ +// printf("Number of grey-solid labels: %lu \n",NLABELS); +// for (unsigned int idx=0; idxSendHalo(Phi); if (greyMode==true){ - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + //Model-1 + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } else{ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, @@ -1039,9 +1033,14 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + //Model-1 + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } else{ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, @@ -1068,9 +1067,14 @@ void ScaLBL_GreyscaleColorModel::Run(){ } ScaLBL_Comm_Regular->SendHalo(Phi); if (greyMode==true){ - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + //Model-1 + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } else{ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, @@ -1089,9 +1093,14 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den,ColorGrad, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, + //Model-1 + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } else{ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, @@ -2020,25 +2029,415 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,GreySG_Z_FILE); fclose(GreySG_Z_FILE); - ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); - FILE *CGX_FILE; - sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); - CGX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGX_FILE); - fclose(CGX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); - FILE *CGY_FILE; - sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); - CGY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGY_FILE); - fclose(CGY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); - FILE *CGZ_FILE; - sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); - CGZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGZ_FILE); - fclose(CGZ_FILE); +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); +// FILE *CGX_FILE; +// sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); +// CGX_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGX_FILE); +// fclose(CGX_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); +// FILE *CGY_FILE; +// sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); +// CGY_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGY_FILE); +// fclose(CGY_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); +// FILE *CGZ_FILE; +// sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); +// CGZ_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,CGZ_FILE); +// fclose(CGZ_FILE); } + +//void ScaLBL_GreyscaleColorModel::AssignGreySolidLabels()//Model-1 +//{ +// // ONLY initialize grey nodes +// // Key input parameters: +// // 1. GreySolidLabels +// // labels for grey nodes +// // 2. GreySolidAffinity +// // affinity ranges [-1,1] +// // oil-wet > 0 +// // water-wet < 0 +// // neutral = 0 +// double *SolidPotential_host = new double [Nx*Ny*Nz]; +// double *GreySolidGrad_host = new double [3*Np]; +// +// size_t NLABELS=0; +// signed char VALUE=0; +// double AFFINITY=0.f; +// +// auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); +// auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); +// +// NLABELS=LabelList.size(); +// if (NLABELS != AffinityList.size()){ +// ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); +// } +// +// for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// SolidPotential_host[n] = AFFINITY; +// } +// } +// } +// +// // Calculate grey-solid color-gradient +// double *Dst; +// Dst = new double [3*3*3]; +// for (int kk=0; kk<3; kk++){ +// for (int jj=0; jj<3; jj++){ +// for (int ii=0; ii<3; ii++){ +// int index = kk*9+jj*3+ii; +// Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); +// } +// } +// } +// double w_face = 1.f; +// double w_edge = 0.5; +// double w_corner = 0.f; +// //local +// Dst[13] = 0.f; +// //faces +// Dst[4] = w_face; +// Dst[10] = w_face; +// Dst[12] = w_face; +// Dst[14] = w_face; +// Dst[16] = w_face; +// Dst[22] = w_face; +// // corners +// Dst[0] = w_corner; +// Dst[2] = w_corner; +// Dst[6] = w_corner; +// Dst[8] = w_corner; +// Dst[18] = w_corner; +// Dst[20] = w_corner; +// Dst[24] = w_corner; +// Dst[26] = w_corner; +// // edges +// Dst[1] = w_edge; +// Dst[3] = w_edge; +// Dst[5] = w_edge; +// Dst[7] = w_edge; +// Dst[9] = w_edge; +// Dst[11] = w_edge; +// Dst[15] = w_edge; +// Dst[17] = w_edge; +// Dst[19] = w_edge; +// Dst[21] = w_edge; +// Dst[23] = w_edge; +// Dst[25] = w_edge; +// +// for (int k=1; k 0 + // water-wet < 0 + // neutral = 0 + double *SolidPotential_host = new double [Nx*Ny*Nz]; + double *GreySolidGrad_host = new double [3*Np]; + + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); + } + + for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + SolidPotential_host[n] = AFFINITY; + } + } + } + + // Calculate grey-solid color-gradient + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f; + double w_edge = 0.5; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kSDs(i,j,k)<2.0){ + GreySolidGrad_host[idx+0*Np] = phi_x; + GreySolidGrad_host[idx+1*Np] = phi_y; + GreySolidGrad_host[idx+2*Np] = phi_z; + } + else{ + GreySolidGrad_host[idx+0*Np] = 0.0; + GreySolidGrad_host[idx+1*Np] = 0.0; + GreySolidGrad_host[idx+2*Np] = 0.0; + } + } + } + } + } + + + if (rank==0){ + printf("Number of Grey-solid labels: %lu \n",NLABELS); + for (unsigned int idx=0; idx 0 +// // water-wet < 0 +// // neutral = 0 +// +// //double *SolidPotential_host = new double [Nx*Ny*Nz]; +// double *GreySolidPhi_host = new double [Nx*Ny*Nz]; +// signed char VALUE=0; +// double AFFINITY=0.f; +// +// auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); +// auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); +// +// size_t NLABELS=0; +// NLABELS=LabelList.size(); +// if (NLABELS != AffinityList.size()){ +// ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); +// } +// +// for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// GreySolidPhi_host[n] = AFFINITY; +// } +// } +// } +// +// if (rank==0){ +// printf("Number of grey-solid labels: %lu \n",NLABELS); +// for (unsigned int idx=0; idx Date: Wed, 29 Jul 2020 09:53:39 -0400 Subject: [PATCH 190/270] copy electrokinetic from color --- models/ElectroKinetic.cpp | 1568 +++++++++++++++++++++++++++++++++++++ models/ElectroKinetic.h | 88 +++ 2 files changed, 1656 insertions(+) create mode 100644 models/ElectroKinetic.cpp create mode 100644 models/ElectroKinetic.h diff --git a/models/ElectroKinetic.cpp b/models/ElectroKinetic.cpp new file mode 100644 index 00000000..a8c21a75 --- /dev/null +++ b/models/ElectroKinetic.cpp @@ -0,0 +1,1568 @@ +/* +color lattice boltzmann model + */ +#include "models/ColorModel.h" +#include "analysis/distance.h" +#include "analysis/morphology.h" +#include "common/Communication.h" +#include "common/ReadMicroCT.h" +#include +#include + +ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + REVERSE_FLOW_DIRECTION = false; +} +ScaLBL_ColorModel::~ScaLBL_ColorModel(){ + +} + +/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) +{ + int q,n; + double value; + ofstream File(FILENAME,ios::binary); + for (n=0; n( filename ); + domain_db = db->getDatabase( "Domain" ); + color_db = db->getDatabase( "Color" ); + analysis_db = db->getDatabase( "Analysis" ); + vis_db = db->getDatabase( "Visualization" ); + + // set defaults + timestepMax = 100000; + tauA = tauB = 1.0; + rhoA = rhoB = 1.0; + Fx = Fy = Fz = 0.0; + alpha=1e-3; + beta=0.95; + Restart=false; + din=dout=1.0; + flux=0.0; + + // Color Model parameters + if (color_db->keyExists( "timestepMax" )){ + timestepMax = color_db->getScalar( "timestepMax" ); + } + if (color_db->keyExists( "tauA" )){ + tauA = color_db->getScalar( "tauA" ); + } + if (color_db->keyExists( "tauB" )){ + tauB = color_db->getScalar( "tauB" ); + } + if (color_db->keyExists( "rhoA" )){ + rhoA = color_db->getScalar( "rhoA" ); + } + if (color_db->keyExists( "rhoB" )){ + rhoB = color_db->getScalar( "rhoB" ); + } + if (color_db->keyExists( "F" )){ + Fx = color_db->getVector( "F" )[0]; + Fy = color_db->getVector( "F" )[1]; + Fz = color_db->getVector( "F" )[2]; + } + if (color_db->keyExists( "alpha" )){ + alpha = color_db->getScalar( "alpha" ); + } + if (color_db->keyExists( "beta" )){ + beta = color_db->getScalar( "beta" ); + } + if (color_db->keyExists( "Restart" )){ + Restart = color_db->getScalar( "Restart" ); + } + if (color_db->keyExists( "din" )){ + din = color_db->getScalar( "din" ); + } + if (color_db->keyExists( "dout" )){ + dout = color_db->getScalar( "dout" ); + } + if (color_db->keyExists( "flux" )){ + flux = color_db->getScalar( "flux" ); + } + inletA=1.f; + inletB=0.f; + outletA=0.f; + outletB=1.f; + //if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details) + + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + + // Override user-specified boundary condition for specific protocols + auto protocol = color_db->getWithDefault( "protocol", "none" ); + if (protocol == "seed water"){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } + else if (protocol == "open connected oil"){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } + else if (protocol == "shell aggregation"){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ + BoundaryCondition = 0; + if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n"); + } + domain_db->putScalar( "BC", BoundaryCondition ); + } +} + +void ScaLBL_ColorModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + N = Nx*Ny*Nz; + id = new signed char [N]; + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + // Read domain parameters + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_ColorModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + if (color_db->keyExists( "image_sequence" )){ + auto ImageList = color_db->getVector( "image_sequence"); + int IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + std::string first_image = ImageList[IMAGE_INDEX]; + Mask->Decomp(first_image); + IMAGE_INDEX++; + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else{ + Mask->ReadIDs(); + } + for (int i=0; iid[i]; // save what was read + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + // Solve for the position of the solid phase + for (int k=0;kid[n]; + if (label > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0; + } + } + } +// MeanFilter(Averages->SDs); + if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + CalcDist(Averages->SDs,id_solid,*Mask); + + if (rank == 0) cout << "Domain set." << endl; + + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); +} + +void ScaLBL_ColorModel::AssignComponentLabels(double *phase) +{ + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = color_db->getVector( "ComponentLabels" ); + auto AffinityList = color_db->getVector( "ComponentAffinity" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: ComponentLabels and ComponentAffinity must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (size_t idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + // fluid labels are reserved + if (VALUE == 1) AFFINITY=1.0; + else if (VALUE == 2) AFFINITY=-1.0; + phase[n] = AFFINITY; + } + } + } + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (size_t idx=0; idxComm, label_count[idx]); + + if (rank==0){ + printf("Component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + dist_mem_size = Np*sizeof(double); + neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nx*Ny*Nz ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; + + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + // initialize phi based on PhaseLabel (include solid component labels) + double *PhaseLabel; + PhaseLabel = new double[N]; + AssignComponentLabels(PhaseLabel); + ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); +} + +/******************************************************** + * AssignComponentLabels * + ********************************************************/ + +void ScaLBL_ColorModel::Initialize(){ + + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + /* + * This function initializes model + */ + if (Restart == true){ + if (rank==0){ + printf("Reading restart file! \n"); + } + + // Read in the restart file to CPU buffers + int *TmpMap; + TmpMap = new int[Np]; + + double *cPhi, *cDist, *cDen; + cPhi = new double[N]; + cDen = new double[2*Np]; + cDist = new double[19*Np]; + ScaLBL_CopyToHost(TmpMap, dvcMap, Np*sizeof(int)); + ScaLBL_CopyToHost(cPhi, Phi, N*sizeof(double)); + + ifstream File(LocalRestartFile,ios::binary); + int idx; + double value,va,vb; + for (int n=0; nLastExterior(); n++){ + va = cDen[n]; + vb = cDen[Np + n]; + value = (va-vb)/(va+vb); + idx = TmpMap[n]; + if (!(idx < 0) && idxFirstInterior(); nLastInterior(); n++){ + va = cDen[n]; + vb = cDen[Np + n]; + value = (va-vb)/(va+vb); + idx = TmpMap[n]; + if (!(idx < 0) && idxLastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + // establish reservoirs for external bC + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4 ){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); +} + +void ScaLBL_ColorModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + int IMAGE_INDEX = 0; + int IMAGE_COUNT = 0; + std::vector ImageList; + bool SET_CAPILLARY_NUMBER = false; + bool RESCALE_FORCE = false; + bool MORPH_ADAPT = false; + bool USE_MORPH = false; + bool USE_SEED = false; + bool USE_DIRECT = false; + bool USE_MORPHOPEN_OIL = false; + int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine + int MIN_STEADY_TIMESTEPS = 100000; + int MAX_STEADY_TIMESTEPS = 200000; + int RESCALE_FORCE_AFTER_TIMESTEP = 0; + int RAMP_TIMESTEPS = 0;//50000; // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in) + int CURRENT_MORPH_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) + int CURRENT_STEADY_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) + int morph_interval = 100000; + int analysis_interval = 1000; // number of timesteps in between in situ analysis + int morph_timesteps = 0; + double morph_delta = 0.0; + double seed_water = 0.0; + double capillary_number = 0.0; + double tolerance = 0.01; + double Ca_previous = 0.f; + double initial_volume = 0.0; + double delta_volume = 0.0; + double delta_volume_target = 0.0; + + /* history for morphological algoirthm */ + double KRA_MORPH_FACTOR=0.5; + double volA_prev = 0.0; + double log_krA_prev = 1.0; + double log_krA_target = 1.0; + double log_krA = 1.0; + double slope_krA_volume = 0.0; + if (color_db->keyExists( "vol_A_previous" )){ + volA_prev = color_db->getScalar( "vol_A_previous" ); + } + if (color_db->keyExists( "log_krA_previous" )){ + log_krA_prev = color_db->getScalar( "log_krA_previous" ); + } + if (color_db->keyExists( "krA_morph_factor" )){ + KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); + } + + /* defaults for simulation protocols */ + auto protocol = color_db->getWithDefault( "protocol", "none" ); + if (protocol == "image sequence"){ + // Get the list of images + USE_DIRECT = true; + ImageList = color_db->getVector( "image_sequence"); + IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); + IMAGE_COUNT = ImageList.size(); + morph_interval = 10000; + USE_MORPH = true; + } + else if (protocol == "seed water"){ + morph_delta = -0.05; + seed_water = 0.01; + USE_SEED = true; + USE_MORPH = true; + } + else if (protocol == "open connected oil"){ + morph_delta = -0.05; + USE_MORPH = true; + USE_MORPHOPEN_OIL = true; + } + else if (protocol == "shell aggregation"){ + morph_delta = -0.05; + USE_MORPH = true; + } + if (color_db->keyExists( "capillary_number" )){ + capillary_number = color_db->getScalar( "capillary_number" ); + SET_CAPILLARY_NUMBER=true; + } + if (color_db->keyExists( "rescale_force_after_timestep" )){ + RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar( "rescale_force_after_timestep" ); + RESCALE_FORCE = true; + } + if (color_db->keyExists( "timestep" )){ + timestep = color_db->getScalar( "timestep" ); + } + if (BoundaryCondition != 0 && BoundaryCondition != 5 && SET_CAPILLARY_NUMBER==true){ + if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 or 5 \n"); + SET_CAPILLARY_NUMBER=false; + } + if (analysis_db->keyExists( "seed_water" )){ + seed_water = analysis_db->getScalar( "seed_water" ); + if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water); + USE_SEED = true; + } + if (analysis_db->keyExists( "morph_delta" )){ + morph_delta = analysis_db->getScalar( "morph_delta" ); + if (rank == 0) printf("Target volume change %f (morph_delta) \n",morph_delta); + } + if (analysis_db->keyExists( "morph_interval" )){ + morph_interval = analysis_db->getScalar( "morph_interval" ); + USE_MORPH = true; + } + if (analysis_db->keyExists( "use_morphopen_oil" )){ + USE_MORPHOPEN_OIL = analysis_db->getScalar( "use_morphopen_oil" ); + if (rank == 0 && USE_MORPHOPEN_OIL) printf("Volume change by morphological opening \n"); + USE_MORPH = true; + } + if (analysis_db->keyExists( "tolerance" )){ + tolerance = analysis_db->getScalar( "tolerance" ); + } + if (analysis_db->keyExists( "analysis_interval" )){ + analysis_interval = analysis_db->getScalar( "analysis_interval" ); + } + if (analysis_db->keyExists( "min_steady_timesteps" )){ + MIN_STEADY_TIMESTEPS = analysis_db->getScalar( "min_steady_timesteps" ); + } + if (analysis_db->keyExists( "max_steady_timesteps" )){ + MAX_STEADY_TIMESTEPS = analysis_db->getScalar( "max_steady_timesteps" ); + } + if (analysis_db->keyExists( "max_morph_timesteps" )){ + MAX_MORPH_TIMESTEPS = analysis_db->getScalar( "max_morph_timesteps" ); + } + + + if (rank==0){ + printf("********************************************************\n"); + if (protocol == "image sequence"){ + printf(" using protocol = image sequence \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + std::string first_image = ImageList[IMAGE_INDEX]; + printf(" first image in sequence: %s ***\n", first_image.c_str()); + } + else if (protocol == "seed water"){ + printf(" using protocol = seed water \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + printf(" seed_water = %f \n",seed_water); + } + else if (protocol == "open connected oil"){ + printf(" using protocol = open connected oil \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + } + else if (protocol == "shell aggregation"){ + printf(" using protocol = shell aggregation \n"); + printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); + printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); + printf(" tolerance = %f \n",tolerance); + printf(" morph_delta = %f \n",morph_delta); + } + printf("No. of timesteps: %i \n", timestepMax); + fflush(stdout); + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); + //......................................... + + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + //std::shared_ptr analysis_db; + bool Regular = false; + auto current_db = db->cloneDatabase(); + runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); + //analysis.createThreads( analysis_method, 4 ); + while (timestep < timestepMax ) { + //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } + PROFILE_START("Update"); + // *************ODD TIMESTEP************* + timestep++; + // Compute the Phase indicator field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + if (BoundaryCondition > 0 && BoundaryCondition < 5){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + // Halo exchange for phase field + ScaLBL_Comm_Regular->SendHalo(Phi); + + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set BCs + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + + // *************EVEN TIMESTEP************* + timestep++; + // Compute the Phase indicator field + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); + + // Perform the collision operation + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + // Halo exchange for phase field + if (BoundaryCondition > 0 && BoundaryCondition < 5){ + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Phi); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + //************************************************************************ + PROFILE_STOP("Update"); + + if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ + printf("%i %f \n",timestep,din); + } + // Run the analysis + analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + + // allow initial ramp-up to get closer to steady state + if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ + analysis.finish(); + CURRENT_STEADY_TIMESTEPS += analysis_interval; + + double volB = Averages->gwb.V; + double volA = Averages->gnb.V; + volA /= Dm->Volume; + volB /= Dm->Volume;; + //initial_volume = volA*Dm->Volume; + double vA_x = Averages->gnb.Px/Averages->gnb.M; + double vA_y = Averages->gnb.Py/Averages->gnb.M; + double vA_z = Averages->gnb.Pz/Averages->gnb.M; + double vB_x = Averages->gwb.Px/Averages->gwb.M; + double vB_y = Averages->gwb.Py/Averages->gwb.M; + double vB_z = Averages->gwb.Pz/Averages->gwb.M; + double muA = rhoA*(tauA-0.5)/3.f; + double muB = rhoB*(tauB-0.5)/3.f; + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double current_saturation = volB/(volA+volB); + double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); + double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); + double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); + + if ( morph_timesteps > morph_interval ){ + + bool isSteady = false; + if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_STEADY_TIMESTEPS > MIN_STEADY_TIMESTEPS)) + isSteady = true; + if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) + isSteady = true; + if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ + RESCALE_FORCE = false; + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + if ( isSteady ){ + MORPH_ADAPT = true; + CURRENT_MORPH_TIMESTEPS=0; + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ + double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); + log_krA = log(krA_TMP); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + } + } + log_krA_prev = log_krA; + volA_prev = volA; + //******************************** **/ + /** compute averages & write data **/ + Averages->Full(); + Averages->Write(timestep); + analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.finish(); + + if (rank==0){ + printf("** WRITE STEADY POINT *** "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + double h = Dm->voxel_length; + // pressures + double pA = Averages->gnb.p; + double pB = Averages->gwb.p; + double pAc = Averages->gnc.p; + double pBc = Averages->gwc.p; + double pAB = (pA-pB)/(h*6.0*alpha); + double pAB_connected = (pAc-pBc)/(h*6.0*alpha); + // connected contribution + double Vol_nc = Averages->gnc.V/Dm->Volume; + double Vol_wc = Averages->gwc.V/Dm->Volume; + double Vol_nd = Averages->gnd.V/Dm->Volume; + double Vol_wd = Averages->gwd.V/Dm->Volume; + double Mass_n = Averages->gnc.M + Averages->gnd.M; + double Mass_w = Averages->gwc.M + Averages->gwd.M; + double vAc_x = Averages->gnc.Px/Mass_n; + double vAc_y = Averages->gnc.Py/Mass_n; + double vAc_z = Averages->gnc.Pz/Mass_n; + double vBc_x = Averages->gwc.Px/Mass_w; + double vBc_y = Averages->gwc.Py/Mass_w; + double vBc_z = Averages->gwc.Pz/Mass_w; + // disconnected contribution + double vAd_x = Averages->gnd.Px/Mass_n; + double vAd_y = Averages->gnd.Py/Mass_n; + double vAd_z = Averages->gnd.Pz/Mass_n; + double vBd_x = Averages->gwd.Px/Mass_w; + double vBd_y = Averages->gwd.Py/Mass_w; + double vBd_z = Averages->gwd.Pz/Mass_w; + + double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); + double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); + double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); + double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); + + double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); + double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); + + double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); + double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + + double kAeff = h*h*muA*(flow_rate_A)/(force_mag); + double kBeff = h*h*muB*(flow_rate_B)/(force_mag); + + double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; + double Mobility = muA/muB; + + bool WriteHeader=false; + FILE * kr_log_file = fopen("relperm.csv","r"); + if (kr_log_file != NULL) + fclose(kr_log_file); + else + WriteHeader=true; + kr_log_file = fopen("relperm.csv","a"); + if (WriteHeader) + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fclose(kr_log_file); + + printf(" Measured capillary number %f \n ",Ca); + } + if (SET_CAPILLARY_NUMBER ){ + Fx *= capillary_number / Ca; + Fy *= capillary_number / Ca; + Fz *= capillary_number / Ca; + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + color_db->putVector("F",{Fx,Fy,Fz}); + } + + CURRENT_STEADY_TIMESTEPS = 0; + } + else{ + if (rank==0){ + printf("** Continue to simulate steady *** \n "); + printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + } + } + morph_timesteps=0; + Ca_previous = Ca; + } + + if (MORPH_ADAPT ){ + CURRENT_MORPH_TIMESTEPS += analysis_interval; + if (USE_DIRECT){ + // Use image sequence + IMAGE_INDEX++; + MORPH_ADAPT = false; + if (IMAGE_INDEX < IMAGE_COUNT){ + std::string next_image = ImageList[IMAGE_INDEX]; + if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); + color_db->putScalar("image_index",IMAGE_INDEX); + ImageInit(next_image); + } + else{ + if (rank==0) printf("Finished simulating image sequence \n"); + timestep = timestepMax; + } + } + else if (USE_SEED){ + delta_volume = volA*Dm->Volume - initial_volume; + CURRENT_MORPH_TIMESTEPS += analysis_interval; + double massChange = SeedPhaseField(seed_water); + if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); + } + else if (USE_MORPHOPEN_OIL){ + delta_volume = volA*Dm->Volume - initial_volume; + if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); + MorphOpenConnected(delta_volume_target); + } + else { + if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); + //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A + delta_volume += MorphInit(beta,delta_volume_target-delta_volume); + } + + if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; + } + else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { + MORPH_ADAPT = false; + CURRENT_STEADY_TIMESTEPS=0; + initial_volume = volA*Dm->Volume; + delta_volume = 0.0; + RESCALE_FORCE = true; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; + } + } + morph_timesteps += analysis_interval; + } + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + } + analysis.finish(); + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_color_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} + +double ScaLBL_ColorModel::ImageInit(std::string Filename){ + + if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); + Mask->Decomp(Filename); + for (int i=0; iid[i]; // save what was read + for (int i=0; iid[i] = Mask->id[i]; // save what was read + + double *PhaseLabel; + PhaseLabel = new double[Nx*Ny*Nz]; + AssignComponentLabels(PhaseLabel); + + double Count = 0.0; + double PoreCount = 0.0; + for (int k=1; kComm, Count); + PoreCount=sumReduce( Dm->Comm, PoreCount); + + if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); + ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + + ScaLBL_D3Q19_Init(fq, Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); + + double saturation = Count/PoreCount; + return saturation; + +} + +double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ + + int nx = Nx; + int ny = Ny; + int nz = Nz; + int n; + int N = nx*ny*nz; + double volume_change=0.0; + + if (target_volume_change < 0.0){ + Array id_solid(nx,ny,nz); + Array phase_label(nx,ny,nz); + DoubleArray distance(Nx,Ny,Nz); + DoubleArray phase(nx,ny,nz); + signed char *id_connected; + id_connected = new signed char [nx*ny*nz]; + + ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); + + // Extract only the connected part of NWP + BlobIDstruct new_index; + double vF=0.0; double vS=0.0; + ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); + MPI_Barrier(Dm->Comm); + + long long count_connected=0; + long long count_porespace=0; + long long count_water=0; + for (int k=1; k 0){ + count_porespace++; + } + if (id[n] == 2){ + count_water++; + } + } + } + } + count_connected=sumReduce( Dm->Comm, count_connected); + count_porespace=sumReduce( Dm->Comm, count_porespace); + count_water=sumReduce( Dm->Comm, count_water); + + for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } + } + } + + int count_morphopen=0.0; + for (int k=1; kComm, count_morphopen); + volume_change = double(count_morphopen - count_connected); + + if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); + + ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + } + return(volume_change); +} +double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ + srand(time(NULL)); + double mass_loss =0.f; + double count =0.f; + double *Aq_tmp, *Bq_tmp; + + Aq_tmp = new double [7*Np]; + Bq_tmp = new double [7*Np]; + + ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); + + + for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } + + for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } + + count= sumReduce( Dm->Comm, count); + mass_loss= sumReduce( Dm->Comm, mass_loss); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + + // Need to initialize Aq, Bq, Den, Phi directly + //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); + + return(mass_loss); +} + +double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + + double vF = 0.f; + double vS = 0.f; + double delta_volume; + double WallFactor = 0.0; + bool USE_CONNECTED_NWP = false; + + DoubleArray phase(Nx,Ny,Nz); + IntArray phase_label(Nx,Ny,Nz);; + DoubleArray phase_distance(Nx,Ny,Nz); + Array phase_id(Nx,Ny,Nz); + fillHalo fillDouble(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); + + + // Basic algorithm to + // 1. Copy phase field to CPU + ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); + + double count = 0.f; + for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f) count+=1.f; + } + } + } + double volume_initial = sumReduce( Dm->Comm, count); + /* + sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); + FILE *INPUT = fopen(LocalRankFilename,"wb"); + fwrite(phase.data(),8,N,INPUT); + fclose(INPUT); + */ + // 2. Identify connected components of phase field -> phase_label + + double volume_connected = 0.0; + double second_biggest = 0.0; + if (USE_CONNECTED_NWP){ + BlobIDstruct new_index; + ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); + MPI_Barrier(Dm->Comm); + + // only operate on component "0" + count = 0.0; + + for (int k=0; kComm, count); + second_biggest = sumReduce( Dm->Comm, second_biggest); + } + else { + // use the whole NWP + for (int k=0; kSDs(i,j,k) > 0.f){ + if (phase(i,j,k) > 0.f ){ + phase_id(i,j,k) = 0; + } + else { + phase_id(i,j,k) = 1; + } + } + else { + phase_id(i,j,k) = 1; + } + } + } + } + } + + /*int reach_x, reach_y, reach_z; + for (int k=0; k phase_distance + CalcDist(phase_distance,phase_id,*Dm); + + double temp,value; + double factor=0.5/beta; + for (int k=0; k 1.f) value=1.f; + if (value < -1.f) value=-1.f; + // temp -- distance based on analytical form McClure, Prins et al, Comp. Phys. Comm. + temp = -factor*log((1.0+value)/(1.0-value)); + /// use this approximation close to the object + if (fabs(value) < 0.8 && Averages->SDs(i,j,k) > 1.f ){ + phase_distance(i,j,k) = temp; + } + // erase the original object + phase(i,j,k) = -1.0; + } + } + } + } + + if (USE_CONNECTED_NWP){ + if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){ + // if connected volume is less than 2% just delete the whole thing + if (rank==0) printf("Connected region has shrunk! \n"); + REVERSE_FLOW_DIRECTION = true; + } + +/* else{*/ + if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest ); + } + if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); + double target_delta_volume_incremental = target_delta_volume; + if (fabs(target_delta_volume) > 0.01*volume_initial) + target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); + + for (int k=0; kSDs(i,j,k) > 0.f){ + if (d < 3.f){ + //phase(i,j,k) = -1.0; + phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); + } + } + } + } + } + fillDouble.fill(phase); + //} + + count = 0.f; + for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f){ + count+=1.f; + } + } + } + } + double volume_final= sumReduce( Dm->Comm, count); + + delta_volume = (volume_final-volume_initial); + if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); + if (rank == 0) printf(" new saturation = %f \n", volume_final/(0.238323*double((Nx-2)*(Ny-2)*(Nz-2)*nprocs))); + + // 6. copy back to the device + //if (rank==0) printf("MorphInit: copy data back to device\n"); + ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); + /* + sprintf(LocalRankFilename,"dist_final.%05i.raw",rank); + FILE *DIST = fopen(LocalRankFilename,"wb"); + fwrite(phase_distance.data(),8,N,DIST); + fclose(DIST); + + sprintf(LocalRankFilename,"phi_final.%05i.raw",rank); + FILE *PHI = fopen(LocalRankFilename,"wb"); + fwrite(phase.data(),8,N,PHI); + fclose(PHI); + */ + // 7. Re-initialize phase field and density + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); + ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); + ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); + } + } + return delta_volume; +} + +void ScaLBL_ColorModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); + ScaLBL_CopyToHost(PhaseField.data(), Phi, sizeof(double)*N); + + FILE *OUTFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); + FILE *AFILE; + sprintf(LocalRankFilename,"A.%05i.raw",rank); + AFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,AFILE); + fclose(AFILE); + + ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); + FILE *BFILE; + sprintf(LocalRankFilename,"B.%05i.raw",rank); + BFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,BFILE); + fclose(BFILE); + + ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); + FILE *PFILE; + sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); + PFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,PFILE); + fclose(PFILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + +/* ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); + FILE *CGX_FILE; + sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); + CGX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGX_FILE); + fclose(CGX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); + FILE *CGY_FILE; + sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); + CGY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGY_FILE); + fclose(CGY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); + FILE *CGZ_FILE; + sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); + CGZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGZ_FILE); + fclose(CGZ_FILE); +*/ +} diff --git a/models/ElectroKinetic.h b/models/ElectroKinetic.h new file mode 100644 index 00000000..a3b3a124 --- /dev/null +++ b/models/ElectroKinetic.h @@ -0,0 +1,88 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "analysis/TwoPhase.h" +#include "analysis/runAnalysis.h" +#include "common/MPI_Helpers.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_ColorModel{ +public: + ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_ColorModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void WriteDebug(); + + bool Restart,pBC; + bool REVERSE_FLOW_DIRECTION; + int timestep,timestepMax; + int BoundaryCondition; + double tauA,tauB,rhoA,rhoB,alpha,beta; + double Fx,Fy,Fz,flux; + double din,dout,inletA,inletB,outletA,outletB; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + std::shared_ptr ScaLBL_Comm_Regular; + //std::shared_ptr Averages; + std::shared_ptr Averages; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr color_db; + std::shared_ptr analysis_db; + std::shared_ptr vis_db; + + IntArray Map; + signed char *id; + int *NeighborList; + int *dvcMap; + double *fq, *Aq, *Bq; + double *Den, *Phi; + double *ColorGrad; + double *Velocity; + double *Pressure; + +private: + MPI_Comm comm; + + int dist_mem_size; + int neighborSize; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); + void AssignComponentLabels(double *phase); + double ImageInit(std::string filename); + double MorphInit(const double beta, const double morph_delta); + double SeedPhaseField(const double seed_water_in_oil); + double MorphOpenConnected(double target_volume_change); +}; + From 9d333a737173e6459884aaa205b904d684524ed0 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 29 Jul 2020 09:56:52 -0400 Subject: [PATCH 191/270] adding option to refine distance with phase field --- analysis/Minkowski.cpp | 32 ++++++++++++++++++++++++++++++++ analysis/Minkowski.h | 1 + 2 files changed, 33 insertions(+) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 9e6e0f43..ddc6382e 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -156,6 +156,38 @@ void Minkowski::MeasureObject(){ } +void Minkowski::MeasureObject(double factor, const DoubleArray &Phi){ + /* + * compute the distance to an object + * + * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects + * 0 - labels the object + * 1 - labels the rest of the + */ + for (int k=0; k -2.5) + distance(i,j,k) = factor*log((1.0+value)/(1.0-value)); + } + } + } + + ComputeScalar(distance,0.0); + +} + + int Minkowski::MeasureConnectedPathway(){ /* * compute the connected pathway for object with LABEL in id field diff --git a/analysis/Minkowski.h b/analysis/Minkowski.h index bcdf95f7..d6f06f56 100644 --- a/analysis/Minkowski.h +++ b/analysis/Minkowski.h @@ -63,6 +63,7 @@ public: Minkowski(std::shared_ptr Dm); ~Minkowski(); void MeasureObject(); + void MeasureObject(double factor, const DoubleArray &Phi); int MeasureConnectedPathway(); void ComputeScalar(const DoubleArray& Field, const double isovalue); From 6e9b808925738fd0c79cb6ca3a012ba0f03cbda6 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 29 Jul 2020 10:03:16 -0400 Subject: [PATCH 192/270] adding option to refine distance with phase field --- analysis/Minkowski.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++ analysis/Minkowski.h | 1 + 2 files changed, 45 insertions(+) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index ddc6382e..7228157f 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -232,6 +232,50 @@ int Minkowski::MeasureConnectedPathway(){ return n_connected_components; } +int Minkowski::MeasureConnectedPathway(double factor, const DoubleArray &Phi){ + /* + * compute the connected pathway for object with LABEL in id field + * compute the labels for connected components + * compute the distance to the connected pathway + * + * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects + */ + + char LABEL = 0; + for (int k=0; krank_info,distance,distance,vF,vF,label,Dm->Comm); +// int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm ) + MPI_Barrier(Dm->Comm); + + for (int k=0; k Date: Wed, 29 Jul 2020 10:08:29 -0400 Subject: [PATCH 193/270] using phase to update distance in subphase --- analysis/SubPhase.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/SubPhase.cpp b/analysis/SubPhase.cpp index 2a5e3350..8920fabd 100644 --- a/analysis/SubPhase.cpp +++ b/analysis/SubPhase.cpp @@ -427,13 +427,13 @@ void SubPhase::Full(){ } } // measure the whole object - morph_n->MeasureObject(); + morph_n->MeasureObject(0.5/beta,Phi); nd.V = morph_n->V(); nd.A = morph_n->A(); nd.H = morph_n->H(); nd.X = morph_n->X(); // measure only the connected part - nd.Nc = morph_n->MeasureConnectedPathway(); + nd.Nc = morph_n->MeasureConnectedPathway(0.5/beta,Phi); nc.V = morph_n->V(); nc.A = morph_n->A(); nc.H = morph_n->H(); @@ -475,13 +475,13 @@ void SubPhase::Full(){ } } } - morph_w->MeasureObject(); + morph_w->MeasureObject(-0.5/beta,Phi); wd.V = morph_w->V(); wd.A = morph_w->A(); wd.H = morph_w->H(); wd.X = morph_w->X(); // measure only the connected part - wd.Nc = morph_w->MeasureConnectedPathway(); + wd.Nc = morph_w->MeasureConnectedPathway(-0.5/beta,Phi); wc.V = morph_w->V(); wc.A = morph_w->A(); wc.H = morph_w->H(); From d88455a854d7b8226b77022aac2dd68c847d047a Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 31 Jul 2020 22:09:34 -0400 Subject: [PATCH 194/270] distance from color, fix sign --- analysis/Minkowski.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 7228157f..70d38173 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -176,9 +176,14 @@ void Minkowski::MeasureObject(double factor, const DoubleArray &Phi){ for (int k=0; k -2.5) - distance(i,j,k) = factor*log((1.0+value)/(1.0-value)); + double value = Phi(i,j,k); + double dist_value = distance(i,j,k); + if (dist_value < 2.5 && dist_value > -2.5) { + double new_distance = factor*log((1.0+value)/(1.0-value)); + if (dist_value*new_distance < 0.0 ) + new_distance = (-1.0)*new_distance; + distance(i,j,k) = new_distance; + } } } } From a1df2b57dee08a53fd85063e9969f1b381a044a6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 3 Aug 2020 07:08:14 -0400 Subject: [PATCH 195/270] update subphase / minkowski for Euler --- analysis/Minkowski.cpp | 9 +++++---- analysis/SubPhase.cpp | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/analysis/Minkowski.cpp b/analysis/Minkowski.cpp index 70d38173..668875b9 100644 --- a/analysis/Minkowski.cpp +++ b/analysis/Minkowski.cpp @@ -94,11 +94,13 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue) //Xi += 0.25*double(object.VertexCount); // check if vertices are at corners for (int idx=0; idxMeasureObject(0.5/beta,Phi); + morph_n->MeasureObject();//0.5/beta,Phi); nd.V = morph_n->V(); nd.A = morph_n->A(); nd.H = morph_n->H(); nd.X = morph_n->X(); // measure only the connected part - nd.Nc = morph_n->MeasureConnectedPathway(0.5/beta,Phi); + nd.Nc = morph_n->MeasureConnectedPathway();//0.5/beta,Phi); nc.V = morph_n->V(); nc.A = morph_n->A(); nc.H = morph_n->H(); @@ -475,13 +475,13 @@ void SubPhase::Full(){ } } } - morph_w->MeasureObject(-0.5/beta,Phi); + morph_w->MeasureObject();//-0.5/beta,Phi); wd.V = morph_w->V(); wd.A = morph_w->A(); wd.H = morph_w->H(); wd.X = morph_w->X(); // measure only the connected part - wd.Nc = morph_w->MeasureConnectedPathway(-0.5/beta,Phi); + wd.Nc = morph_w->MeasureConnectedPathway();//-0.5/beta,Phi); wc.V = morph_w->V(); wc.A = morph_w->A(); wc.H = morph_w->H(); From 99ee51d8e11a4e8a40d140b6324b79330367ec65 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 5 Aug 2020 17:33:54 -0400 Subject: [PATCH 196/270] add a full form of Guo-type body force scheme, but now using it now --- gpu/GreyscaleColor.cu | 84 ++++- models/GreyscaleColorModel.cpp | 635 +++++++++++++++++---------------- 2 files changed, 398 insertions(+), 321 deletions(-) diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 9ac4ce00..2af5760b 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -4,7 +4,7 @@ #define NBLOCKS 1024 #define NTHREADS 256 -//Model-1 +//Model-1 & 4 __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, @@ -516,6 +516,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma //........................................................................ //..............carry out relaxation process.............................. //..........Toelke, Fruediger et. al. 2006................................ + //---------------- NO higher-order force -------------------------------// if (C == 0.0) nx = ny = nz = 0.0; m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); @@ -540,6 +541,43 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); + //----------------------------------------------------------------------// + + //----------------With higher-order force ------------------------------// + //if (C == 0.0) nx = ny = nz = 0.0; + //m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1) + // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; + //m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2) + // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + //jx = jx + Fx; + //m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + //jy = jy + Fy; + //m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + //jz = jz + Fz; + //m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + //m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9) + // + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; + ////m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) + // + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; + //m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11) + // + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; + ////m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) + // + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; + //m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); + // + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; + //m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); + // + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; + //m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); + // + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + //m16 = m16 + rlx_setB*( - m16); + //m17 = m17 + rlx_setB*( - m17); + //m18 = m18 + rlx_setB*( - m18); + //----------------------------------------------------------------------// //.................inverse transformation...................................................... // q=0 @@ -727,7 +765,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma } } -//Model-1 +//Model-1 & 4 __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, @@ -1183,6 +1221,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, //........................................................................ //..............carry out relaxation process.............................. //..........Toelke, Fruediger et. al. 2006................................ + //---------------- NO higher-order force -------------------------------// if (C == 0.0) nx = ny = nz = 0.0; m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1); m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2); @@ -1207,6 +1246,43 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); + //----------------------------------------------------------------------// + + //----------------With higher-order force ------------------------------// + //if (C == 0.0) nx = ny = nz = 0.0; + //m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) -19*alpha*C - m1) + // + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; + //m2 = m2 + rlx_setA*((3*rho - 5.5*(ux*ux+uy*uy+uz*uz)*rho0/porosity)- m2) + // + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; + //jx = jx + Fx; + //m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0)- m4) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); + //jy = jy + Fy; + //m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0)- m6) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); + //jz = jz + Fz; + //m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0)- m8) + // + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); + //m9 = m9 + rlx_setA*(((2*ux*ux-uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(2*nx*nx-ny*ny-nz*nz) - m9) + // + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; + ////m10 = m10 + rlx_setA*( - m10); + //m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) + // + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; + //m11 = m11 + rlx_setA*(((uy*uy-uz*uz)*rho0/porosity) + 0.5*alpha*C*(ny*ny-nz*nz)- m11) + // + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; + ////m12 = m12 + rlx_setA*( - m12); + //m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) + // + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; + //m13 = m13 + rlx_setA*( (ux*uy*rho0/porosity) + 0.5*alpha*C*nx*ny - m13); + // + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; + //m14 = m14 + rlx_setA*( (uy*uz*rho0/porosity) + 0.5*alpha*C*ny*nz - m14); + // + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; + //m15 = m15 + rlx_setA*( (ux*uz*rho0/porosity) + 0.5*alpha*C*nx*nz - m15); + // + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; + //m16 = m16 + rlx_setB*( - m16); + //m17 = m17 + rlx_setB*( - m17); + //m18 = m18 + rlx_setB*( - m18); + //----------------------------------------------------------------------// //.................inverse transformation...................................................... // q=0 @@ -2840,7 +2916,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, // } //} -//Model-1 +//Model-1 & 4 extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, @@ -2859,7 +2935,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl } -//Model-1 +//Model-1 & 4 extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 16e5ece5..49f1e14d 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -332,158 +332,168 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels() delete [] phase; } -//void ScaLBL_GreyscaleColorModel::AssignGreySolidLabels()//Model-2 & Model-3 -//{ -// // ONLY initialize grey nodes -// // Key input parameters: -// // 1. GreySolidLabels -// // labels for grey nodes -// // 2. GreySolidAffinity -// // affinity ranges [-1,1] -// // oil-wet > 0 -// // water-wet < 0 -// // neutral = 0 -// -// double *GreySolidPhi_host = new double [Nx*Ny*Nz]; -// //initialize grey solid phase field -// for (int k=0;kgetVector( "GreySolidLabels" ); -// auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); -// -// size_t NLABELS=0; -// NLABELS=LabelList.size(); -// if (NLABELS != AffinityList.size()){ -// ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); -// } -// -// double *Dst; -// Dst = new double [3*3*3]; -// for (int kk=0; kk<3; kk++){ -// for (int jj=0; jj<3; jj++){ -// for (int ii=0; ii<3; ii++){ -// int index = kk*9+jj*3+ii; -// Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); -// } -// } -// } -// double w_face = 1.f; -// double w_edge = 1.f; -// double w_corner = 0.f; -// //local -// Dst[13] = 0.f; -// //faces -// Dst[4] = w_face; -// Dst[10] = w_face; -// Dst[12] = w_face; -// Dst[14] = w_face; -// Dst[16] = w_face; -// Dst[22] = w_face; -// // corners -// Dst[0] = w_corner; -// Dst[2] = w_corner; -// Dst[6] = w_corner; -// Dst[8] = w_corner; -// Dst[18] = w_corner; -// Dst[20] = w_corner; -// Dst[24] = w_corner; -// Dst[26] = w_corner; -// // edges -// Dst[1] = w_edge; -// Dst[3] = w_edge; -// Dst[5] = w_edge; -// Dst[7] = w_edge; -// Dst[9] = w_edge; -// Dst[11] = w_edge; -// Dst[15] = w_edge; -// Dst[17] = w_edge; -// Dst[19] = w_edge; -// Dst[21] = w_edge; -// Dst[23] = w_edge; -// Dst[25] = w_edge; -// -// for (int k=1; kid[n]; -// double AFFINITY=0.f; -// // Assign the affinity from the paired list -// for (unsigned int idx=0; idx < NLABELS; idx++){ -// //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); -// if (VALUE == LabelList[idx]){ -// AFFINITY=AffinityList[idx]; -// idx = NLABELS; -// //Mask->id[n] = 0; // set mask to zero since this is an immobile component -// } -// } -// -// if (VALUE>2){//i.e. a grey node -// double neighbor_counter = 0; -// for (int kk=0; kk<3; kk++){ -// for (int jj=0; jj<3; jj++){ -// for (int ii=0; ii<3; ii++){ -// -// int index = kk*9+jj*3+ii; -// double weight= Dst[index]; -// -// int idi=i+ii-1; -// int idj=j+jj-1; -// int idk=k+kk-1; -// -// if (idi < 0) idi=0; -// if (idj < 0) idj=0; -// if (idk < 0) idk=0; -// if (!(idi < Nx)) idi=Nx-1; -// if (!(idj < Ny)) idj=Ny-1; -// if (!(idk < Nz)) idk=Nz-1; -// -// int nn = idk*Nx*Ny + idj*Nx + idi; -// //if (Mask->id[nn] != VALUE){//Model-2:i.e. open nodes, impermeable solid nodes or any other type of greynodes -// if (Mask->id[nn] <=0){//Model-3:i.e. only impermeable solid nodes or any other type of greynodes -// neighbor_counter +=weight; -// } -// } -// } -// } -// if (neighbor_counter>0){ -// GreySolidPhi_host[n] = AFFINITY; -// } -// } -// } -// } -// } -// -// if (rank==0){ -// printf("Number of grey-solid labels: %lu \n",NLABELS); -// for (unsigned int idx=0; idx 0 + // water-wet < 0 + // neutral = 0 + double *SolidPotential_host = new double [Nx*Ny*Nz]; + double *GreySolidGrad_host = new double [3*Np]; + + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); + auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); + } + + for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + SolidPotential_host[n] = AFFINITY; + } + } + } + + // Calculate grey-solid color-gradient + double *Dst; + Dst = new double [3*3*3]; + for (int kk=0; kk<3; kk++){ + for (int jj=0; jj<3; jj++){ + for (int ii=0; ii<3; ii++){ + int index = kk*9+jj*3+ii; + Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); + } + } + } + double w_face = 1.f; + double w_edge = 0.5; + double w_corner = 0.f; + //local + Dst[13] = 0.f; + //faces + Dst[4] = w_face; + Dst[10] = w_face; + Dst[12] = w_face; + Dst[14] = w_face; + Dst[16] = w_face; + Dst[22] = w_face; + // corners + Dst[0] = w_corner; + Dst[2] = w_corner; + Dst[6] = w_corner; + Dst[8] = w_corner; + Dst[18] = w_corner; + Dst[20] = w_corner; + Dst[24] = w_corner; + Dst[26] = w_corner; + // edges + Dst[1] = w_edge; + Dst[3] = w_edge; + Dst[5] = w_edge; + Dst[7] = w_edge; + Dst[9] = w_edge; + Dst[11] = w_edge; + Dst[15] = w_edge; + Dst[17] = w_edge; + Dst[19] = w_edge; + Dst[21] = w_edge; + Dst[23] = w_edge; + Dst[25] = w_edge; + + for (int k=1; kSDs(i,j,k)<2.0){ + GreySolidGrad_host[idx+0*Np] = phi_x; + GreySolidGrad_host[idx+1*Np] = phi_y; + GreySolidGrad_host[idx+2*Np] = phi_z; + } + else{ + GreySolidGrad_host[idx+0*Np] = 0.0; + GreySolidGrad_host[idx+1*Np] = 0.0; + GreySolidGrad_host[idx+2*Np] = 0.0; + } + } + } + } + } + + + if (rank==0){ + printf("Number of Grey-solid labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxSendHalo(Phi); if (greyMode==true){ - //Model-1 + //Model-1&4 ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -1033,7 +1043,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - //Model-1 + //Model-1&4 ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); @@ -1067,7 +1077,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ } ScaLBL_Comm_Regular->SendHalo(Phi); if (greyMode==true){ - //Model-1 + //Model-1&4 ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -1093,7 +1103,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (greyMode==true){ - //Model-1 + //Model-1&4 ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); @@ -2207,168 +2217,159 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ //} ////----------------------------------------------------------------------------------------------------------// -void ScaLBL_GreyscaleColorModel::AssignGreySolidLabels()//Model-4 -{ - // ONLY initialize grey nodes - // Key input parameters: - // 1. GreySolidLabels - // labels for grey nodes - // 2. GreySolidAffinity - // affinity ranges [-1,1] - // oil-wet > 0 - // water-wet < 0 - // neutral = 0 - double *SolidPotential_host = new double [Nx*Ny*Nz]; - double *GreySolidGrad_host = new double [3*Np]; - size_t NLABELS=0; - signed char VALUE=0; - double AFFINITY=0.f; - - auto LabelList = greyscaleColor_db->getVector( "GreySolidLabels" ); - auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); - - NLABELS=LabelList.size(); - if (NLABELS != AffinityList.size()){ - ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); - } - - for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component - } - } - SolidPotential_host[n] = AFFINITY; - } - } - } - - // Calculate grey-solid color-gradient - double *Dst; - Dst = new double [3*3*3]; - for (int kk=0; kk<3; kk++){ - for (int jj=0; jj<3; jj++){ - for (int ii=0; ii<3; ii++){ - int index = kk*9+jj*3+ii; - Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); - } - } - } - double w_face = 1.f; - double w_edge = 0.5; - double w_corner = 0.f; - //local - Dst[13] = 0.f; - //faces - Dst[4] = w_face; - Dst[10] = w_face; - Dst[12] = w_face; - Dst[14] = w_face; - Dst[16] = w_face; - Dst[22] = w_face; - // corners - Dst[0] = w_corner; - Dst[2] = w_corner; - Dst[6] = w_corner; - Dst[8] = w_corner; - Dst[18] = w_corner; - Dst[20] = w_corner; - Dst[24] = w_corner; - Dst[26] = w_corner; - // edges - Dst[1] = w_edge; - Dst[3] = w_edge; - Dst[5] = w_edge; - Dst[7] = w_edge; - Dst[9] = w_edge; - Dst[11] = w_edge; - Dst[15] = w_edge; - Dst[17] = w_edge; - Dst[19] = w_edge; - Dst[21] = w_edge; - Dst[23] = w_edge; - Dst[25] = w_edge; - - for (int k=1; kSDs(i,j,k)<2.0){ - GreySolidGrad_host[idx+0*Np] = phi_x; - GreySolidGrad_host[idx+1*Np] = phi_y; - GreySolidGrad_host[idx+2*Np] = phi_z; - } - else{ - GreySolidGrad_host[idx+0*Np] = 0.0; - GreySolidGrad_host[idx+1*Np] = 0.0; - GreySolidGrad_host[idx+2*Np] = 0.0; - } - } - } - } - } - - - if (rank==0){ - printf("Number of Grey-solid labels: %lu \n",NLABELS); - for (unsigned int idx=0; idx 0 +// // water-wet < 0 +// // neutral = 0 +// +// double *GreySolidPhi_host = new double [Nx*Ny*Nz]; +// //initialize grey solid phase field +// for (int k=0;kgetVector( "GreySolidLabels" ); +// auto AffinityList = greyscaleColor_db->getVector( "GreySolidAffinity" ); +// +// size_t NLABELS=0; +// NLABELS=LabelList.size(); +// if (NLABELS != AffinityList.size()){ +// ERROR("Error: GreySolidLabels and GreySolidAffinity must be the same length! \n"); +// } +// +// double *Dst; +// Dst = new double [3*3*3]; +// for (int kk=0; kk<3; kk++){ +// for (int jj=0; jj<3; jj++){ +// for (int ii=0; ii<3; ii++){ +// int index = kk*9+jj*3+ii; +// Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); +// } +// } +// } +// double w_face = 1.f; +// double w_edge = 1.f; +// double w_corner = 0.f; +// //local +// Dst[13] = 0.f; +// //faces +// Dst[4] = w_face; +// Dst[10] = w_face; +// Dst[12] = w_face; +// Dst[14] = w_face; +// Dst[16] = w_face; +// Dst[22] = w_face; +// // corners +// Dst[0] = w_corner; +// Dst[2] = w_corner; +// Dst[6] = w_corner; +// Dst[8] = w_corner; +// Dst[18] = w_corner; +// Dst[20] = w_corner; +// Dst[24] = w_corner; +// Dst[26] = w_corner; +// // edges +// Dst[1] = w_edge; +// Dst[3] = w_edge; +// Dst[5] = w_edge; +// Dst[7] = w_edge; +// Dst[9] = w_edge; +// Dst[11] = w_edge; +// Dst[15] = w_edge; +// Dst[17] = w_edge; +// Dst[19] = w_edge; +// Dst[21] = w_edge; +// Dst[23] = w_edge; +// Dst[25] = w_edge; +// +// for (int k=1; kid[n]; +// double AFFINITY=0.f; +// // Assign the affinity from the paired list +// for (unsigned int idx=0; idx < NLABELS; idx++){ +// //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); +// if (VALUE == LabelList[idx]){ +// AFFINITY=AffinityList[idx]; +// idx = NLABELS; +// //Mask->id[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// +// if (VALUE>2){//i.e. a grey node +// double neighbor_counter = 0; +// for (int kk=0; kk<3; kk++){ +// for (int jj=0; jj<3; jj++){ +// for (int ii=0; ii<3; ii++){ +// +// int index = kk*9+jj*3+ii; +// double weight= Dst[index]; +// +// int idi=i+ii-1; +// int idj=j+jj-1; +// int idk=k+kk-1; +// +// if (idi < 0) idi=0; +// if (idj < 0) idj=0; +// if (idk < 0) idk=0; +// if (!(idi < Nx)) idi=Nx-1; +// if (!(idj < Ny)) idj=Ny-1; +// if (!(idk < Nz)) idk=Nz-1; +// +// int nn = idk*Nx*Ny + idj*Nx + idi; +// //if (Mask->id[nn] != VALUE){//Model-2:i.e. open nodes, impermeable solid nodes or any other type of greynodes +// if (Mask->id[nn] <=0){//Model-3:i.e. only impermeable solid nodes or any other type of greynodes +// neighbor_counter +=weight; +// } +// } +// } +// } +// if (neighbor_counter>0){ +// GreySolidPhi_host[n] = AFFINITY; +// } +// } +// } +// } +// } +// +// if (rank==0){ +// printf("Number of grey-solid labels: %lu \n",NLABELS); +// for (unsigned int idx=0; idx Date: Thu, 6 Aug 2020 15:41:40 -0400 Subject: [PATCH 197/270] Adding skeleton for electrokinetic LBM --- common/ScaLBL.cpp | 109 +++++++++ common/ScaLBL.h | 13 + cpu/Ion.cpp | 123 ++++++++++ cpu/Poisson.cpp | 123 ++++++++++ models/IonModel.cpp | 225 +++++++++++++++++ models/IonModel.h | 72 ++++++ models/PoissonSolver.cpp | 253 ++++++++++++++++++++ models/PoissonSolver.h | 68 ++++++ tests/lbpm_electrokinetic_dfh_simulator.cpp | 78 ++++++ 9 files changed, 1064 insertions(+) create mode 100644 cpu/Ion.cpp create mode 100644 cpu/Poisson.cpp create mode 100644 models/IonModel.cpp create mode 100644 models/IonModel.h create mode 100644 models/PoissonSolver.cpp create mode 100644 models/PoissonSolver.h create mode 100644 tests/lbpm_electrokinetic_dfh_simulator.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 07aa3f1d..eace3f3f 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1175,6 +1175,115 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ //................................................................................... } +/** + * + */ +void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + if (Lock==true){ + ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + } + else{ + Lock=true; + } + // assign tag of 19 to D3Q19 communication + sendtag = recvtag = 7; + ScaLBL_DeviceBarrier(); + // Pack the distributions + //...Packing for x face(2,8,10,12,14)................................ + ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*N],N); + + MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); + MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); + + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*N],N); + + MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); + MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); + + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*N],N); + + MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); + MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); + + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); + + MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); + MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); + + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*N],N); + + MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); + MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); + + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*N],N); + + //................................................................................... + // Send all the distributions + MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); + MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); + +} + + +void ScaLBL_Communicator::RecvD3Q7AA(double *Aq, int Component){ + + // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 + //................................................................................... + // Wait for completion of D3Q19 communication + MPI_Waitall(6,req1,stat1); + MPI_Waitall(6,req2,stat2); + ScaLBL_DeviceBarrier(); + + //................................................................................... + // NOTE: AA Routine writes to opposite + // Unpack the distributions on the device + //................................................................................... + //...Unpacking for x face(2,8,10,12,14)................................ + ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,&Aq[Component*N],N); + //................................................................................... + //...Packing for X face(1,7,9,11,13)................................ + ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,&Aq[Component*N],N); + //................................................................................... + //...Packing for y face(4,8,9,16,18)................................. + ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,&Aq[Component*N],N); + //................................................................................... + //...Packing for Y face(3,7,10,15,17)................................. + ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,&Aq[Component*N],N); + //................................................................................... + + if (BoundaryCondition > 0){ + if (kproc != 0){ + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*N],N); + } + if (kproc != nprocz-1){ + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*N],N); + } + } + else { + //...Packing for z face(6,12,13,16,17)................................ + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*N],N); + //...Packing for Z face(5,11,14,15,18)................................ + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*N],N); + } + + //................................................................................... + Lock=false; // unlock the communicator after communications complete + //................................................................................... + +} +/* + */ + void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ diff --git a/common/ScaLBL.h b/common/ScaLBL.h index dec8b3d1..23cf6936 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -72,6 +72,17 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); +// ION TRANSPORT MODEL +extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Velocity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + +extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Velocity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + + +// ION TRANSPORT MODEL +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, @@ -185,6 +196,8 @@ public: int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np); void SendD3Q19AA(double *dist); void RecvD3Q19AA(double *dist); + void SendD3Q7AA(double *fq, int Component); + void RecvD3Q7AA(double *fq, int Component) void BiSendD3Q7AA(double *Aq, double *Bq); void BiRecvD3Q7AA(double *Aq, double *Bq); void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq); diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp new file mode 100644 index 00000000..569c9a0a --- /dev/null +++ b/cpu/Ion.cpp @@ -0,0 +1,123 @@ +extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Velocity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; + // conserved momemnts + double rho,ux,uy,uz,uu; + // non-conserved moments + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + + for (int n=start; n 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + rho = f0+f2+f1+f4+f3+f6; + ux = Velocity[n]; + uy = Velocity[n+Np]; + uz = Velocity[n+2*Np]; + uu = 1.5*(ux*ux+uy*uy+uz*uz); + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx; + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx; + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy; + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy; + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz; + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz; + + + } +} \ No newline at end of file diff --git a/cpu/Poisson.cpp b/cpu/Poisson.cpp new file mode 100644 index 00000000..355e4223 --- /dev/null +++ b/cpu/Poisson.cpp @@ -0,0 +1,123 @@ +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + int n; + // conserved momemnts + double rho,ux,uy,uz,uu; + // non-conserved moments + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + + for (int n=start; n 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + rho = f0+f2+f1+f4+f3+f6+f5; + ux = f1-f2; + uy = f3-f4; + uz = f5-f6; + uu = 1.5*(ux*ux+uy*uy+uz*uz); + + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx; + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx; + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy; + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy; + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + + rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz; + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + + rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz; + + + } +} \ No newline at end of file diff --git a/models/IonModel.cpp b/models/IonModel.cpp new file mode 100644 index 00000000..38088587 --- /dev/null +++ b/models/IonModel.cpp @@ -0,0 +1,225 @@ +/* + * Multi-relaxation time LBM Model + */ +#include "models/MRT.h" +#include "models/ElectroModel.h" +#include "analysis/distance.h" +#include "common/ReadMicroCT.h" + +ScaLBL_IonModel::ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + +} +ScaLBL_IonModel::~ScaLBL_IonModel(){ + +} + +void ScaLBL_IonModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + ion_db = db->getDatabase( "Ions" ); + + tau = 1.0; + timestepMax = 100000; + tolerance = 1.0e-8; + Fx = Fy = 0.0; + Fz = 1.0e-5; + + // Color Model parameters + if (ion_db->keyExists( "timestepMax" )){ + timestepMax = mrt_db->getScalar( "timestepMax" ); + } + + mu=(tau-0.5)/3.0; +} +void ScaLBL_IonModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + + N = Nx*Ny*Nz; + Distance.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_IonModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",Dm->rank()); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, comm ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else{ + Mask->ReadIDs(); + } + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + // Solve for the position of the solid phase + for (int k=0;kid[n] > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs); + if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + CalcDist(Distance,id_solid,*Dm); + if (rank == 0) cout << "Domain set." << endl; +} + +void ScaLBL_IonModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + int rank=Mask->rank(); + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout \n"); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &fq, number_ion_species*7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Ci, number_ion_species*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ChargeDensity, sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + MPI_Barrier(comm); + +} + +void ScaLBL_IonModel::Initialize(){ + /* + * This function initializes model + */ + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); +} + +void ScaLBL_IonModel::Run(double *Velocity){ + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + starttime = MPI_Wtime(); + if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); + if (rank==0) printf("********************************************************\n"); + timestep=0; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + /* ... */ + ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q7_AAeven_Ion(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + /* ... */ + ScaLBL_D3Q7_AAeven_Ion(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + } + //************************************************************************/ + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + +} + diff --git a/models/IonModel.h b/models/IonModel.h new file mode 100644 index 00000000..f6ffad5e --- /dev/null +++ b/models/IonModel.h @@ -0,0 +1,72 @@ +/* + * Multi-relaxation time LBM Model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "analysis/Minkowski.h" +#include "ProfilerApp.h" + +class ScaLBL_IonModel{ +public: + ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_IonModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(double *Velocity); + void VelocityField(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + double tau,mu; + double Fx,Fy,Fz,flux; + double din,dout; + double tolerance; + + int number_ion_species; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr ion_db; + + IntArray Map; + DoubleArray Distance; + int *NeighborList; + double *fq; + double *Ci; + double *ChargeDensity; + +private: + MPI_Comm comm; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); +}; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp new file mode 100644 index 00000000..f4f15224 --- /dev/null +++ b/models/PoissonSolver.cpp @@ -0,0 +1,253 @@ +/* + * Multi-relaxation time LBM Model + */ +#include "models/MRT.h" +#include "models/ElectroModel.h" +#include "analysis/distance.h" +#include "common/ReadMicroCT.h" + +ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + +} +ScaLBL_Poisson::~ScaLBL_Poisson(){ + +} + +void ScaLBL_Poisson::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + mrt_db = db->getDatabase( "MRT" ); + electric_db = db->getDatabase( "Electrochemistry" ); + + tau = 1.0; + timestepMax = 100000; + tolerance = 1.0e-8; + Fx = Fy = 0.0; + Fz = 1.0e-5; + + // Color Model parameters + if (mrt_db->keyExists( "timestepMax" )){ + timestepMax = mrt_db->getScalar( "timestepMax" ); + } + if (mrt_db->keyExists( "tolerance" )){ + tolerance = mrt_db->getScalar( "tolerance" ); + } + if (mrt_db->keyExists( "tau" )){ + tau = mrt_db->getScalar( "tau" ); + } + if (mrt_db->keyExists( "F" )){ + Fx = mrt_db->getVector( "F" )[0]; + Fy = mrt_db->getVector( "F" )[1]; + Fz = mrt_db->getVector( "F" )[2]; + } + if (mrt_db->keyExists( "Restart" )){ + Restart = mrt_db->getScalar( "Restart" ); + } + if (mrt_db->keyExists( "din" )){ + din = mrt_db->getScalar( "din" ); + } + if (mrt_db->keyExists( "dout" )){ + dout = mrt_db->getScalar( "dout" ); + } + if (mrt_db->keyExists( "flux" )){ + flux = mrt_db->getScalar( "flux" ); + } + + // Read domain parameters + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + + + mu=(tau-0.5)/3.0; +} +void ScaLBL_Poisson::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + + N = Nx*Ny*Nz; + Distance.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_Poisson::ReadInput(){ + + sprintf(LocalRankString,"%05d",Dm->rank()); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, comm ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else{ + Mask->ReadIDs(); + } + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + // Solve for the position of the solid phase + for (int k=0;kid[n] > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs); + if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + CalcDist(Distance,id_solid,*Dm); + if (rank == 0) cout << "Domain set." << endl; +} + +void ScaLBL_Poisson::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + int rank=Mask->rank(); + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout \n"); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &fq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + MPI_Barrier(comm); + +} + +void ScaLBL_Poisson::Initialize(){ + /* + * This function initializes model + */ + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); +} + +void ScaLBL_Poisson::Run(double *ChargeDensity){ + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + starttime = MPI_Wtime(); + if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); + if (rank==0) printf("********************************************************\n"); + timestep=0; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + /* ... */ + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q7_AAeven_Poisson(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + /* ... */ + ScaLBL_D3Q7_AAeven_Poisson(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + } + //************************************************************************/ + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + +} diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h new file mode 100644 index 00000000..63e10df0 --- /dev/null +++ b/models/PoissonSolver.h @@ -0,0 +1,68 @@ +/* + * Multi-relaxation time LBM Model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "analysis/Minkowski.h" +#include "ProfilerApp.h" + +class ScaLBL_Poisson{ +public: + ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_Poisson(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(double *ChargeDensity); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + double tau,mu; + double Fx,Fy,Fz,flux; + double din,dout; + double tolerance; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr electric_db; + + IntArray Map; + DoubleArray Distance; + int *NeighborList; + double *fq; + double *Psi; + +private: + MPI_Comm comm; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); +}; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp new file mode 100644 index 00000000..e3765d12 --- /dev/null +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "models/DFHModel.h" +#include "models/IonModel.h" +#include "models/PoissonSolver.h" + +//#define WRE_SURFACES + +/* + * Simulator for two-phase flow in porous media + * James E. McClure 2013-2014 + */ + +using namespace std; + +//************************************************************************* +// Implementation of Two-Phase Immiscible LBM using CUDA +//************************************************************************* + +int main(int argc, char **argv) +{ + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_support Date: Thu, 6 Aug 2020 15:42:36 -0400 Subject: [PATCH 198/270] Adding skeleton for electrokinetic LBM --- models/ElectroKinetic.cpp | 1568 ------------------------------------- models/ElectroKinetic.h | 88 --- 2 files changed, 1656 deletions(-) delete mode 100644 models/ElectroKinetic.cpp delete mode 100644 models/ElectroKinetic.h diff --git a/models/ElectroKinetic.cpp b/models/ElectroKinetic.cpp deleted file mode 100644 index a8c21a75..00000000 --- a/models/ElectroKinetic.cpp +++ /dev/null @@ -1,1568 +0,0 @@ -/* -color lattice boltzmann model - */ -#include "models/ColorModel.h" -#include "analysis/distance.h" -#include "analysis/morphology.h" -#include "common/Communication.h" -#include "common/ReadMicroCT.h" -#include -#include - -ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) -{ - REVERSE_FLOW_DIRECTION = false; -} -ScaLBL_ColorModel::~ScaLBL_ColorModel(){ - -} - -/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) -{ - int q,n; - double value; - ofstream File(FILENAME,ios::binary); - for (n=0; n( filename ); - domain_db = db->getDatabase( "Domain" ); - color_db = db->getDatabase( "Color" ); - analysis_db = db->getDatabase( "Analysis" ); - vis_db = db->getDatabase( "Visualization" ); - - // set defaults - timestepMax = 100000; - tauA = tauB = 1.0; - rhoA = rhoB = 1.0; - Fx = Fy = Fz = 0.0; - alpha=1e-3; - beta=0.95; - Restart=false; - din=dout=1.0; - flux=0.0; - - // Color Model parameters - if (color_db->keyExists( "timestepMax" )){ - timestepMax = color_db->getScalar( "timestepMax" ); - } - if (color_db->keyExists( "tauA" )){ - tauA = color_db->getScalar( "tauA" ); - } - if (color_db->keyExists( "tauB" )){ - tauB = color_db->getScalar( "tauB" ); - } - if (color_db->keyExists( "rhoA" )){ - rhoA = color_db->getScalar( "rhoA" ); - } - if (color_db->keyExists( "rhoB" )){ - rhoB = color_db->getScalar( "rhoB" ); - } - if (color_db->keyExists( "F" )){ - Fx = color_db->getVector( "F" )[0]; - Fy = color_db->getVector( "F" )[1]; - Fz = color_db->getVector( "F" )[2]; - } - if (color_db->keyExists( "alpha" )){ - alpha = color_db->getScalar( "alpha" ); - } - if (color_db->keyExists( "beta" )){ - beta = color_db->getScalar( "beta" ); - } - if (color_db->keyExists( "Restart" )){ - Restart = color_db->getScalar( "Restart" ); - } - if (color_db->keyExists( "din" )){ - din = color_db->getScalar( "din" ); - } - if (color_db->keyExists( "dout" )){ - dout = color_db->getScalar( "dout" ); - } - if (color_db->keyExists( "flux" )){ - flux = color_db->getScalar( "flux" ); - } - inletA=1.f; - inletB=0.f; - outletA=0.f; - outletB=1.f; - //if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details) - - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } - - // Override user-specified boundary condition for specific protocols - auto protocol = color_db->getWithDefault( "protocol", "none" ); - if (protocol == "seed water"){ - if (BoundaryCondition != 0 && BoundaryCondition != 5){ - BoundaryCondition = 0; - if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n"); - } - domain_db->putScalar( "BC", BoundaryCondition ); - } - else if (protocol == "open connected oil"){ - if (BoundaryCondition != 0 && BoundaryCondition != 5){ - BoundaryCondition = 0; - if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n"); - } - domain_db->putScalar( "BC", BoundaryCondition ); - } - else if (protocol == "shell aggregation"){ - if (BoundaryCondition != 0 && BoundaryCondition != 5){ - BoundaryCondition = 0; - if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n"); - } - domain_db->putScalar( "BC", BoundaryCondition ); - } -} - -void ScaLBL_ColorModel::SetDomain(){ - Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - // domain parameters - Nx = Dm->Nx; - Ny = Dm->Ny; - Nz = Dm->Nz; - Lx = Dm->Lx; - Ly = Dm->Ly; - Lz = Dm->Lz; - N = Nx*Ny*Nz; - id = new signed char [N]; - for (int i=0; iid[i] = 1; // initialize this way - //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object - Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object - MPI_Barrier(comm); - Dm->CommInit(); - MPI_Barrier(comm); - // Read domain parameters - rank = Dm->rank(); - nprocx = Dm->nprocx(); - nprocy = Dm->nprocy(); - nprocz = Dm->nprocz(); -} - -void ScaLBL_ColorModel::ReadInput(){ - - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - - if (color_db->keyExists( "image_sequence" )){ - auto ImageList = color_db->getVector( "image_sequence"); - int IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); - std::string first_image = ImageList[IMAGE_INDEX]; - Mask->Decomp(first_image); - IMAGE_INDEX++; - } - else if (domain_db->keyExists( "GridFile" )){ - // Read the local domain data - auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD ); - // Fill the halo (assuming GCW of 1) - array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; - ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; - ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); - fillHalo fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); - Array id_view; - id_view.viewRaw( size1, Mask->id ); - fill.copy( input_id, id_view ); - fill.fill( id_view ); - } - else if (domain_db->keyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask->Decomp(Filename); - } - else{ - Mask->ReadIDs(); - } - for (int i=0; iid[i]; // save what was read - - // Generate the signed distance map - // Initialize the domain and communication - Array id_solid(Nx,Ny,Nz); - // Solve for the position of the solid phase - for (int k=0;kid[n]; - if (label > 0) id_solid(i,j,k) = 1; - else id_solid(i,j,k) = 0; - } - } - } - // Initialize the signed distance function - for (int k=0;kSDs(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0; - } - } - } -// MeanFilter(Averages->SDs); - if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); - CalcDist(Averages->SDs,id_solid,*Mask); - - if (rank == 0) cout << "Domain set." << endl; - - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); -} - -void ScaLBL_ColorModel::AssignComponentLabels(double *phase) -{ - size_t NLABELS=0; - signed char VALUE=0; - double AFFINITY=0.f; - - auto LabelList = color_db->getVector( "ComponentLabels" ); - auto AffinityList = color_db->getVector( "ComponentAffinity" ); - - NLABELS=LabelList.size(); - if (NLABELS != AffinityList.size()){ - ERROR("Error: ComponentLabels and ComponentAffinity must be the same length! \n"); - } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - // Assign the labels - - for (size_t idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component - } - } - // fluid labels are reserved - if (VALUE == 1) AFFINITY=1.0; - else if (VALUE == 2) AFFINITY=-1.0; - phase[n] = AFFINITY; - } - } - } - - // Set Dm to match Mask - for (int i=0; iid[i] = Mask->id[i]; - - for (size_t idx=0; idxComm, label_count[idx]); - - if (rank==0){ - printf("Component labels: %lu \n",NLABELS); - for (unsigned int idx=0; idxid[i] = Mask->id[i]; - Mask->CommInit(); - Np=Mask->PoreCount(); - //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - // Create a communicator for the device (will use optimized layout) - // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); - ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); - - int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); - Map.resize(Nx,Ny,Nz); Map.fill(-2); - auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - MPI_Barrier(comm); - - //........................................................................... - // MAIN VARIABLES ALLOCATED HERE - //........................................................................... - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - //......................device distributions................................. - dist_mem_size = Np*sizeof(double); - neighborSize=18*(Np*sizeof(int)); - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); - fflush(stdout); - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; kLastExterior(); idx++){ - auto n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n", n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - auto n = TmpMap[idx]; - if ( n > Nx*Ny*Nz ){ - printf("Bad value! idx=%i \n",n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; - - // copy the neighbor list - ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - // initialize phi based on PhaseLabel (include solid component labels) - double *PhaseLabel; - PhaseLabel = new double[N]; - AssignComponentLabels(PhaseLabel); - ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); -} - -/******************************************************** - * AssignComponentLabels * - ********************************************************/ - -void ScaLBL_ColorModel::Initialize(){ - - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); - /* - * This function initializes model - */ - if (Restart == true){ - if (rank==0){ - printf("Reading restart file! \n"); - } - - // Read in the restart file to CPU buffers - int *TmpMap; - TmpMap = new int[Np]; - - double *cPhi, *cDist, *cDen; - cPhi = new double[N]; - cDen = new double[2*Np]; - cDist = new double[19*Np]; - ScaLBL_CopyToHost(TmpMap, dvcMap, Np*sizeof(int)); - ScaLBL_CopyToHost(cPhi, Phi, N*sizeof(double)); - - ifstream File(LocalRestartFile,ios::binary); - int idx; - double value,va,vb; - for (int n=0; nLastExterior(); n++){ - va = cDen[n]; - vb = cDen[Np + n]; - value = (va-vb)/(va+vb); - idx = TmpMap[n]; - if (!(idx < 0) && idxFirstInterior(); nLastInterior(); n++){ - va = cDen[n]; - vb = cDen[Np + n]; - value = (va-vb)/(va+vb); - idx = TmpMap[n]; - if (!(idx < 0) && idxLastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - - // establish reservoirs for external bC - if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - } - ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); -} - -void ScaLBL_ColorModel::Run(){ - int nprocs=nprocx*nprocy*nprocz; - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - int IMAGE_INDEX = 0; - int IMAGE_COUNT = 0; - std::vector ImageList; - bool SET_CAPILLARY_NUMBER = false; - bool RESCALE_FORCE = false; - bool MORPH_ADAPT = false; - bool USE_MORPH = false; - bool USE_SEED = false; - bool USE_DIRECT = false; - bool USE_MORPHOPEN_OIL = false; - int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine - int MIN_STEADY_TIMESTEPS = 100000; - int MAX_STEADY_TIMESTEPS = 200000; - int RESCALE_FORCE_AFTER_TIMESTEP = 0; - int RAMP_TIMESTEPS = 0;//50000; // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in) - int CURRENT_MORPH_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) - int CURRENT_STEADY_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) - int morph_interval = 100000; - int analysis_interval = 1000; // number of timesteps in between in situ analysis - int morph_timesteps = 0; - double morph_delta = 0.0; - double seed_water = 0.0; - double capillary_number = 0.0; - double tolerance = 0.01; - double Ca_previous = 0.f; - double initial_volume = 0.0; - double delta_volume = 0.0; - double delta_volume_target = 0.0; - - /* history for morphological algoirthm */ - double KRA_MORPH_FACTOR=0.5; - double volA_prev = 0.0; - double log_krA_prev = 1.0; - double log_krA_target = 1.0; - double log_krA = 1.0; - double slope_krA_volume = 0.0; - if (color_db->keyExists( "vol_A_previous" )){ - volA_prev = color_db->getScalar( "vol_A_previous" ); - } - if (color_db->keyExists( "log_krA_previous" )){ - log_krA_prev = color_db->getScalar( "log_krA_previous" ); - } - if (color_db->keyExists( "krA_morph_factor" )){ - KRA_MORPH_FACTOR = color_db->getScalar( "krA_morph_factor" ); - } - - /* defaults for simulation protocols */ - auto protocol = color_db->getWithDefault( "protocol", "none" ); - if (protocol == "image sequence"){ - // Get the list of images - USE_DIRECT = true; - ImageList = color_db->getVector( "image_sequence"); - IMAGE_INDEX = color_db->getWithDefault( "image_index", 0 ); - IMAGE_COUNT = ImageList.size(); - morph_interval = 10000; - USE_MORPH = true; - } - else if (protocol == "seed water"){ - morph_delta = -0.05; - seed_water = 0.01; - USE_SEED = true; - USE_MORPH = true; - } - else if (protocol == "open connected oil"){ - morph_delta = -0.05; - USE_MORPH = true; - USE_MORPHOPEN_OIL = true; - } - else if (protocol == "shell aggregation"){ - morph_delta = -0.05; - USE_MORPH = true; - } - if (color_db->keyExists( "capillary_number" )){ - capillary_number = color_db->getScalar( "capillary_number" ); - SET_CAPILLARY_NUMBER=true; - } - if (color_db->keyExists( "rescale_force_after_timestep" )){ - RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar( "rescale_force_after_timestep" ); - RESCALE_FORCE = true; - } - if (color_db->keyExists( "timestep" )){ - timestep = color_db->getScalar( "timestep" ); - } - if (BoundaryCondition != 0 && BoundaryCondition != 5 && SET_CAPILLARY_NUMBER==true){ - if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 or 5 \n"); - SET_CAPILLARY_NUMBER=false; - } - if (analysis_db->keyExists( "seed_water" )){ - seed_water = analysis_db->getScalar( "seed_water" ); - if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water); - USE_SEED = true; - } - if (analysis_db->keyExists( "morph_delta" )){ - morph_delta = analysis_db->getScalar( "morph_delta" ); - if (rank == 0) printf("Target volume change %f (morph_delta) \n",morph_delta); - } - if (analysis_db->keyExists( "morph_interval" )){ - morph_interval = analysis_db->getScalar( "morph_interval" ); - USE_MORPH = true; - } - if (analysis_db->keyExists( "use_morphopen_oil" )){ - USE_MORPHOPEN_OIL = analysis_db->getScalar( "use_morphopen_oil" ); - if (rank == 0 && USE_MORPHOPEN_OIL) printf("Volume change by morphological opening \n"); - USE_MORPH = true; - } - if (analysis_db->keyExists( "tolerance" )){ - tolerance = analysis_db->getScalar( "tolerance" ); - } - if (analysis_db->keyExists( "analysis_interval" )){ - analysis_interval = analysis_db->getScalar( "analysis_interval" ); - } - if (analysis_db->keyExists( "min_steady_timesteps" )){ - MIN_STEADY_TIMESTEPS = analysis_db->getScalar( "min_steady_timesteps" ); - } - if (analysis_db->keyExists( "max_steady_timesteps" )){ - MAX_STEADY_TIMESTEPS = analysis_db->getScalar( "max_steady_timesteps" ); - } - if (analysis_db->keyExists( "max_morph_timesteps" )){ - MAX_MORPH_TIMESTEPS = analysis_db->getScalar( "max_morph_timesteps" ); - } - - - if (rank==0){ - printf("********************************************************\n"); - if (protocol == "image sequence"){ - printf(" using protocol = image sequence \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - std::string first_image = ImageList[IMAGE_INDEX]; - printf(" first image in sequence: %s ***\n", first_image.c_str()); - } - else if (protocol == "seed water"){ - printf(" using protocol = seed water \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - printf(" morph_delta = %f \n",morph_delta); - printf(" seed_water = %f \n",seed_water); - } - else if (protocol == "open connected oil"){ - printf(" using protocol = open connected oil \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - printf(" morph_delta = %f \n",morph_delta); - } - else if (protocol == "shell aggregation"){ - printf(" using protocol = shell aggregation \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - printf(" morph_delta = %f \n",morph_delta); - } - printf("No. of timesteps: %i \n", timestepMax); - fflush(stdout); - } - - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... - - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - //std::shared_ptr analysis_db; - bool Regular = false; - auto current_db = db->cloneDatabase(); - runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); - //analysis.createThreads( analysis_method, 4 ); - while (timestep < timestepMax ) { - //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } - PROFILE_START("Update"); - // *************ODD TIMESTEP************* - timestep++; - // Compute the Phase indicator field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - - // Perform the collision operation - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - if (BoundaryCondition > 0 && BoundaryCondition < 5){ - ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - // Halo exchange for phase field - ScaLBL_Comm_Regular->SendHalo(Phi); - - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(Phi); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - if (BoundaryCondition == 4){ - din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - else if (BoundaryCondition == 5){ - ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); - ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); - } - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - - // *************EVEN TIMESTEP************* - timestep++; - // Compute the Phase indicator field - ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - - // Perform the collision operation - ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - // Halo exchange for phase field - if (BoundaryCondition > 0 && BoundaryCondition < 5){ - ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - ScaLBL_Comm_Regular->SendHalo(Phi); - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(Phi); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 3){ - ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - else if (BoundaryCondition == 4){ - din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - else if (BoundaryCondition == 5){ - ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); - ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); - } - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - //************************************************************************ - PROFILE_STOP("Update"); - - if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ - printf("%i %f \n",timestep,din); - } - // Run the analysis - analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - - // allow initial ramp-up to get closer to steady state - if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ - analysis.finish(); - CURRENT_STEADY_TIMESTEPS += analysis_interval; - - double volB = Averages->gwb.V; - double volA = Averages->gnb.V; - volA /= Dm->Volume; - volB /= Dm->Volume;; - //initial_volume = volA*Dm->Volume; - double vA_x = Averages->gnb.Px/Averages->gnb.M; - double vA_y = Averages->gnb.Py/Averages->gnb.M; - double vA_z = Averages->gnb.Pz/Averages->gnb.M; - double vB_x = Averages->gwb.Px/Averages->gwb.M; - double vB_y = Averages->gwb.Py/Averages->gwb.M; - double vB_z = Averages->gwb.Pz/Averages->gwb.M; - double muA = rhoA*(tauA-0.5)/3.f; - double muB = rhoB*(tauB-0.5)/3.f; - double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - double dir_x = Fx/force_mag; - double dir_y = Fy/force_mag; - double dir_z = Fz/force_mag; - if (force_mag == 0.0){ - // default to z direction - dir_x = 0.0; - dir_y = 0.0; - dir_z = 1.0; - force_mag = 1.0; - } - double current_saturation = volB/(volA+volB); - double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); - double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); - double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); - - if ( morph_timesteps > morph_interval ){ - - bool isSteady = false; - if ( (fabs((Ca - Ca_previous)/Ca) < tolerance && CURRENT_STEADY_TIMESTEPS > MIN_STEADY_TIMESTEPS)) - isSteady = true; - if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) - isSteady = true; - if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ - RESCALE_FORCE = false; - double RESCALE_FORCE_FACTOR = capillary_number / Ca; - if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; - if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; - Fx *= RESCALE_FORCE_FACTOR; - Fy *= RESCALE_FORCE_FACTOR; - Fz *= RESCALE_FORCE_FACTOR; - force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - if ( isSteady ){ - MORPH_ADAPT = true; - CURRENT_MORPH_TIMESTEPS=0; - delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - //****** ENDPOINT ADAPTATION ********/ - double krA_TMP= fabs(muA*flow_rate_A / force_mag); - double krB_TMP= fabs(muB*flow_rate_B / force_mag); - log_krA = log(krA_TMP); - if (krA_TMP < 0.0){ - // cannot do endpoint adaptation if kr is negative - log_krA = log_krA_prev; - } - else if (krA_TMP < krB_TMP && morph_delta > 0.0){ - /** morphological target based on relative permeability for A **/ - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); - if (rank==0){ - printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); - } - } - log_krA_prev = log_krA; - volA_prev = volA; - //******************************** **/ - /** compute averages & write data **/ - Averages->Full(); - Averages->Write(timestep); - analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - analysis.finish(); - - if (rank==0){ - printf("** WRITE STEADY POINT *** "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - double h = Dm->voxel_length; - // pressures - double pA = Averages->gnb.p; - double pB = Averages->gwb.p; - double pAc = Averages->gnc.p; - double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*6.0*alpha); - double pAB_connected = (pAc-pBc)/(h*6.0*alpha); - // connected contribution - double Vol_nc = Averages->gnc.V/Dm->Volume; - double Vol_wc = Averages->gwc.V/Dm->Volume; - double Vol_nd = Averages->gnd.V/Dm->Volume; - double Vol_wd = Averages->gwd.V/Dm->Volume; - double Mass_n = Averages->gnc.M + Averages->gnd.M; - double Mass_w = Averages->gwc.M + Averages->gwd.M; - double vAc_x = Averages->gnc.Px/Mass_n; - double vAc_y = Averages->gnc.Py/Mass_n; - double vAc_z = Averages->gnc.Pz/Mass_n; - double vBc_x = Averages->gwc.Px/Mass_w; - double vBc_y = Averages->gwc.Py/Mass_w; - double vBc_z = Averages->gwc.Pz/Mass_w; - // disconnected contribution - double vAd_x = Averages->gnd.Px/Mass_n; - double vAd_y = Averages->gnd.Py/Mass_n; - double vAd_z = Averages->gnd.Pz/Mass_n; - double vBd_x = Averages->gwd.Px/Mass_w; - double vBd_y = Averages->gwd.Py/Mass_w; - double vBd_z = Averages->gwd.Pz/Mass_w; - - double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); - double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); - double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); - double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); - - double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); - double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); - - double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); - double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); - - double kAeff = h*h*muA*(flow_rate_A)/(force_mag); - double kBeff = h*h*muB*(flow_rate_B)/(force_mag); - - double viscous_pressure_drop = (rhoA*volA + rhoB*volB)*force_mag; - double Mobility = muA/muB; - - bool WriteHeader=false; - FILE * kr_log_file = fopen("relperm.csv","r"); - if (kr_log_file != NULL) - fclose(kr_log_file); - else - WriteHeader=true; - kr_log_file = fopen("relperm.csv","a"); - if (WriteHeader) - fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); - - fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); - fclose(kr_log_file); - - printf(" Measured capillary number %f \n ",Ca); - } - if (SET_CAPILLARY_NUMBER ){ - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - color_db->putVector("F",{Fx,Fy,Fz}); - } - - CURRENT_STEADY_TIMESTEPS = 0; - } - else{ - if (rank==0){ - printf("** Continue to simulate steady *** \n "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); - } - } - morph_timesteps=0; - Ca_previous = Ca; - } - - if (MORPH_ADAPT ){ - CURRENT_MORPH_TIMESTEPS += analysis_interval; - if (USE_DIRECT){ - // Use image sequence - IMAGE_INDEX++; - MORPH_ADAPT = false; - if (IMAGE_INDEX < IMAGE_COUNT){ - std::string next_image = ImageList[IMAGE_INDEX]; - if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); - color_db->putScalar("image_index",IMAGE_INDEX); - ImageInit(next_image); - } - else{ - if (rank==0) printf("Finished simulating image sequence \n"); - timestep = timestepMax; - } - } - else if (USE_SEED){ - delta_volume = volA*Dm->Volume - initial_volume; - CURRENT_MORPH_TIMESTEPS += analysis_interval; - double massChange = SeedPhaseField(seed_water); - if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); - } - else if (USE_MORPHOPEN_OIL){ - delta_volume = volA*Dm->Volume - initial_volume; - if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); - MorphOpenConnected(delta_volume_target); - } - else { - if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); - //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A - delta_volume += MorphInit(beta,delta_volume_target-delta_volume); - } - - if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; - } - else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - initial_volume = volA*Dm->Volume; - delta_volume = 0.0; - RESCALE_FORCE = true; - if (RESCALE_FORCE_AFTER_TIMESTEP > 0) - RESCALE_FORCE = true; - } - } - morph_timesteps += analysis_interval; - } - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - } - analysis.finish(); - PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_color_simulator",1); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // ************************************************************************ -} - -double ScaLBL_ColorModel::ImageInit(std::string Filename){ - - if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); - Mask->Decomp(Filename); - for (int i=0; iid[i]; // save what was read - for (int i=0; iid[i] = Mask->id[i]; // save what was read - - double *PhaseLabel; - PhaseLabel = new double[Nx*Ny*Nz]; - AssignComponentLabels(PhaseLabel); - - double Count = 0.0; - double PoreCount = 0.0; - for (int k=1; kComm, Count); - PoreCount=sumReduce( Dm->Comm, PoreCount); - - if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); - ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - - ScaLBL_D3Q19_Init(fq, Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - - ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); - - double saturation = Count/PoreCount; - return saturation; - -} - -double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){ - - int nx = Nx; - int ny = Ny; - int nz = Nz; - int n; - int N = nx*ny*nz; - double volume_change=0.0; - - if (target_volume_change < 0.0){ - Array id_solid(nx,ny,nz); - Array phase_label(nx,ny,nz); - DoubleArray distance(Nx,Ny,Nz); - DoubleArray phase(nx,ny,nz); - signed char *id_connected; - id_connected = new signed char [nx*ny*nz]; - - ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); - - // Extract only the connected part of NWP - BlobIDstruct new_index; - double vF=0.0; double vS=0.0; - ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); - MPI_Barrier(Dm->Comm); - - long long count_connected=0; - long long count_porespace=0; - long long count_water=0; - for (int k=1; k 0){ - count_porespace++; - } - if (id[n] == 2){ - count_water++; - } - } - } - } - count_connected=sumReduce( Dm->Comm, count_connected); - count_porespace=sumReduce( Dm->Comm, count_porespace); - count_water=sumReduce( Dm->Comm, count_water); - - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } - } - } - - int count_morphopen=0.0; - for (int k=1; kComm, count_morphopen); - volume_change = double(count_morphopen - count_connected); - - if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); - - ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - } - } - return(volume_change); -} -double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){ - srand(time(NULL)); - double mass_loss =0.f; - double count =0.f; - double *Aq_tmp, *Bq_tmp; - - Aq_tmp = new double [7*Np]; - Bq_tmp = new double [7*Np]; - - ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); - - - for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ - double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; - double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; - double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; - double phase_id = (dA - dB) / (dA + dB); - if (phase_id > 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - } - mass_loss += random_value*seed_water_in_oil; - } - - for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ - double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; - double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; - double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; - double phase_id = (dA - dB) / (dA + dB); - if (phase_id > 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - } - mass_loss += random_value*seed_water_in_oil; - } - - count= sumReduce( Dm->Comm, count); - mass_loss= sumReduce( Dm->Comm, mass_loss); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); - - // Need to initialize Aq, Bq, Den, Phi directly - //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); - - return(mass_loss); -} - -double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){ - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - double vF = 0.f; - double vS = 0.f; - double delta_volume; - double WallFactor = 0.0; - bool USE_CONNECTED_NWP = false; - - DoubleArray phase(Nx,Ny,Nz); - IntArray phase_label(Nx,Ny,Nz);; - DoubleArray phase_distance(Nx,Ny,Nz); - Array phase_id(Nx,Ny,Nz); - fillHalo fillDouble(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); - - - // Basic algorithm to - // 1. Copy phase field to CPU - ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); - - double count = 0.f; - for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f) count+=1.f; - } - } - } - double volume_initial = sumReduce( Dm->Comm, count); - /* - sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); - FILE *INPUT = fopen(LocalRankFilename,"wb"); - fwrite(phase.data(),8,N,INPUT); - fclose(INPUT); - */ - // 2. Identify connected components of phase field -> phase_label - - double volume_connected = 0.0; - double second_biggest = 0.0; - if (USE_CONNECTED_NWP){ - BlobIDstruct new_index; - ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(Dm->Comm); - - // only operate on component "0" - count = 0.0; - - for (int k=0; kComm, count); - second_biggest = sumReduce( Dm->Comm, second_biggest); - } - else { - // use the whole NWP - for (int k=0; kSDs(i,j,k) > 0.f){ - if (phase(i,j,k) > 0.f ){ - phase_id(i,j,k) = 0; - } - else { - phase_id(i,j,k) = 1; - } - } - else { - phase_id(i,j,k) = 1; - } - } - } - } - } - - /*int reach_x, reach_y, reach_z; - for (int k=0; k phase_distance - CalcDist(phase_distance,phase_id,*Dm); - - double temp,value; - double factor=0.5/beta; - for (int k=0; k 1.f) value=1.f; - if (value < -1.f) value=-1.f; - // temp -- distance based on analytical form McClure, Prins et al, Comp. Phys. Comm. - temp = -factor*log((1.0+value)/(1.0-value)); - /// use this approximation close to the object - if (fabs(value) < 0.8 && Averages->SDs(i,j,k) > 1.f ){ - phase_distance(i,j,k) = temp; - } - // erase the original object - phase(i,j,k) = -1.0; - } - } - } - } - - if (USE_CONNECTED_NWP){ - if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){ - // if connected volume is less than 2% just delete the whole thing - if (rank==0) printf("Connected region has shrunk! \n"); - REVERSE_FLOW_DIRECTION = true; - } - -/* else{*/ - if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest ); - } - if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); - double target_delta_volume_incremental = target_delta_volume; - if (fabs(target_delta_volume) > 0.01*volume_initial) - target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); - - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - //phase(i,j,k) = -1.0; - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } - } - } - fillDouble.fill(phase); - //} - - count = 0.f; - for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f){ - count+=1.f; - } - } - } - } - double volume_final= sumReduce( Dm->Comm, count); - - delta_volume = (volume_final-volume_initial); - if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); - if (rank == 0) printf(" new saturation = %f \n", volume_final/(0.238323*double((Nx-2)*(Ny-2)*(Nz-2)*nprocs))); - - // 6. copy back to the device - //if (rank==0) printf("MorphInit: copy data back to device\n"); - ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); - /* - sprintf(LocalRankFilename,"dist_final.%05i.raw",rank); - FILE *DIST = fopen(LocalRankFilename,"wb"); - fwrite(phase_distance.data(),8,N,DIST); - fclose(DIST); - - sprintf(LocalRankFilename,"phi_final.%05i.raw",rank); - FILE *PHI = fopen(LocalRankFilename,"wb"); - fwrite(phase.data(),8,N,PHI); - fclose(PHI); - */ - // 7. Re-initialize phase field and density - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - } - return delta_volume; -} - -void ScaLBL_ColorModel::WriteDebug(){ - // Copy back final phase indicator field and convert to regular layout - DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), Phi, sizeof(double)*N); - - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,OUTFILE); - fclose(OUTFILE); - - ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - FILE *AFILE; - sprintf(LocalRankFilename,"A.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - FILE *BFILE; - sprintf(LocalRankFilename,"B.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); - - ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField); - FILE *PFILE; - sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); - PFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PFILE); - fclose(PFILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); - FILE *VELX_FILE; - sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); - VELX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELX_FILE); - fclose(VELX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); - FILE *VELY_FILE; - sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); - VELY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELY_FILE); - fclose(VELY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); - FILE *VELZ_FILE; - sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); - VELZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELZ_FILE); - fclose(VELZ_FILE); - -/* ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); - FILE *CGX_FILE; - sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); - CGX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGX_FILE); - fclose(CGX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); - FILE *CGY_FILE; - sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); - CGY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGY_FILE); - fclose(CGY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); - FILE *CGZ_FILE; - sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); - CGZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,CGZ_FILE); - fclose(CGZ_FILE); -*/ -} diff --git a/models/ElectroKinetic.h b/models/ElectroKinetic.h deleted file mode 100644 index a3b3a124..00000000 --- a/models/ElectroKinetic.h +++ /dev/null @@ -1,88 +0,0 @@ -/* -Implementation of color lattice boltzmann model - */ -#include -#include -#include -#include -#include -#include -#include - -#include "common/Communication.h" -#include "analysis/TwoPhase.h" -#include "analysis/runAnalysis.h" -#include "common/MPI_Helpers.h" -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" - -class ScaLBL_ColorModel{ -public: - ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_ColorModel(); - - // functions in they should be run - void ReadParams(string filename); - void ReadParams(std::shared_ptr db0); - void SetDomain(); - void ReadInput(); - void Create(); - void Initialize(); - void Run(); - void WriteDebug(); - - bool Restart,pBC; - bool REVERSE_FLOW_DIRECTION; - int timestep,timestepMax; - int BoundaryCondition; - double tauA,tauB,rhoA,rhoB,alpha,beta; - double Fx,Fy,Fz,flux; - double din,dout,inletA,inletB,outletA,outletB; - - int Nx,Ny,Nz,N,Np; - int rank,nprocx,nprocy,nprocz,nprocs; - double Lx,Ly,Lz; - - std::shared_ptr Dm; // this domain is for analysis - std::shared_ptr Mask; // this domain is for lbm - std::shared_ptr ScaLBL_Comm; - std::shared_ptr ScaLBL_Comm_Regular; - //std::shared_ptr Averages; - std::shared_ptr Averages; - - // input database - std::shared_ptr db; - std::shared_ptr domain_db; - std::shared_ptr color_db; - std::shared_ptr analysis_db; - std::shared_ptr vis_db; - - IntArray Map; - signed char *id; - int *NeighborList; - int *dvcMap; - double *fq, *Aq, *Bq; - double *Den, *Phi; - double *ColorGrad; - double *Velocity; - double *Pressure; - -private: - MPI_Comm comm; - - int dist_mem_size; - int neighborSize; - // filenames - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - - //int rank,nprocs; - void LoadParams(std::shared_ptr db0); - void AssignComponentLabels(double *phase); - double ImageInit(std::string filename); - double MorphInit(const double beta, const double morph_delta); - double SeedPhaseField(const double seed_water_in_oil); - double MorphOpenConnected(double target_volume_change); -}; - From fcdb84b2ad735fd51ad6b71901e5b5b774c910b5 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 6 Aug 2020 16:06:52 -0400 Subject: [PATCH 199/270] electro skeleton compiles --- common/ScaLBL.cpp | 1 - common/ScaLBL.h | 9 ++-- models/IonModel.cpp | 25 ++++------- models/IonModel.h | 2 +- models/PoissonSolver.cpp | 49 +++++---------------- tests/CMakeLists.txt | 1 + tests/lbpm_electrokinetic_dfh_simulator.cpp | 2 +- 7 files changed, 25 insertions(+), 64 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index eace3f3f..6ae49cca 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1211,7 +1211,6 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){ //...Packing for Y face(3,7,10,15,17)................................. ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*N],N); - ScaLBL_D3Q19_Pack(3,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 23cf6936..e029ddda 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -78,10 +78,10 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Velocity, int start extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Velocity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); -// ION TRANSPORT MODEL -extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); +// LBM Poisson solver +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, double *ChargeDensity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); -extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *ChargeDensity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz); // MRT MODEL @@ -92,7 +92,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int st double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); // COLOR MODEL - extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); @@ -197,7 +196,7 @@ public: void SendD3Q19AA(double *dist); void RecvD3Q19AA(double *dist); void SendD3Q7AA(double *fq, int Component); - void RecvD3Q7AA(double *fq, int Component) + void RecvD3Q7AA(double *fq, int Component); void BiSendD3Q7AA(double *Aq, double *Bq); void BiRecvD3Q7AA(double *Aq, double *Bq); void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq); diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 38088587..92dcd652 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -1,8 +1,7 @@ /* * Multi-relaxation time LBM Model */ -#include "models/MRT.h" -#include "models/ElectroModel.h" +#include "models/IonModel.h" #include "analysis/distance.h" #include "common/ReadMicroCT.h" @@ -26,15 +25,10 @@ void ScaLBL_IonModel::ReadParams(string filename){ tau = 1.0; timestepMax = 100000; tolerance = 1.0e-8; - Fx = Fy = 0.0; - Fz = 1.0e-5; - - // Color Model parameters + // Model parameters if (ion_db->keyExists( "timestepMax" )){ - timestepMax = mrt_db->getScalar( "timestepMax" ); + timestepMax = ion_db->getScalar( "timestepMax" ); } - - mu=(tau-0.5)/3.0; } void ScaLBL_IonModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis @@ -50,9 +44,6 @@ void ScaLBL_IonModel::SetDomain(){ N = Nx*Ny*Nz; Distance.resize(Nx,Ny,Nz); - Velocity_x.resize(Nx,Ny,Nz); - Velocity_y.resize(Nx,Ny,Nz); - Velocity_z.resize(Nx,Ny,Nz); for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object @@ -176,7 +167,7 @@ void ScaLBL_IonModel::Initialize(){ } void ScaLBL_IonModel::Run(double *Velocity){ - + double rlx = 1.0/tau; //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -190,19 +181,19 @@ void ScaLBL_IonModel::Run(double *Velocity){ //************************************************************************/ timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Ion(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAeven_Ion(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } diff --git a/models/IonModel.h b/models/IonModel.h index f6ffad5e..0b1d7e38 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -1,5 +1,5 @@ /* - * Multi-relaxation time LBM Model + * Ion transporte LB Model */ #include #include diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index f4f15224..776134cc 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -1,8 +1,7 @@ /* * Multi-relaxation time LBM Model */ -#include "models/MRT.h" -#include "models/ElectroModel.h" +#include "models/PoissonSolver.h" #include "analysis/distance.h" #include "common/ReadMicroCT.h" @@ -21,8 +20,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); - mrt_db = db->getDatabase( "MRT" ); - electric_db = db->getDatabase( "Electrochemistry" ); + electric_db = db->getDatabase( "Electric" ); tau = 1.0; timestepMax = 100000; @@ -31,38 +29,14 @@ void ScaLBL_Poisson::ReadParams(string filename){ Fz = 1.0e-5; // Color Model parameters - if (mrt_db->keyExists( "timestepMax" )){ - timestepMax = mrt_db->getScalar( "timestepMax" ); + if (electric_db->keyExists( "timestepMax" )){ + timestepMax = electric_db->getScalar( "timestepMax" ); } - if (mrt_db->keyExists( "tolerance" )){ - tolerance = mrt_db->getScalar( "tolerance" ); - } - if (mrt_db->keyExists( "tau" )){ - tau = mrt_db->getScalar( "tau" ); - } - if (mrt_db->keyExists( "F" )){ - Fx = mrt_db->getVector( "F" )[0]; - Fy = mrt_db->getVector( "F" )[1]; - Fz = mrt_db->getVector( "F" )[2]; - } - if (mrt_db->keyExists( "Restart" )){ - Restart = mrt_db->getScalar( "Restart" ); - } - if (mrt_db->keyExists( "din" )){ - din = mrt_db->getScalar( "din" ); - } - if (mrt_db->keyExists( "dout" )){ - dout = mrt_db->getScalar( "dout" ); - } - if (mrt_db->keyExists( "flux" )){ - flux = mrt_db->getScalar( "flux" ); - } // Read domain parameters if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } - mu=(tau-0.5)/3.0; } @@ -80,10 +54,7 @@ void ScaLBL_Poisson::SetDomain(){ N = Nx*Ny*Nz; Distance.resize(Nx,Ny,Nz); - Velocity_x.resize(Nx,Ny,Nz); - Velocity_y.resize(Nx,Ny,Nz); - Velocity_z.resize(Nx,Ny,Nz); - + for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); @@ -205,7 +176,7 @@ void ScaLBL_Poisson::Initialize(){ } void ScaLBL_Poisson::Run(double *ChargeDensity){ - + double rlx = 1.0/tau; //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -219,19 +190,19 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ //************************************************************************/ timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAeven_Poisson(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e703c9d..8c937107 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,6 +4,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_dfh_simulator.cpp ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index e3765d12..5209565c 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -61,7 +61,7 @@ int main(int argc, char **argv) DFHModel.Run(); // Solve the N-S equations to get velocity IonModel.Run(DFHModel.Velocity); //solve for ion transport and electric potential - IonModel.Run(DFHModel.Velocity, DFHModel.Phi); //solve for ion transport and electric potential with multiphase system + //IonModel.Run(DFHModel.Velocity, DFHModel.Phi); //solve for ion transport and electric potential with multiphase system PoissonSolver.Run(IonModel.ChargeDensity); DFHModel.WriteDebug(); From d9cde3c76cb1fd206a7c2002f323ec7de4e480f6 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 6 Aug 2020 16:12:18 -0400 Subject: [PATCH 200/270] use generalized D3Q7 MPI structures for multi-ion --- models/IonModel.cpp | 20 +++++++++++++------- models/PoissonSolver.cpp | 8 ++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 92dcd652..687deb9e 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -25,9 +25,11 @@ void ScaLBL_IonModel::ReadParams(string filename){ tau = 1.0; timestepMax = 100000; tolerance = 1.0e-8; - // Model parameters - if (ion_db->keyExists( "timestepMax" )){ - timestepMax = ion_db->getScalar( "timestepMax" ); + // Model parameters + + number_ion_species = 1; + if (ion_db->keyExists( "number_ion_species" )){ + number_ion_species = ion_db->getScalar( "number_ion_species" ); } } void ScaLBL_IonModel::SetDomain(){ @@ -180,17 +182,21 @@ void ScaLBL_IonModel::Run(double *Velocity){ while (timestep < timestepMax && error > tolerance) { //************************************************************************/ timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + for (int ic=0; icSendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 776134cc..9668dd0b 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -189,17 +189,17 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ while (timestep < timestepMax && error > tolerance) { //************************************************************************/ timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); From dff4e3d5362914e36ec2ec38f6de8826ed55c1eb Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 7 Aug 2020 17:44:02 -0400 Subject: [PATCH 201/270] save the work;to be continued --- common/ScaLBL.cpp | 6 +- cpu/Poisson.cpp | 180 +++++++++++--------- models/PoissonSolver.cpp | 110 +++++++++--- models/PoissonSolver.h | 12 +- tests/lbpm_electrokinetic_dfh_simulator.cpp | 2 +- 5 files changed, 200 insertions(+), 110 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6ae49cca..d9b75c43 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1182,7 +1182,7 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); } else{ Lock=true; @@ -1288,7 +1288,7 @@ void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + ERROR("ScaLBL Error (BiSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); } else{ Lock=true; @@ -1415,7 +1415,7 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); + ERROR("ScaLBL Error (TriSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); } else{ Lock=true; diff --git a/cpu/Poisson.cpp b/cpu/Poisson.cpp index 355e4223..d24bc12f 100644 --- a/cpu/Poisson.cpp +++ b/cpu/Poisson.cpp @@ -1,65 +1,18 @@ -extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ + +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *Den_charge, double *Psi, double *ElectricField, double rlx, double epsilon_LB,double deltaT, + int start, int finish, int Np){ int n; - // conserved momemnts - double rho,ux,uy,uz,uu; - // non-conserved moments - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; + double psi;//electric potential + double Ex,Ey,Ez;//electrical field + double rho_e;//local charge density + double f0,f1,f2,f3,f4,f5,f6; + int nr1,nr2,nr3,nr4,nr5,nr6; for (int n=start; n( filename ); domain_db = db->getDatabase( "Domain" ); - electric_db = db->getDatabase( "Electric" ); + electric_db = db->getDatabase( "Poisson" ); - tau = 1.0; + k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice + deltaT = 0.3;//time step of LB-Poisson equation + tau = 0.5+k2_inv*deltaT; timestepMax = 100000; - tolerance = 1.0e-8; - Fx = Fy = 0.0; - Fz = 1.0e-5; + tolerance = 1.0e-6;//stopping criterion for obtaining steady-state electricla potential + h = 1.0;//resolution; unit: um/lu + epsilon0 = 8.85e-12;//electrical permittivity of vaccum; unit:[C/(V*m)] + epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] + epsilonR = 78.4;//default dielectric constant for water + epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity + analysis_interval = 1000; - // Color Model parameters + // LB-Poisson Model parameters if (electric_db->keyExists( "timestepMax" )){ timestepMax = electric_db->getScalar( "timestepMax" ); } - + if (electric_db->keyExists( "analysis_interval" )){ + analysis_interval = electric_db->getScalar( "analysis_interval" ); + } + if (electric_db->keyExists( "tolerance" )){ + tolerance = electric_db->getScalar( "tolerance" ); + } + if (electric_db->keyExists( "deltaT" )){ + deltaT = electric_db->getScalar( "deltaT" ); + } + if (electric_db->keyExists( "epsilonR" )){ + epsilonR = electric_db->getScalar( "epsilonR" ); + } // Read domain parameters + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } + //Re-calcualte model parameters if user updates input + epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] + epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity + tau = 0.5+k2_inv*deltaT; - mu=(tau-0.5)/3.0; } void ScaLBL_Poisson::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis @@ -54,6 +77,7 @@ void ScaLBL_Poisson::SetDomain(){ N = Nx*Ny*Nz; Distance.resize(Nx,Ny,Nz); + Psi_host.resize(Nx,Ny,Nz); for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object @@ -158,6 +182,7 @@ void ScaLBL_Poisson::Create(){ ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &fq, 7*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &ElectricField, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); @@ -171,54 +196,95 @@ void ScaLBL_Poisson::Initialize(){ /* * This function initializes model */ - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf ("Initializing D3Q7 distributions for LB-Poisson solver\n"); + ScaLBL_D3Q7_Poisson_Init(fq, Np); } void ScaLBL_Poisson::Run(double *ChargeDensity){ - double rlx = 1.0/tau; + + //LB-related parameter + double rlx = 1.0/tau; + //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); - if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); - if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("***************************************************************************\n"); + if (rank==0) printf("LB-Poisson Solver: timestepMax = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance); + if (rank==0) printf("***************************************************************************\n"); timestep=0; double error = 1.0; - double flow_rate_previous = 0.0; + double psi_avg_previous = 0.0; while (timestep < timestepMax && error > tolerance) { //************************************************************************/ - timestep++; + // *************ODD TIMESTEP*************// + timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ + + // Check convergence of steady-state solution + if (timestep%analysis_interval==0){ + + ScaLBL_Comm->RegularLayout(Map,&Psi,Psi_host); + double count_loc=0; + double count; + double psi_avg; + double psi_loc=0.f; + + for (int k=1; k 0){ + psi_loc += Psi_host(i,j,k); + count_loc+=1.0; + } + } + } + } + MPI_Allreduce(&psi_loc,&psi_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + psi_avg /= count; + double psi_avg_mag=psi_avg; + if (psi_avg==0.0) psi_avg_mag=1.0; + error = fabs(psi_avg-psi_avg_previous)/fabs(psi_avg_mag); + psi_avg_previous = psi_avg; + } } //************************************************************************/ stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); + if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n"); + if (rank==0) printf("---------------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; // Performance obtained from each node double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("******************* LB-Poisson Solver Statistics ********************\n"); if (rank==0) printf("CPU time = %f \n", cputime); if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); MLUPS *= nprocs; if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("*********************************************************************\n"); } + +//void ScaLBL_Poisson::get_ElectricField(){ +//// ??? +//} diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 63e10df0..7eb7ac83 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -29,17 +29,19 @@ public: void Initialize(); void Run(double *ChargeDensity); - bool Restart,pBC; + //bool Restart,pBC; int timestep,timestepMax; + int analysis_interval; int BoundaryCondition; - double tau,mu; - double Fx,Fy,Fz,flux; - double din,dout; + double tau; double tolerance; + double k2_inv,deltaT; + double epsilon0,epsilon0_LB,epsilonR,epsilon_LB; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; double Lx,Ly,Lz; + double h;//image resolution std::shared_ptr Dm; // this domain is for analysis std::shared_ptr Mask; // this domain is for lbm @@ -51,9 +53,11 @@ public: IntArray Map; DoubleArray Distance; + DoubleArray Psi_host; int *NeighborList; double *fq; double *Psi; + double *ElectricField; private: MPI_Comm comm; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 5209565c..9725dda4 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -38,7 +38,7 @@ int main(int argc, char **argv) if (rank == 0){ printf("********************************************************\n"); - printf("Running Color LBM \n"); + printf("Running Electrokinetic LBM Simulator \n"); printf("********************************************************\n"); } PROFILE_ENABLE(1); From 28988ef6ba15ed6f4f866bfd9bd583ebda36a930 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 10 Aug 2020 12:03:28 -0400 Subject: [PATCH 202/270] save the work;untested --- cpu/Ion.cpp | 316 ++++++++++++++------ cpu/Poisson.cpp | 4 +- models/IonModel.cpp | 186 ++++++++++-- models/IonModel.h | 10 +- tests/lbpm_electrokinetic_dfh_simulator.cpp | 6 +- 5 files changed, 395 insertions(+), 127 deletions(-) diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index 569c9a0a..d6874a76 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -1,12 +1,177 @@ -extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Velocity, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){ - int n; - // conserved momemnts - double rho,ux,uy,uz,uu; - // non-conserved moments - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; - for (int n=start; n 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + // q=4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + // q=6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(ux+uEPx)); + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(ux+uEPx)); + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uy+uEPy)); + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uy+uEPy)); + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uz+uEPz)); + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uz+uEPz)); + + + } +} + +extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, + double Di, double zi, double rlx, double deltaT, double Vt, int start, int finish, int Np){ + int n; + double Ci; + double ux,uy,uz; + double uEPx,uEPy,uEPz;//electrochemical induced velocity + double Ex,Ey,Ez;//electrical field + double f0,f1,f2,f3,f4,f5,f6; + + for (n=start; n 10Np => odd part of dist) - f1 = dist[nr1]; // reading the f1 data into register fq - - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - f2 = dist[nr2]; // reading the f2 data into register fq - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - f3 = dist[nr3]; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - f4 = dist[nr4]; - - // q=5 - nr5 = neighborList[n+4*Np]; - f5 = dist[nr5]; - - // q = 6 - nr6 = neighborList[n+5*Np]; - f6 = dist[nr6]; - - rho = f0+f2+f1+f4+f3+f6; - ux = Velocity[n]; - uy = Velocity[n+Np]; - uz = Velocity[n+2*Np]; - uu = 1.5*(ux*ux+uy*uy+uz*uz); - - // q=0 - dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu); + dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; // q = 1 - dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx; + dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(ux+uEPx)); // q=2 - dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx; + dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(ux+uEPx)); // q = 3 - dist[nr4] = f3*(1.0-rlx) + - rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy; + dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uy+uEPy)); // q = 4 - dist[nr3] = f4*(1.0-rlx) + - rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy; + dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uy+uEPy)); // q = 5 - dist[nr6] = f5*(1.0-rlx) + - rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz; + dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uz+uEPz)); // q = 6 - dist[nr5] = f6*(1.0-rlx) + - rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz; + dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uz+uEPz)); } -} \ No newline at end of file +} + +extern "C" void ScaLBL_D3Q7_Poisson_Init(double *dist, double *Den, double DenInit, int Np) +{ + int n; + for (n=0; n& IonValence, int number_ion_species, int start, int finish, int Np){ + + int n; + int ic=number_ion_species; + double Ci;//ion concentration of species i + double CD;//charge density + double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant + for (n=start; n0){ + for (n=start; n( filename ); domain_db = db->getDatabase( "Domain" ); ion_db = db->getDatabase( "Ions" ); - tau = 1.0; + // Default model parameters + T = 300.0;//temperature; unit [K] + Vt = kb*T/electron_charge;//thermal voltage; unit [V] + k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice + h = 1.0;//resolution; unit: um/lu timestepMax = 100000; tolerance = 1.0e-8; - // Model parameters - number_ion_species = 1; + IonDiffusivity.push_back(1.0e-9);//User input unit [m^2/sec] + //TODO needs to scale the unit of diffusivity! + IonValence.push_back(1); + IonConcentration.push_back(1.0e-3);//unit [mol/m^3] + // TODO rescale ion concentration unit + deltaT.push_back(1.0); + tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivisty[0]); + + // LB-Ion Model parameters + if (ion_db->keyExists( "timestepMax" )){ + timestepMax = ion_db->getScalar( "timestepMax" ); + } + if (ion_db->keyExists( "analysis_interval" )){ + analysis_interval = ion_db->getScalar( "analysis_interval" ); + } + if (ion_db->keyExists( "tolerance" )){ + tolerance = ion_db->getScalar( "tolerance" ); + } + if (ion_db->keyExists( "temperature" )){ + T = ion_db->getScalar( "temperature" ); + } + if (ion_db->keyExists( "epsilonR" )){ + epsilonR = ion_db->getScalar( "epsilonR" ); + } if (ion_db->keyExists( "number_ion_species" )){ number_ion_species = ion_db->getScalar( "number_ion_species" ); } + + //read ion related list + if (ion_db->keyExists( "deltaT" )){ + deltaT.clear(); + deltaT = ion_db->getVector( "deltaT" ); + if (deltaT.size()!=number_ion_species){ + ERROR("Error: number_ion_species and deltaT must be the same length! \n"); + } + } + //NOTE: Ion diffusivity has unit: [m^2/sec] + if (ion_db->keyExists("IonDiffusivityList")){ + IonDiffusivity.clear(); + IonDiffusivity = ion_db->getVector( "IonDiffusivityList" ); + if (IonDiffusivity.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonDiffusivityList must be the same length! \n"); + } + } + //read ion algebric valence list + if (ion_db->keyExists("IonValenceList")){ + IonValence.clear(); + IonValence = ion_db->getVector( "IonValenceList" ); + if (IonValence.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonValenceList must be the same length! \n"); + } + } + //read initial ion concentration list; unit [mol/m^3] + if (ion_db->keyExists("IonConcentrationList")){ + IonConcentration.clear(); + IonConcentration = ion_db->getVector( "IonConcentrationList" ); + if (IonConcentration.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonConcentrationList must be the same length! \n"); + } + } + + // Read domain parameters + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + //Re-calcualte model parameters if user updates input + //TODO ion diffusivity needs rescale unit to LB unit + //TODO rescale ion initial concentration unit to LB unit + if (deltaT.size()>1){ + tau.clear(); + for (int i=0;i(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases @@ -164,42 +246,104 @@ void ScaLBL_IonModel::Initialize(){ /* * This function initializes model */ - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf ("Initializing D3Q7 distributions for ion transport\n"); + for (int ic=0; ic rlx(tau.begin(),tau.end()); + for (double item : rlx){ + item = 1.0/item; + } //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); - if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax); - if (rank==0) printf("********************************************************\n"); + + if (rank==0) printf("***************************************************\n"); + if (rank==0) printf("LB-Ion Transport: timestepMax = %i\n", timestepMax); + if (rank==0) printf("***************************************************\n"); timestep=0; - double error = 1.0; - double flow_rate_previous = 0.0; - while (timestep < timestepMax && error > tolerance) { + while (timestep < timestepMax) { //************************************************************************/ + // *************ODD TIMESTEP*************// timestep++; - for (int ic=0; icSendD3Q7AA(fq, ic); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + } + for (int ic=0; icLastExterior(), Np); + } + ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); + + //LB-Ion collison + for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAodd_Ion(NeighborList, fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + + for (int ic=0; icLastExterior(), Np); + } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP*************// timestep++; - for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, Fx, Fy, Fz); - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + + for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + } + for (int ic=0; icLastExterior(), Np); + } + ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); + + //LB-Ion collison + for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAeven_Ion(fq, Velocity, 0, ScaLBL_Comm->LastExterior(), Np, rlx, Fx, Fy, Fz); + + for (int ic=0; icLastExterior(), Np); + } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } diff --git a/models/IonModel.h b/models/IonModel.h index 0b1d7e38..8be11468 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -8,6 +8,7 @@ #include #include #include +#include #include "common/ScaLBL.h" #include "common/Communication.h" @@ -33,12 +34,13 @@ public: bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; - double tau,mu; - double Fx,Fy,Fz,flux; - double din,dout; - double tolerance; int number_ion_species; + vector IonDiffusivity;//User input unit [m^2/sec] + vector IonValence; + vector IonConcentration;//unit [mol/m^3] + vector deltaT; + vector tau; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 9725dda4..0c9da812 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -59,10 +59,10 @@ int main(int argc, char **argv) DFHModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables DFHModel.Initialize(); // initializing the model will set initial conditions for variables - DFHModel.Run(); // Solve the N-S equations to get velocity - IonModel.Run(DFHModel.Velocity); //solve for ion transport and electric potential - //IonModel.Run(DFHModel.Velocity, DFHModel.Phi); //solve for ion transport and electric potential with multiphase system + DFHModel.Run(IonModel.ChargeDensity); // Solve the N-S equations to get velocity PoissonSolver.Run(IonModel.ChargeDensity); + IonModel.Run(DFHModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential + //IonModel.Run(DFHModel.Velocity, DFHModel.Phi,PoissonSolver.ElectricField); DFHModel.WriteDebug(); From b4d50ee821d58c1262a83c43aceed791ce809b33 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 10 Aug 2020 12:54:55 -0400 Subject: [PATCH 203/270] example for how to structure ion comm loop --- models/IonModel.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 46859eab..237ebcf1 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -275,17 +275,13 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ //Update ion concentration and charge density for (int ic=0; icSendD3Q7AA(fq, ic); //READ FROM NORMAL - } - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + /* Update exterior ion concentration */ } + ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE - } for (int ic=0; ic Date: Tue, 11 Aug 2020 14:35:46 -0400 Subject: [PATCH 204/270] distinguish parallel comm / bounceback sites at halo --- common/ScaLBL.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index d9b75c43..928e3d5e 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -372,7 +372,11 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis for (k=0;k 0) + Map(i,j,k) = -2; // this label is for parallel communication sites + else + Map(i,j,k) = -1; // this label is for solid bounce-back sites } } } From e0b0e0566495f27bb1cd185f4978b67ff70dde5e Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 11 Aug 2020 15:16:40 -0400 Subject: [PATCH 205/270] added bounce-back interaction capability --- common/ScaLBL.cpp | 215 +++++++++++++++++++++++++++++++++++++++++++++- common/ScaLBL.h | 6 ++ 2 files changed, 220 insertions(+), 1 deletion(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 928e3d5e..dbb75380 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -526,7 +526,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis } } } - + //for (idx=0; idx Date: Tue, 11 Aug 2020 15:34:12 -0400 Subject: [PATCH 206/270] fixed bugs in bounceback list --- common/ScaLBL.cpp | 13 +++++++------ common/ScaLBL.h | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index dbb75380..18238a8f 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -854,9 +854,11 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis } -void ScaLBL_Communicator::SetupBounceBackList() +void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, int Np) { - + + int idx,i,j,k; + int neighbor; // save list of bounce-back distributions and interaction sites n_bb_d3q7 = 0; n_bb_d3q19 = 0; @@ -866,9 +868,8 @@ void ScaLBL_Communicator::SetupBounceBackList() for (i=1;i Date: Tue, 11 Aug 2020 16:52:56 -0400 Subject: [PATCH 207/270] add zeta potential to Poisson --- common/ScaLBL.cpp | 10 ++++++++++ common/ScaLBL.h | 4 ++-- models/PoissonSolver.cpp | 11 +++++++++++ models/PoissonSolver.h | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 18238a8f..28459298 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1067,6 +1067,16 @@ void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, in ScaLBL_CopyToDevice(bb_dist, bb_dist_tmp, local_count*sizeof(int)); } +void ScaLBL_Communicator::SolidDirichletD3Q7(double *fq, double *assignValues){ + // fq is a D3Q7 distribution + // assignValues is a list of values to assign at bounce-back sites + for (int idx=0; idxn_bb_d3q7); //........................................................................... + // initialize the zeta function (example is zeta is constant on solid surface) + double *tmpZeta = new double[ScaLBL_Comm->n_bb_d3q7]; + for int (i=0; in_bb_d3q7; i++){ + tmpZeta[i] = 1.0/k2_inv; // this has to be read from input file + } + ScaLBL_CopyToDevice(zeta, tmpZeta, sizeof(double)*ScaLBL_Comm->n_bb_d3q7); + delete [] tmpZeta; + // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); // copy the neighbor list @@ -225,6 +234,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->SolidDirichletD3Q7(fq, zeta); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// @@ -235,6 +245,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->SolidDirichletD3Q7(fq, zeta); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 7eb7ac83..625c602f 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -58,6 +58,7 @@ public: double *fq; double *Psi; double *ElectricField; + double *zeta; private: MPI_Comm comm; From 3a162849a03db896cbaa76d1b34117e247870496 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 14 Aug 2020 14:23:22 -0400 Subject: [PATCH 208/270] save the work;untested --- cpu/Ion.cpp | 33 +- cpu/Poisson.cpp | 22 +- cpu/Stokes.cpp | 957 ++++++++++++++++++++ models/IonModel.cpp | 190 ++-- models/IonModel.h | 12 +- models/MultiPhysController.cpp | 57 ++ models/MultiPhysController.h | 47 + models/PoissonSolver.cpp | 81 +- models/StokesModel.cpp | 543 +++++++++++ models/StokesModel.h | 75 ++ tests/lbpm_electrokinetic_dfh_simulator.cpp | 137 +-- 11 files changed, 1932 insertions(+), 222 deletions(-) create mode 100644 cpu/Stokes.cpp create mode 100644 models/MultiPhysController.cpp create mode 100644 models/MultiPhysController.h create mode 100644 models/StokesModel.cpp create mode 100644 models/StokesModel.h diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index d6874a76..4f760e4f 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -80,7 +80,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, i } extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, - double Di, double zi, double rlx, double deltaT, double Vt, int start, int finish, int Np){ + double Di, double zi, double rlx, double Vt, int start, int finish, int Np){ int n; double Ci; double ux,uy,uz; @@ -128,29 +128,28 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *D dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; // q = 1 - dist[nr2] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(ux+uEPx)); + dist[nr2] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(ux+uEPx)); // q=2 - dist[nr1] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(ux+uEPx)); + dist[nr1] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(ux+uEPx)); // q = 3 - dist[nr4] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uy+uEPy)); + dist[nr4] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uy+uEPy)); // q = 4 - dist[nr3] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uy+uEPy)); + dist[nr3] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uy+uEPy)); // q = 5 - dist[nr6] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uz+uEPz)); + dist[nr6] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uz+uEPz)); // q = 6 - dist[nr5] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uz+uEPz)); - + dist[nr5] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uz+uEPz)); } } extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, - double Di, double zi, double rlx, double deltaT, double Vt, int start, int finish, int Np){ + double Di, double zi, double rlx, double Vt, int start, int finish, int Np){ int n; double Ci; double ux,uy,uz; @@ -184,28 +183,28 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Veloci dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; // q = 1 - dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(ux+uEPx)); + dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(ux+uEPx)); // q=2 - dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(ux+uEPx)); + dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(ux+uEPx)); // q = 3 - dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uy+uEPy)); + dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uy+uEPy)); // q = 4 - dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uy+uEPy)); + dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uy+uEPy)); // q = 5 - dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*deltaT*(uz+uEPz)); + dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uz+uEPz)); // q = 6 - dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*deltaT*(uz+uEPz)); + dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uz+uEPz)); } } -extern "C" void ScaLBL_D3Q7_Poisson_Init(double *dist, double *Den, double DenInit, int Np) +extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np) { int n; for (n=0; n& IonValence, int number_ion_species, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, vector& IonValence, int number_ion_species, int start, int finish, int Np){ int n; int ic=number_ion_species; diff --git a/cpu/Poisson.cpp b/cpu/Poisson.cpp index 7eac7401..d33ab658 100644 --- a/cpu/Poisson.cpp +++ b/cpu/Poisson.cpp @@ -1,5 +1,5 @@ -extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *Den_charge, double *Psi, double *ElectricField, double rlx, double epsilon_LB,double deltaT, +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, int start, int finish, int Np){ int n; double psi;//electric potential @@ -7,12 +7,13 @@ extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, doubl double rho_e;//local charge density double f0,f1,f2,f3,f4,f5,f6; int nr1,nr2,nr3,nr4,nr5,nr6; + double rlx=1.0/tau; for (n=start; n + +extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, + double Gx, double Gy, double Gz, int start, int finish, int Np) +{ + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double ux,uy,uz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + // body force due to electric field + double rhoE;//charge density + double Ex,Ey,Ez; + // total body force + double Fx,Fy,Fz; + + constexpr double mrt_V1=0.05263157894736842; + constexpr double mrt_V2=0.012531328320802; + constexpr double mrt_V3=0.04761904761904762; + constexpr double mrt_V4=0.004594820384294068; + constexpr double mrt_V5=0.01587301587301587; + constexpr double mrt_V6=0.0555555555555555555555555; + constexpr double mrt_V7=0.02777777777777778; + constexpr double mrt_V8=0.08333333333333333; + constexpr double mrt_V9=0.003341687552213868; + constexpr double mrt_V10=0.003968253968253968; + constexpr double mrt_V11=0.01388888888888889; + constexpr double mrt_V12=0.04166666666666666; + + for (int n=start; n 10Np => odd part of dist) + fq = dist[nread]; // reading the f1 data into register fq + //fp = dist[10*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // f2 = dist[10*Np+n]; + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nread]; // reading the f2 data into register fq + //fq = dist[Np+n]; + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nread]; + //fq = dist[11*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nread]; + //fq = dist[2*Np+n]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = dist[nread]; + //fq = dist[12*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + nread = neighborList[n+5*Np]; + fq = dist[nread]; + //fq = dist[3*Np+n]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + //fq = dist[13*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + //fq = dist[4*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + //fq = dist[14*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + //fq = dist[5*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + //fq = dist[15*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + //fq = dist[6*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + //fq = dist[16*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + //fq = dist[7*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + + // write the velocity + ux = jx / rho; + uy = jy / rho; + uz = jz / rho; + Velocity[n] = ux; + Velocity[Np+n] = uy; + Velocity[2*Np+n] = uz; + + //..............incorporate external force................................................ + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); + m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); + m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); + m13 = m13 + rlx_setA*((jx*jy/rho) - m13); + m14 = m14 + rlx_setA*((jy*jz/rho) - m14); + m15 = m15 + rlx_setA*((jx*jz/rho) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + //.................inverse transformation...................................................... + + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + nread = neighborList[n+Np]; + dist[nread] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + nread = neighborList[n]; + dist[nread] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + nread = neighborList[n+3*Np]; + dist[nread] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + nread = neighborList[n+2*Np]; + dist[nread] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + nread = neighborList[n+5*Np]; + dist[nread] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + nread = neighborList[n+4*Np]; + dist[nread] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + + } +} + diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 237ebcf1..2e5b4e71 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -16,68 +16,70 @@ ScaLBL_IonModel::~ScaLBL_IonModel(){ } -void ScaLBL_IonModel::ReadParams(string filename){ - - //fundamental constant - kb = 1.38e-23;//Boltzmann constant;unit [J/K] - electron_charge = 1.6e-19;//electron charge;unit [C] +void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stokes,double time_conv_Stokes){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); ion_db = db->getDatabase( "Ions" ); - // Default model parameters + //------ Load number of iteration from multiphysics controller ------// + timestepMax = num_iter; + //compute time conversion factor for ion model + time_conv = num_iter_Stokes*time_conv_Stokes/num_iter; + //-------------------------------------------------------------------// + + // Universal constant + kb = 1.38e-23;//Boltzmann constant;unit [J/K] + electron_charge = 1.6e-19;//electron charge;unit [C] + + //---------------------- Default model parameters --------------------------// T = 300.0;//temperature; unit [K] - Vt = kb*T/electron_charge;//thermal voltage; unit [V] + Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice h = 1.0;//resolution; unit: um/lu - timestepMax = 100000; tolerance = 1.0e-8; number_ion_species = 1; - IonDiffusivity.push_back(1.0e-9);//User input unit [m^2/sec] - //TODO needs to scale the unit of diffusivity! - IonValence.push_back(1); - IonConcentration.push_back(1.0e-3);//unit [mol/m^3] - // TODO rescale ion concentration unit - deltaT.push_back(1.0); - tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivisty[0]); + IonDiffusivity.push_back(1.0e-9);//user-input diffusivity has physical unit [m^2/sec] + IonValence.push_back(1);//algebraic valence charge + IonConcentration.push_back(1.0e-3);//user-input ion concentration has physical unit [mol/m^3] + //deltaT.push_back(1.0); + //tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivisty[0]); + tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivisty[0]); + //--------------------------------------------------------------------------// // LB-Ion Model parameters - if (ion_db->keyExists( "timestepMax" )){ - timestepMax = ion_db->getScalar( "timestepMax" ); - } - if (ion_db->keyExists( "analysis_interval" )){ - analysis_interval = ion_db->getScalar( "analysis_interval" ); - } + //if (ion_db->keyExists( "timestepMax" )){ + // timestepMax = ion_db->getScalar( "timestepMax" ); + //} if (ion_db->keyExists( "tolerance" )){ tolerance = ion_db->getScalar( "tolerance" ); } if (ion_db->keyExists( "temperature" )){ T = ion_db->getScalar( "temperature" ); } - if (ion_db->keyExists( "epsilonR" )){ - epsilonR = ion_db->getScalar( "epsilonR" ); - } if (ion_db->keyExists( "number_ion_species" )){ number_ion_species = ion_db->getScalar( "number_ion_species" ); } - //read ion related list - if (ion_db->keyExists( "deltaT" )){ - deltaT.clear(); - deltaT = ion_db->getVector( "deltaT" ); - if (deltaT.size()!=number_ion_species){ - ERROR("Error: number_ion_species and deltaT must be the same length! \n"); - } - } - //NOTE: Ion diffusivity has unit: [m^2/sec] + //NOTE: ion diffusivity has INPUT unit: [m^2/sec] + // it must be converted to LB unit: [lu^2/lt] if (ion_db->keyExists("IonDiffusivityList")){ IonDiffusivity.clear(); IonDiffusivity = ion_db->getVector( "IonDiffusivityList" ); + // time relaxation parameters tau also needs update + tau.clear(); if (IonDiffusivity.size()!=number_ion_species){ ERROR("Error: number_ion_species and IonDiffusivityList must be the same length! \n"); } + else{ + for (int i=0; ikeyExists("IonValenceList")){ @@ -87,14 +89,34 @@ void ScaLBL_IonModel::ReadParams(string filename){ ERROR("Error: number_ion_species and IonValenceList must be the same length! \n"); } } - //read initial ion concentration list; unit [mol/m^3] + //read initial ion concentration list; INPUT unit [mol/m^3] + //it must be converted to LB unit [mol/lu^3] if (ion_db->keyExists("IonConcentrationList")){ IonConcentration.clear(); IonConcentration = ion_db->getVector( "IonConcentrationList" ); if (IonConcentration.size()!=number_ion_species){ ERROR("Error: number_ion_species and IonConcentrationList must be the same length! \n"); } + else{ + for (int i=0; ikeyExists( "deltaT" )){ + // deltaT.clear(); + // tau.clear(); + // deltaT = ion_db->getVector( "deltaT" ); + // if (deltaT.size()!=number_ion_species){ + // ERROR("Error: number_ion_species and deltaT must be the same length! \n"); + // } + // else{//update relaxation parameter tau + // for (int i=0;ikeyExists( "voxel_length" )){//default unit: um/lu @@ -103,15 +125,15 @@ void ScaLBL_IonModel::ReadParams(string filename){ if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } - //Re-calcualte model parameters if user updates input - //TODO ion diffusivity needs rescale unit to LB unit - //TODO rescale ion initial concentration unit to LB unit - if (deltaT.size()>1){ - tau.clear(); - for (int i=0;iSDs); - if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + if (rank==0) printf("LB Ion Solver: Initialized solid phase & converting to Signed Distance function \n"); CalcDist(Distance,id_solid,*Dm); - if (rank == 0) cout << "Domain set." << endl; + if (rank == 0) cout << " Domain set." << endl; } void ScaLBL_IonModel::Create(){ @@ -209,13 +231,13 @@ void ScaLBL_IonModel::Create(){ Mask->CommInit(); Np=Mask->PoreCount(); //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); + if (rank==0) printf ("LB Ion Solver: Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout \n"); + if (rank==0) printf ("LB Ion Solver: Set up memory efficient layout \n"); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); @@ -224,7 +246,7 @@ void ScaLBL_IonModel::Create(){ // MAIN VARIABLES ALLOCATED HERE //........................................................................... // LBM variables - if (rank==0) printf ("Allocating distributions \n"); + if (rank==0) printf ("LB Ion Solver: Allocating distributions \n"); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); int neighborSize=18*(Np*sizeof(int)); @@ -235,7 +257,7 @@ void ScaLBL_IonModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &ChargeDensity, sizeof(double)*Np); //........................................................................... // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); + if (rank==0) printf ("LB Ion Solver: Setting up device map and neighbor list \n"); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); MPI_Barrier(comm); @@ -246,10 +268,13 @@ void ScaLBL_IonModel::Initialize(){ /* * This function initializes model */ - if (rank==0) printf ("Initializing D3Q7 distributions for ion transport\n"); + if (rank==0) printf ("LB Ion Solver: initializing D3Q7 distributions\n"); for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); } void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ @@ -260,13 +285,10 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ item = 1.0/item; } //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - starttime = MPI_Wtime(); + //double starttime,stoptime,cputime; + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //starttime = MPI_Wtime(); - if (rank==0) printf("***************************************************\n"); - if (rank==0) printf("LB-Ion Transport: timestepMax = %i\n", timestepMax); - if (rank==0) printf("***************************************************\n"); timestep=0; while (timestep < timestepMax) { //************************************************************************/ @@ -277,22 +299,15 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->SendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE - /* Update exterior ion concentration */ - } - - ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - - for (int ic=0; icLastExterior(), Np); } - ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); //LB-Ion collison for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + rlx[ic],Vt,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } // Set boundary conditions @@ -300,7 +315,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ for (int ic=0; icLastExterior(), Np); + rlx[ic],Vt,0, ScaLBL_Comm->LastExterior(), Np); } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -309,28 +324,17 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ //Update ion concentration and charge density for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL - } - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } - ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - - for (int ic=0; icRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE - } - for (int ic=0; icLastExterior(), Np); } - ScaLBL_D3Q7_IonChargeDensity(Ci, ChargeDensity, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); //LB-Ion collison for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + rlx[ic],Vt,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); } // Set boundary conditions @@ -338,25 +342,25 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ for (int ic=0; icLastExterior(), Np); + rlx[ic],Vt,0, ScaLBL_Comm->LastExterior(), Np); } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } //************************************************************************/ - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; + //stoptime = MPI_Wtime(); + //if (rank==0) printf("-------------------------------------------------------------------\n"); + //// Compute the walltime per timestep + //cputime = (stoptime - starttime)/timestep; + //// Performance obtained from each node + //double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); + //if (rank==0) printf("********************************************************\n"); + //if (rank==0) printf("CPU time = %f \n", cputime); + //if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + //MLUPS *= nprocs; + //if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + //if (rank==0) printf("********************************************************\n"); } diff --git a/models/IonModel.h b/models/IonModel.h index 8be11468..9fe0a146 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -22,24 +22,28 @@ public: ~ScaLBL_IonModel(); // functions in they should be run - void ReadParams(string filename); + void ReadParams(string filename,int num_iter,int num_iter_Stokes,double time_conv_Stokes); void ReadParams(std::shared_ptr db0); void SetDomain(); void ReadInput(); void Create(); void Initialize(); - void Run(double *Velocity); - void VelocityField(); + void Run(double *Velocity, double *ElectricField); bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; + double h;//domain resolution, unit [um/lu] + double time_conv; + double kb,electron_charge,T,Vt; + double k2_inv; + double tolerance; int number_ion_species; vector IonDiffusivity;//User input unit [m^2/sec] vector IonValence; vector IonConcentration;//unit [mol/m^3] - vector deltaT; + //vector deltaT; vector tau; int Nx,Ny,Nz,N,Np; diff --git a/models/MultiPhysController.cpp b/models/MultiPhysController.cpp new file mode 100644 index 00000000..f79db6b7 --- /dev/null +++ b/models/MultiPhysController.cpp @@ -0,0 +1,57 @@ +#include "models/MultiPhysController.h" + +ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM): +rank(RANK),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),SchmidtNum(0),comm(COMM) +{ + +} +ScaLBL_Multiphys_Controller::~ScaLBL_Multiphys_Controller(){ + +} + +void ScaLBL_Multiphys_Controller::ReadParams(string filename){ + + // read the input database + db = std::make_shared( filename ); + study_db = db->getDatabase( "MultiphysController" ); + + + // Default parameters + timestepMax = 10000; + Restart = false; + SchmidtNum = 1.0; + num_iter_Stokes=1; + num_iter_Ion=1; + + // load input parameters + if (study_db->keyExists( "timestepMax" )){ + timestepMax = study_db->getScalar( "timestepMax" ); + } + if (study_db->keyExists( "Schmidt_Number" )){ + SchmidtNum = study_db->getScalar( "Schmidt_Number" ); + } + // recalculate relevant parameters + if (SchmidtNum>1){ + num_iter_Stokes = int(round(SchmidtNum/2)*2); + num_iter_Ion = 1; + } + else if (SchmidtNum>0 && SchmidtNum<1){ + num_iter_Ion = int(round((1.0/SchmidtNum)/2)*2); + num_iter_Stokes = 1; + } + else{ + ERROR("Error: SchmidtNum (Schmidt number) must be a positive number! \n"); + } + + // load input parameters + // in case user wants to have an absolute control over the iternal iteration + if (study_db->keyExists( "num_iter_Ion" )){ + num_iter_Ion = study_db->getScalar( "num_iter_Ion" ); + } + if (study_db->keyExists( "num_iter_Stokes" )){ + num_iter_Stokes = study_db->getScalar( "num_iter_Stokes" ); + } + +} + + diff --git a/models/MultiPhysController.h b/models/MultiPhysController.h new file mode 100644 index 00000000..b108e28a --- /dev/null +++ b/models/MultiPhysController.h @@ -0,0 +1,47 @@ +/* + * Multiphysics controller that coordinates the coupling between different models + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "analysis/Minkowski.h" +#include "ProfilerApp.h" + +class ScaLBL_Multiphys_Controller{ +public: + ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_Multiphys_Controller(); + + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + + bool Restart; + //int timestep; + int timestepMax; + int num_iter_Stokes; + int num_iter_Ion; + double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity + + // input database + std::shared_ptr db; + std::shared_ptr study_db; + +private: + MPI_Comm comm; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); +}; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 9b6f3c89..ca251221 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -23,8 +23,8 @@ void ScaLBL_Poisson::ReadParams(string filename){ electric_db = db->getDatabase( "Poisson" ); k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice - deltaT = 0.3;//time step of LB-Poisson equation - tau = 0.5+k2_inv*deltaT; + gamma = 0.3;//time step of LB-Poisson equation + tau = 0.5+k2_inv*gamma; timestepMax = 100000; tolerance = 1.0e-6;//stopping criterion for obtaining steady-state electricla potential h = 1.0;//resolution; unit: um/lu @@ -44,8 +44,8 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "tolerance" )){ tolerance = electric_db->getScalar( "tolerance" ); } - if (electric_db->keyExists( "deltaT" )){ - deltaT = electric_db->getScalar( "deltaT" ); + if (electric_db->keyExists( "gamma" )){ + gamma = electric_db->getScalar( "gamma" ); } if (electric_db->keyExists( "epsilonR" )){ epsilonR = electric_db->getScalar( "epsilonR" ); @@ -60,8 +60,12 @@ void ScaLBL_Poisson::ReadParams(string filename){ //Re-calcualte model parameters if user updates input epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity - tau = 0.5+k2_inv*deltaT; + tau = 0.5+k2_inv*gamma; + if (rank==0) printf("***********************************************************************************\n"); + if (rank==0) printf("LB-Poisson Solver: steady-state MaxTimeStep = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance); + if (rank==0) printf(" LB relaxation tau = %.5g \n", tau); + if (rank==0) printf("***********************************************************************************\n"); } void ScaLBL_Poisson::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis @@ -143,9 +147,9 @@ void ScaLBL_Poisson::ReadInput(){ } } // MeanFilter(Averages->SDs); - if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); + if (rank==0) printf("LB-Poisson Solver: Initialized solid phase & converting to Signed Distance function \n"); CalcDist(Distance,id_solid,*Dm); - if (rank == 0) cout << "Domain set." << endl; + if (rank == 0) cout << " Domain set." << endl; } void ScaLBL_Poisson::Create(){ @@ -159,13 +163,13 @@ void ScaLBL_Poisson::Create(){ Mask->CommInit(); Np=Mask->PoreCount(); //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); + if (rank==0) printf ("LB-Poisson Solver: Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout \n"); + if (rank==0) printf ("LB-Poisson Solver: Set up memory efficient layout \n"); Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); @@ -174,7 +178,7 @@ void ScaLBL_Poisson::Create(){ // MAIN VARIABLES ALLOCATED HERE //........................................................................... // LBM variables - if (rank==0) printf ("Allocating distributions \n"); + if (rank==0) printf ("LB-Poisson Solver: Allocating distributions \n"); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); int neighborSize=18*(Np*sizeof(int)); @@ -185,7 +189,7 @@ void ScaLBL_Poisson::Create(){ ScaLBL_AllocateDeviceMemory((void **) &ElectricField, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); + if (rank==0) printf ("LB-Poisson Solver: Setting up device map and neighbor list \n"); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); MPI_Barrier(comm); @@ -196,22 +200,17 @@ void ScaLBL_Poisson::Initialize(){ /* * This function initializes model */ - if (rank==0) printf ("Initializing D3Q7 distributions for LB-Poisson solver\n"); + if (rank==0) printf ("LB-Poisson Solver: initializing D3Q7 distributions\n"); ScaLBL_D3Q7_Poisson_Init(fq, Np); } void ScaLBL_Poisson::Run(double *ChargeDensity){ - //LB-related parameter - double rlx = 1.0/tau; - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - starttime = MPI_Wtime(); - if (rank==0) printf("***************************************************************************\n"); - if (rank==0) printf("LB-Poisson Solver: timestepMax = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance); - if (rank==0) printf("***************************************************************************\n"); + //double starttime,stoptime,cputime; + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //starttime = MPI_Wtime(); + timestep=0; double error = 1.0; double psi_avg_previous = 0.0; @@ -220,21 +219,21 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // *************ODD TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE // Set boundary conditions /* ... */ - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, rlx, epsilon_LB, deltaT, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -267,24 +266,22 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ psi_avg_previous = psi_avg; } } - //************************************************************************/ - stoptime = MPI_Wtime(); - if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n"); - if (rank==0) printf("---------------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("******************* LB-Poisson Solver Statistics ********************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("*********************************************************************\n"); + //************************************************************************/ + //stoptime = MPI_Wtime(); + ////if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n"); + ////if (rank==0) printf("---------------------------------------------------------------------------\n"); + //// Compute the walltime per timestep + //cputime = (stoptime - starttime)/timestep; + //// Performance obtained from each node + //double MLUPS = double(Np)/cputime/1000000; + + //if (rank==0) printf("******************* LB-Poisson Solver Statistics ********************\n"); + //if (rank==0) printf("CPU time = %f \n", cputime); + //if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + //MLUPS *= nprocs; + //if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + //if (rank==0) printf("*********************************************************************\n"); } -//void ScaLBL_Poisson::get_ElectricField(){ -//// ??? -//} diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp new file mode 100644 index 00000000..8bde5c75 --- /dev/null +++ b/models/StokesModel.cpp @@ -0,0 +1,543 @@ +/* + * Multi-relaxation time LBM Model + */ +#include "models/StokesModel.h" +#include "analysis/distance.h" +#include "common/ReadMicroCT.h" + +ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + +} +ScaLBL_StokesModel::~ScaLBL_StokesModel(){ + +} + +void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + stokes_db = db->getDatabase( "Stokes" ); + + //------ Load number of iteration from multiphysics controller ------// + timestepMax = num_iter; + //-------------------------------------------------------------------// + + //---------------------- Default model parameters --------------------------// + nu_phys = 1.004e-6;//by default use water kinematic viscosity at 20C; unit [m^2/sec] + h = 1.0;//image resolution;[um] + tau = 1.0; + mu = (tau-0.5)/3.0;//LB kinematic viscosity;unit [lu^2/lt] + time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + tolerance = 1.0e-8; + Fx = Fy = 0.0; + Fz = 1.0e-5; + //--------------------------------------------------------------------------// + + // Single-fluid Navier-Stokes Model parameters + //if (stokes_db->keyExists( "timestepMax" )){ + // timestepMax = stokes_db->getScalar( "timestepMax" ); + //} + if (stokes_db->keyExists( "tolerance" )){ + tolerance = stokes_db->getScalar( "tolerance" ); + } + if (stokes_db->keyExists( "tau" )){ + tau = stokes_db->getScalar( "tau" ); + } + if (stokes_db->keyExists( "nu_phys" )){ + nu_phys = stokes_db->getScalar( "nu_phys" ); + } + if (stokes_db->keyExists( "F" )){ + Fx = stokes_db->getVector( "F" )[0]; + Fy = stokes_db->getVector( "F" )[1]; + Fz = stokes_db->getVector( "F" )[2]; + } + if (stokes_db->keyExists( "Restart" )){ + Restart = stokes_db->getScalar( "Restart" ); + } + if (stokes_db->keyExists( "din" )){ + din = stokes_db->getScalar( "din" ); + } + if (stokes_db->keyExists( "dout" )){ + dout = stokes_db->getScalar( "dout" ); + } + if (stokes_db->keyExists( "flux" )){ + flux = stokes_db->getScalar( "flux" ); + } + + // Read domain parameters + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + + // Re-calculate model parameters due to parameter read + mu=(tau-0.5)/3.0; + time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + if (rank==0) printf("*****************************************************\n"); + if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n"); + if (rank==0) printf(" Time conversion factor: %.5g [sec/lt]\n", time_conv); + if (rank==0) printf(" Internal iteration: %i [lt]\n", timestepMax); + if (rank==0) printf("*****************************************************\n"); +} +void ScaLBL_StokesModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + + // domain parameters + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + + N = Nx*Ny*Nz; + Distance.resize(Nx,Ny,Nz); + Velocity_x.resize(Nx,Ny,Nz); + Velocity_y.resize(Nx,Ny,Nz); + Velocity_z.resize(Nx,Ny,Nz); + + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); + + rank = Dm->rank(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); +} + +void ScaLBL_StokesModel::ReadInput(){ + + sprintf(LocalRankString,"%05d",Dm->rank()); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + + if (domain_db->keyExists( "Filename" )){ + auto Filename = domain_db->getScalar( "Filename" ); + Mask->Decomp(Filename); + } + else if (domain_db->keyExists( "GridFile" )){ + // Read the local domain data + auto input_id = readMicroCT( *domain_db, comm ); + // Fill the halo (assuming GCW of 1) + array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; + ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; + ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); + fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + Array id_view; + id_view.viewRaw( size1, Mask->id ); + fill.copy( input_id, id_view ); + fill.fill( id_view ); + } + else{ + Mask->ReadIDs(); + } + + // Generate the signed distance map + // Initialize the domain and communication + Array id_solid(Nx,Ny,Nz); + // Solve for the position of the solid phase + for (int k=0;kid[n] > 0) id_solid(i,j,k) = 1; + else id_solid(i,j,k) = 0; + } + } + } + // Initialize the signed distance function + for (int k=0;kSDs); + if (rank==0) printf("LB Single-Fluid Solver: initialized solid phase & converting to Signed Distance function \n"); + CalcDist(Distance,id_solid,*Dm); + if (rank == 0) cout << " Domain set." << endl; +} + +void ScaLBL_StokesModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + int rank=Mask->rank(); + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("LB Single-Fluid Solver: Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("LB Single-Fluid Solver: Set up memory efficient layout \n"); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("LB Single-Fluid Solver: Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("LB Single-Fluid Solver: Setting up device map and neighbor list \n"); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + MPI_Barrier(comm); + +} + +void ScaLBL_StokesModel::Initialize(){ + /* + * This function initializes model + */ + if (rank==0) printf("LB Single-Fluid Solver: Initializing distributions \n"); + if (rank==0) printf("****************************************************************\n"); + ScaLBL_D3Q19_Init(fq, Np); +} + +void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ + double rlx_setA=1.0/tau; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + timestep = 0; + while (timestep < timestepMax) { + //************************************************************************/ + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + } +} + +//void ScaLBL_StokesModel::computeVelocity_phys(){ +// ScaLBL_D3Q19_Momentum(fq,Velocity, Np); +// ScaLBL_DeviceBarrier(); MPI_Barrier(comm); +//} + +void ScaLBL_StokesModel::Run(){ + double rlx_setA=1.0/tau; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + + Minkowski Morphology(Mask); + + if (rank==0){ + bool WriteHeader=false; + FILE *log_file = fopen("Permeability.csv","r"); + if (log_file != NULL) + fclose(log_file); + else + WriteHeader=true; + + if (WriteHeader){ + log_file = fopen("Permeability.csv","a+"); + fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n"); + fclose(log_file); + } + } + + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + starttime = MPI_Wtime(); + if (rank==0) printf("****************************************************************\n"); + if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: timestepMax = %i\n", timestepMax); + if (rank==0) printf("****************************************************************\n"); + timestep=0; + double error = 1.0; + double flow_rate_previous = 0.0; + while (timestep < timestepMax && error > tolerance) { + //************************************************************************/ + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition == 3){ + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } + ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + + if (timestep%1000==0){ + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int k=1; k 0){ + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; + } + } + } + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = Fx/force_mag; + double dir_y = Fy/force_mag; + double dir_z = Fz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + + error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); + flow_rate_previous = flow_rate; + + //if (rank==0) printf("Computing Minkowski functionals \n"); + Morphology.ComputeScalar(Distance,0.f); + //Morphology.PrintAll(); + double mu = (tau-0.5)/3.f; + double Vs = Morphology.V(); + double As = Morphology.A(); + double Hs = Morphology.H(); + double Xs = Morphology.X(); + Vs=sumReduce( Dm->Comm, Vs); + As=sumReduce( Dm->Comm, As); + Hs=sumReduce( Dm->Comm, Hs); + Xs=sumReduce( Dm->Comm, Xs); + double h = Dm->voxel_length; + double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; + if (rank==0) { + printf(" %f\n",absperm); + FILE * log_file = fopen("Permeability.csv","a"); + fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, + h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); + fclose(log_file); + } + } + } + //************************************************************************/ + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + +} + +void ScaLBL_StokesModel::VelocityField(){ + +/* Minkowski Morphology(Mask); + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); + + memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); + Morphology.Initialize(); + Morphology.UpdateMeshValues(); + Morphology.ComputeLocal(); + Morphology.Reduce(); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int n=0; nLastExterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + + for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ + vax_loc += VELOCITY[n]; + vay_loc += VELOCITY[Np+n]; + vaz_loc += VELOCITY[2*Np+n]; + count_loc+=1.0; + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + double mu = (tau-0.5)/3.f; + if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); + if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, + Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); + */ + + std::vector visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + Array& SignData = visData[0].vars[0]->data; + Array& VelxData = visData[0].vars[1]->data; + Array& VelyData = visData[0].vars[2]->data; + Array& VelzData = visData[0].vars[3]->data; + + ASSERT(visData[0].vars[0]->name=="SignDist"); + ASSERT(visData[0].vars[1]->name=="Velocity_x"); + ASSERT(visData[0].vars[2]->name=="Velocity_y"); + ASSERT(visData[0].vars[3]->name=="Velocity_z"); + + fillData.copy(Distance,SignData); + fillData.copy(Velocity_x,VelxData); + fillData.copy(Velocity_y,VelyData); + fillData.copy(Velocity_z,VelzData); + + IO::writeData( timestep, visData, Dm->Comm ); + +} diff --git a/models/StokesModel.h b/models/StokesModel.h new file mode 100644 index 00000000..6ff8f6fa --- /dev/null +++ b/models/StokesModel.h @@ -0,0 +1,75 @@ +/* + * Multi-relaxation time LBM Model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/ScaLBL.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "analysis/Minkowski.h" +#include "ProfilerApp.h" + +class ScaLBL_StokesModel{ +public: + ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_StokesModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void VelocityField(); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + double tau,mu; + double Fx,Fy,Fz,flux; + double din,dout; + double tolerance; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr stokes_db; + + IntArray Map; + DoubleArray Distance; + int *NeighborList; + double *fq; + double *Velocity; + double *Pressure; + + //Minkowski Morphology; + + DoubleArray Velocity_x; + DoubleArray Velocity_y; + DoubleArray Velocity_z; +private: + MPI_Comm comm; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); +}; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 0c9da812..d4a83384 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -5,74 +5,99 @@ #include #include #include +#include -#include "models/DFHModel.h" +#include "models/StokesModel.h" #include "models/IonModel.h" #include "models/PoissonSolver.h" - -//#define WRE_SURFACES - -/* - * Simulator for two-phase flow in porous media - * James E. McClure 2013-2014 - */ +#include "models/MultiPhysController.h" using namespace std; -//************************************************************************* -// Implementation of Two-Phase Immiscible LBM using CUDA -//************************************************************************* +//*************************************************************************** +// Implementation of Multiphysics simulator using lattice-Boltzmann method +//*************************************************************************** int main(int argc, char **argv) { - // Initialize MPI - int provided_thread_support = -1; - MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - if ( rank==0 && provided_thread_support Date: Sun, 16 Aug 2020 11:20:11 -0400 Subject: [PATCH 209/270] save the work;CPU version compiled; to be tested --- common/ScaLBL.cpp | 67 ++++++++------ common/ScaLBL.h | 38 ++++++-- cpu/D3Q7BC.cpp | 32 +++++++ cpu/Ion.cpp | 23 ++--- models/IonModel.cpp | 155 ++++++++++++++++++++++++++------- models/IonModel.h | 5 +- models/MultiPhysController.cpp | 2 +- models/MultiPhysController.h | 2 + models/PoissonSolver.cpp | 122 +++++++++++++++++++++++--- models/PoissonSolver.h | 6 +- models/StokesModel.cpp | 2 +- models/StokesModel.h | 6 +- tests/CMakeLists.txt | 2 +- 13 files changed, 361 insertions(+), 101 deletions(-) create mode 100644 cpu/D3Q7BC.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 28459298..a77afbca 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -857,8 +857,8 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, int Np) { - int idx,i,j,k; - int neighbor; + int idx,i,j,k; + int neighbor; // save list of bounce-back distributions and interaction sites n_bb_d3q7 = 0; n_bb_d3q19 = 0; @@ -929,8 +929,9 @@ void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, in } int *bb_dist_tmp = new int [local_count]; - bb_interactions = new int [local_count]; + int *bb_interactions_tmp = new int [local_count]; ScaLBL_AllocateDeviceMemory((void **) &bb_dist, sizeof(int)*local_count); + ScaLBL_AllocateDeviceMemory((void **) &bb_interactions, sizeof(int)*local_count); local_count=0; for (k=1;k& IonValence, int number_ion_species, int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){ int n; - int ic=number_ion_species; double Ci;//ion concentration of species i double CD;//charge density + double CD_tmp; double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant - for (n=start; n0){ - for (n=start; n0) + CD_tmp; } } + diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 2e5b4e71..8c12b587 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -1,14 +1,14 @@ /* - * Multi-relaxation time LBM Model + * Dilute Ion Transport LBM Model */ #include "models/IonModel.h" #include "analysis/distance.h" #include "common/ReadMicroCT.h" ScaLBL_IonModel::ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +rank(RANK),nprocs(NP),timestep(0),timestepMax(0),time_conv(0),kb(0),electron_charge(0),T(0),Vt(0),k2_inv(0),h(0), +tolerance(0),number_ion_species(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0), +BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { } @@ -44,8 +44,8 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke IonValence.push_back(1);//algebraic valence charge IonConcentration.push_back(1.0e-3);//user-input ion concentration has physical unit [mol/m^3] //deltaT.push_back(1.0); - //tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivisty[0]); - tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivisty[0]); + //tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivity[0]); + tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); //--------------------------------------------------------------------------// // LB-Ion Model parameters @@ -74,10 +74,10 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke } else{ for (int i=0; ikeyExists( "deltaT" )){ - // deltaT.clear(); - // tau.clear(); - // deltaT = ion_db->getVector( "deltaT" ); - // if (deltaT.size()!=number_ion_species){ - // ERROR("Error: number_ion_species and deltaT must be the same length! \n"); - // } - // else{//update relaxation parameter tau - // for (int i=0;ikeyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } + BoundaryCondition = 0; if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } + BoundaryConditionSolid = 0; + if (domain_db->keyExists( "BC_Solid" )){ + BoundaryConditionSolid = domain_db->getScalar( "BC_Solid" ); + } if (rank==0) printf("*****************************************************\n"); if (rank==0) printf("LB Ion Transport Solver: \n"); @@ -134,6 +125,18 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke if (rank==0) printf(" Ion %i: LB relaxation tau = %.5g\n", i+1,tau[i]); } if (rank==0) printf("*****************************************************\n"); + + switch (BoundaryConditionSolid){ + case 0: + if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned"); + break; + case 1: + if (rank==0) printf("LB Ion Solver: solid boundary: Neumann-type surfacen ion concentration is assigned"); + break; + default: + if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned"); + break; + } } void ScaLBL_IonModel::SetDomain(){ @@ -220,6 +223,64 @@ void ScaLBL_IonModel::ReadInput(){ if (rank == 0) cout << " Domain set." << endl; } + +void ScaLBL_IonModel::AssignSolidBoundary(double *ion_solid) +{ + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = ion_db->getVector( "SolidLabels" ); + auto AffinityList = ion_db->getVector( "SolidValues" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: LB Ion Solver: SolidLabels and SolidValues must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (size_t idx=0; idxid[n]; + AFFINITY=0.f; + // Assign the affinity from the paired list + for (unsigned int idx=0; idx < NLABELS; idx++){ + //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); + if (VALUE == LabelList[idx]){ + AFFINITY=AffinityList[idx]; + //NOTE need to convert the user input phys unit to LB unit + AFFINITY = AFFINITY*(h*h*1.0e-12); + label_count[idx] += 1.0; + idx = NLABELS; + //Mask->id[n] = 0; // set mask to zero since this is an immobile component + } + } + ion_solid[n] = AFFINITY; + } + } + } + + for (size_t idx=0; idxComm, label_count[idx]); + + if (rank==0){ + printf("LB Ion Solver: Ion Solid labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxSetupBounceBackList(Map, Mask->id, Np); + MPI_Barrier(comm); + + double *IonSolid_host; + IonSolid_host = new double[Nx*Ny*Nz]; + AssignSolidBoundary(IonSolid_host); + ScaLBL_CopyToDevice(IonSolid, IonSolid_host, Nx*Ny*Nz*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] IonSolid_host; + } + + } void ScaLBL_IonModel::Initialize(){ @@ -273,8 +351,10 @@ void ScaLBL_IonModel::Initialize(){ ScaLBL_D3Q7_Ion_Init(&fq[ic*Np*7],&Ci[ic*Np],IonConcentration[ic],Np); } if (rank==0) printf ("LB Ion Solver: initializing charge density\n"); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); + for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); + } } void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ @@ -299,10 +379,11 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->SendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); } - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); //LB-Ion collison for (int ic=0; icLastExterior(), Np); } - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + if (BoundaryConditionSolid==1){ + for (int ic=0; icSolidNeumannD3Q7(&fq[ic*Np*7], IonSolid); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + } + } // *************EVEN TIMESTEP*************// timestep++; @@ -327,9 +414,9 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); } - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence, number_ion_species, 0, ScaLBL_Comm->LastExterior(), Np); //LB-Ion collison for (int ic=0; icLastExterior(), Np); } - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + if (BoundaryConditionSolid==1){ + for (int ic=0; icSolidNeumannD3Q7(&fq[ic*Np*7], IonSolid); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + } + } //************************************************************************/ } //************************************************************************/ diff --git a/models/IonModel.h b/models/IonModel.h index 9fe0a146..0edec1b0 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -30,9 +30,10 @@ public: void Initialize(); void Run(double *Velocity, double *ElectricField); - bool Restart,pBC; + //bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; + int BoundaryConditionSolid; double h;//domain resolution, unit [um/lu] double time_conv; double kb,electron_charge,T,Vt; @@ -64,6 +65,7 @@ public: double *fq; double *Ci; double *ChargeDensity; + double *IonSolid; private: MPI_Comm comm; @@ -75,4 +77,5 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); + void AssignSolidBoundary(double *ion_solid); }; diff --git a/models/MultiPhysController.cpp b/models/MultiPhysController.cpp index f79db6b7..1453067a 100644 --- a/models/MultiPhysController.cpp +++ b/models/MultiPhysController.cpp @@ -1,7 +1,7 @@ #include "models/MultiPhysController.h" ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM): -rank(RANK),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),SchmidtNum(0),comm(COMM) +rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),SchmidtNum(0),comm(COMM) { } diff --git a/models/MultiPhysController.h b/models/MultiPhysController.h index b108e28a..e51f2102 100644 --- a/models/MultiPhysController.h +++ b/models/MultiPhysController.h @@ -30,6 +30,8 @@ public: int num_iter_Ion; double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity + int rank,nprocs; + // input database std::shared_ptr db; std::shared_ptr study_db; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 466c8841..a4decc24 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -6,9 +6,9 @@ #include "common/ReadMicroCT.h" ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),gamma(0),tolerance(0),h(0), +epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), +nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { } @@ -54,9 +54,16 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } + + BoundaryCondition = 0; if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } + BoundaryConditionSolid = 1; + if (domain_db->keyExists( "BC_Solid" )){ + BoundaryConditionSolid = domain_db->getScalar( "BC_Solid" ); + } + //Re-calcualte model parameters if user updates input epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity @@ -66,6 +73,18 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (rank==0) printf("LB-Poisson Solver: steady-state MaxTimeStep = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance); if (rank==0) printf(" LB relaxation tau = %.5g \n", tau); if (rank==0) printf("***********************************************************************************\n"); + + switch (BoundaryConditionSolid){ + case 1: + if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned"); + break; + case 2: + if (rank==0) printf("LB-Poisson Solver: solid boundary: Neumann-type surfacen charge density is assigned"); + break; + default: + if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned"); + break; + } } void ScaLBL_Poisson::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis @@ -152,6 +171,77 @@ void ScaLBL_Poisson::ReadInput(){ if (rank == 0) cout << " Domain set." << endl; } +void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) +{ + size_t NLABELS=0; + signed char VALUE=0; + double AFFINITY=0.f; + + auto LabelList = electric_db->getVector( "SolidLabels" ); + auto AffinityList = electric_db->getVector( "SolidValues" ); + + NLABELS=LabelList.size(); + if (NLABELS != AffinityList.size()){ + ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (size_t idx=0; idxid[n]; + AFFINITY=0.f; + // Assign the affinity from the paired list + for (unsigned int idx=0; idx < NLABELS; idx++){ + //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); + if (VALUE == LabelList[idx]){ + AFFINITY=AffinityList[idx]; + //NOTE need to convert the user input phys unit to LB unit + if (BoundaryConditionSolid==2){ + //for BCS=1, i.e. Dirichlet-type, no need for unit conversion + AFFINITY = AFFINITY*(h*h*1.0e-12); + } + label_count[idx] += 1.0; + idx = NLABELS; + //Mask->id[n] = 0; // set mask to zero since this is an immobile component + } + } + poisson_solid[n] = AFFINITY; + } + } + } + + for (size_t idx=0; idxComm, label_count[idx]); + + if (rank==0){ + printf("LB-Poisson Solver: Poisson Solid labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxn_bb_d3q7); + ScaLBL_AllocateDeviceMemory((void **) &PoissonSolid, sizeof(double)*Nx*Ny*Nz); //........................................................................... - // initialize the zeta function (example is zeta is constant on solid surface) - double *tmpZeta = new double[ScaLBL_Comm->n_bb_d3q7]; - for int (i=0; in_bb_d3q7; i++){ - tmpZeta[i] = 1.0/k2_inv; // this has to be read from input file - } - ScaLBL_CopyToDevice(zeta, tmpZeta, sizeof(double)*ScaLBL_Comm->n_bb_d3q7); - delete [] tmpZeta; // Update GPU data structures if (rank==0) printf ("LB-Poisson Solver: Setting up device map and neighbor list \n"); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //Initialize solid boundary for electrical potential + ScaLBL_Comm->SetupBounceBackList(Map, Mask->id, Np); + MPI_Barrier(comm); + + double *PoissonSolid_host; + PoissonSolid_host = new double[Nx*Ny*Nz]; + AssignSolidBoundary(PoissonSolid_host); + ScaLBL_CopyToDevice(PoissonSolid, PoissonSolid_host, Nx*Ny*Nz*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] PoissonSolid_host; } void ScaLBL_Poisson::Initialize(){ @@ -233,7 +327,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->SolidDirichletD3Q7(fq, zeta); + ScaLBL_Comm->SolidDirichletD3Q7(fq, PoissonSolid); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// @@ -244,14 +338,14 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // Set boundary conditions /* ... */ ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->SolidDirichletD3Q7(fq, zeta); + ScaLBL_Comm->SolidDirichletD3Q7(fq, PoissonSolid); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ // Check convergence of steady-state solution if (timestep%analysis_interval==0){ - ScaLBL_Comm->RegularLayout(Map,&Psi,Psi_host); + ScaLBL_Comm->RegularLayout(Map,Psi,Psi_host); double count_loc=0; double count; double psi_avg; diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 625c602f..fe003c18 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -33,9 +33,10 @@ public: int timestep,timestepMax; int analysis_interval; int BoundaryCondition; + int BoundaryConditionSolid; double tau; double tolerance; - double k2_inv,deltaT; + double k2_inv,gamma; double epsilon0,epsilon0_LB,epsilonR,epsilon_LB; int Nx,Ny,Nz,N,Np; @@ -58,7 +59,7 @@ public: double *fq; double *Psi; double *ElectricField; - double *zeta; + double *PoissonSolid; private: MPI_Comm comm; @@ -70,4 +71,5 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); + void AssignSolidBoundary(double *poisson_solid); }; diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 8bde5c75..d0664955 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -7,7 +7,7 @@ ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),h(0),nu_phys(0),time_conv(0),tolerance(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { diff --git a/models/StokesModel.h b/models/StokesModel.h index 6ff8f6fa..c44cb1c5 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -21,13 +21,14 @@ public: ~ScaLBL_StokesModel(); // functions in they should be run - void ReadParams(string filename); + void ReadParams(string filename,int num_iter); void ReadParams(std::shared_ptr db0); void SetDomain(); void ReadInput(); void Create(); void Initialize(); void Run(); + void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); bool Restart,pBC; @@ -37,6 +38,9 @@ public: double Fx,Fy,Fz,flux; double din,dout; double tolerance; + double nu_phys; + double time_conv; + double h;//image resolution int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8c937107..1a8bfac0 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,7 +4,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_dfh_simulator.cpp ) +ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_dfh_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) From 771f679f5c598438ac31b7798e74128bdb26b0ea Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 17 Aug 2020 09:59:22 -0400 Subject: [PATCH 210/270] add output; CPU version compiled; to be tested --- common/ScaLBL.h | 8 ++- cpu/Ion.cpp | 10 ++++ cpu/Stokes.cpp | 60 ++++++++++++++++++--- models/IonModel.cpp | 19 +++++++ models/IonModel.h | 1 + models/PoissonSolver.cpp | 11 ++++ models/PoissonSolver.h | 3 +- models/StokesModel.cpp | 53 +++++++++++++++--- models/StokesModel.h | 2 + tests/lbpm_electrokinetic_dfh_simulator.cpp | 4 +- 10 files changed, 151 insertions(+), 20 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index dc3831c7..4d8a3dc3 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -49,6 +49,8 @@ extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np); extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np); +extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np); + extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np); // BGK MODEL @@ -89,6 +91,8 @@ extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np); +extern "C" void ScaLBL_IonConcentration_Phys(double *Den, double h, int ion_component, int start, int finish, int Np); + // LBM Poisson solver extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, @@ -102,10 +106,10 @@ extern "C" void ScaLBL_D3Q7_Poisson_Init(double *dist, int Np); // LBM Stokes Model (adapted from MRT model) extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, int start, int finish, int Np); + double Gx, double Gy, double Gz, double Ex, double Ey, double Ez, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, int start, int finish, int Np); + double Gx, double Gy, double Gz, double Ex, double Ey, double Ez, int start, int finish, int Np); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index 5586aae4..b04b3c9c 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -235,3 +235,13 @@ extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity } } +extern "C" void ScaLBL_IonConcentration_Phys(double *Den, double h, int ion_component, int start, int finish, int Np){ + //h: resolution [um/lu] + int n; + double Ci; + + for (n=start; n extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, int start, int finish, int Np) + double Gx, double Gy, double Gz, double Ex_const, double Ey_const, double Ez_const, int start, int finish, int Np) { double fq; // conserved momemnts @@ -32,9 +32,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do //Load data rhoE = ChargeDensity[n]; - Ex = ElectricField[n+0*Np]; - Ey = ElectricField[n+1*Np]; - Ez = ElectricField[n+2*Np]; + Ex = ElectricField[n+0*Np]+Ex_const; + Ey = ElectricField[n+1*Np]+Ey_const; + Ez = ElectricField[n+2*Np]+Ez_const; //compute total body force, including input body force (Gx,Gy,Gz) Fx = Gx + rhoE*Ex; Fy = Gy + rhoE*Ey; @@ -455,7 +455,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do } extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, int start, int finish, int Np) + double Gx, double Gy, double Gz, double Ex_const, double Ey_const, double Ez_const, int start, int finish, int Np) { double fq; // conserved momemnts @@ -487,9 +487,9 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do //Load data rhoE = ChargeDensity[n]; - Ex = ElectricField[n+0*Np]; - Ey = ElectricField[n+1*Np]; - Ez = ElectricField[n+2*Np]; + Ex = ElectricField[n+0*Np]+Ex_const; + Ey = ElectricField[n+1*Np]+Ey_const; + Ez = ElectricField[n+2*Np]+Ez_const; //compute total body force, including input body force (Gx,Gy,Gz) Fx = Gx + rhoE*Ex; Fy = Gy + rhoE*Ey; @@ -955,3 +955,47 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do } } +extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np) +{ + //h: resolution [um/lu] + //time_conv: time conversion factor [sec/lt] + int n; + // distributions + double f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double vx,vy,vz; + + for (n=0; nFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } + + DoubleArray PhaseField(Nx,Ny,Nz); + for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + FILE *OUTFILE; + sprintf(LocalRankFilename,"Ion%02i.%05i.raw",ic+1,rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + } + +} + diff --git a/models/IonModel.h b/models/IonModel.h index 0edec1b0..6232d105 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -29,6 +29,7 @@ public: void Create(); void Initialize(); void Run(double *Velocity, double *ElectricField); + void getIonConcentration(); //bool Restart,pBC; int timestep,timestepMax; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index a4decc24..27c0cda5 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -390,3 +390,14 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ } +void ScaLBL_Poisson::getElectricalPotential(){ + + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + FILE *OUTFILE; + sprintf(LocalRankFilename,"Electrical_Potential.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); +} diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index fe003c18..9f66c09c 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -28,7 +28,8 @@ public: void Create(); void Initialize(); void Run(double *ChargeDensity); - + void getElectricalPotential(); + //bool Restart,pBC; int timestep,timestepMax; int analysis_interval; diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index d0664955..46f28f45 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -35,6 +35,9 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ tolerance = 1.0e-8; Fx = Fy = 0.0; Fz = 1.0e-5; + //Body electric field [V/lu] + Ex = Ey = 0.0; + Ez = 1.0e-3; //--------------------------------------------------------------------------// // Single-fluid Navier-Stokes Model parameters @@ -55,6 +58,11 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ Fy = stokes_db->getVector( "F" )[1]; Fz = stokes_db->getVector( "F" )[2]; } + if (stokes_db->keyExists( "ElectricField" )){//NOTE user-input has physical unit [V/m] + Ex = stokes_db->getVector( "ElectricField" )[0]; + Ey = stokes_db->getVector( "ElectricField" )[1]; + Ez = stokes_db->getVector( "ElectricField" )[2]; + } if (stokes_db->keyExists( "Restart" )){ Restart = stokes_db->getScalar( "Restart" ); } @@ -79,6 +87,11 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ // Re-calculate model parameters due to parameter read mu=(tau-0.5)/3.0; time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + // convert user-input electric field ([V/m]) from physical unit to LB unit + Ex = Ex*(h*1.0e-6);//LB electric field: V/lu + Ey = Ey*(h*1.0e-6); + Ez = Ez*(h*1.0e-6); + if (rank==0) printf("*****************************************************\n"); if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n"); if (rank==0) printf(" Time conversion factor: %.5g [sec/lt]\n", time_conv); @@ -232,7 +245,7 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ while (timestep < timestepMax) { //************************************************************************/ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions @@ -248,12 +261,12 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions if (BoundaryCondition == 3){ @@ -268,16 +281,40 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } } -//void ScaLBL_StokesModel::computeVelocity_phys(){ -// ScaLBL_D3Q19_Momentum(fq,Velocity, Np); -// ScaLBL_DeviceBarrier(); MPI_Barrier(comm); -//} +void ScaLBL_StokesModel::getVelocity(){ + //get velocity in physical unit [m/sec] + ScaLBL_D3Q19_Momentum_Phys(fq, Velocity, h, time_conv, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + FILE *VELX_FILE; + sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + VELX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELX_FILE); + fclose(VELX_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + FILE *VELY_FILE; + sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + VELY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELY_FILE); + fclose(VELY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + FILE *VELZ_FILE; + sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + VELZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,VELZ_FILE); + fclose(VELZ_FILE); + +} void ScaLBL_StokesModel::Run(){ double rlx_setA=1.0/tau; diff --git a/models/StokesModel.h b/models/StokesModel.h index c44cb1c5..f0a4de6a 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -30,12 +30,14 @@ public: void Run(); void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); + void getVelocity(); bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; double tau,mu; double Fx,Fy,Fz,flux; + double Ex,Ey,Ez; double din,dout; double tolerance; double nu_phys; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index d4a83384..bdc15ef4 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -89,7 +89,9 @@ int main(int argc, char **argv) //-------------------------------------------- } - //StokesModel.WriteDebug(); + StokesModel.getVelocity(); + PoissonSolver.getElectricalPotential(); + IonModel.getIonConcentration(); PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_electrokinetic_simulator",1); From 5756d6f1386291e181a166bc617d1e7e63a2398d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 18 Aug 2020 12:40:41 -0400 Subject: [PATCH 211/270] fix a few trivial bugs; add some checkpoint print; still debugging --- common/ScaLBL.cpp | 28 ++++++++++----------- cpu/Ion.cpp | 26 +++++++++---------- models/IonModel.cpp | 28 +++++++++++++++------ models/PoissonSolver.cpp | 21 +++++++++------- models/StokesModel.cpp | 2 +- tests/lbpm_electrokinetic_dfh_simulator.cpp | 5 ++++ 6 files changed, 65 insertions(+), 45 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index a77afbca..854c8b49 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1429,37 +1429,37 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){ ScaLBL_DeviceBarrier(); // Pack the distributions //...Packing for x face(2,8,10,12,14)................................ - ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N); MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N); MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N); MPI_Isend(sendbuf_y, 2*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); MPI_Irecv(recvbuf_Y, 2*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N); MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N); MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*N],N); + ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N); //................................................................................... // Send all the distributions @@ -1483,33 +1483,33 @@ void ScaLBL_Communicator::RecvD3Q7AA(double *Aq, int Component){ // Unpack the distributions on the device //................................................................................... //...Unpacking for x face(2,8,10,12,14)................................ - ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,&Aq[Component*7*N],N); //................................................................................... //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,&Aq[Component*7*N],N); //................................................................................... //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,&Aq[Component*7*N],N); //................................................................................... //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,&Aq[Component*7*N],N); //................................................................................... if (BoundaryCondition > 0){ if (kproc != 0){ //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N); } if (kproc != nprocz-1){ //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N); } } else { //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N); //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*N],N); + ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N); } //................................................................................... diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index b04b3c9c..2c7e72a9 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -128,22 +128,22 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *D dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; // q = 1 - dist[nr2] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(ux+uEPx)); + dist[nr2] = f1*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(ux+uEPx)); // q=2 - dist[nr1] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(ux+uEPx)); + dist[nr1] = f2*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(ux+uEPx)); // q = 3 - dist[nr4] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uy+uEPy)); + dist[nr4] = f3*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(uy+uEPy)); // q = 4 - dist[nr3] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uy+uEPy)); + dist[nr3] = f4*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(uy+uEPy)); // q = 5 - dist[nr6] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uz+uEPz)); + dist[nr6] = f5*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(uz+uEPz)); // q = 6 - dist[nr5] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uz+uEPz)); + dist[nr5] = f6*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(uz+uEPz)); } } @@ -183,22 +183,22 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Veloci dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*Ci; // q = 1 - dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(ux+uEPx)); + dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(ux+uEPx)); // q=2 - dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(ux+uEPx)); + dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(ux+uEPx)); // q = 3 - dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uy+uEPy)); + dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(uy+uEPy)); // q = 4 - dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uy+uEPy)); + dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(uy+uEPy)); // q = 5 - dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.1111111111111111*(1.0+4.5*(uz+uEPz)); + dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0+4.5*(uz+uEPz)); // q = 6 - dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.1111111111111111*(1.0-4.5*(uz+uEPz)); + dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.1111111111111111*Ci*(1.0-4.5*(uz+uEPz)); } @@ -231,7 +231,7 @@ extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity Ci = Den[n+ion_component*Np]; CD = ChargeDensity[n]; CD_tmp = F*IonValence*Ci; - ChargeDensity[n] = CD*(IonValence>0) + CD_tmp; + ChargeDensity[n] = CD*(ion_component>0) + CD_tmp; } } diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 14e0a64c..fca2871c 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -36,7 +36,7 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke //---------------------- Default model parameters --------------------------// T = 300.0;//temperature; unit [K] Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] - k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice + k2_inv = 4.5;//speed of sound for D3Q7 lattice h = 1.0;//resolution; unit: um/lu tolerance = 1.0e-8; number_ion_species = 1; @@ -57,6 +57,8 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke } if (ion_db->keyExists( "temperature" )){ T = ion_db->getScalar( "temperature" ); + //re-calculate thermal voltage + Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] } if (ion_db->keyExists( "number_ion_species" )){ number_ion_species = ion_db->getScalar( "number_ion_species" ); @@ -104,6 +106,12 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke } } + //Read solid boundary condition specific to Ion model + BoundaryConditionSolid = 0; + if (ion_db->keyExists( "BC_Solid" )){ + BoundaryConditionSolid = ion_db->getScalar( "BC_Solid" ); + } + // Read domain parameters if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); @@ -112,10 +120,6 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } - BoundaryConditionSolid = 0; - if (domain_db->keyExists( "BC_Solid" )){ - BoundaryConditionSolid = domain_db->getScalar( "BC_Solid" ); - } if (rank==0) printf("*****************************************************\n"); if (rank==0) printf("LB Ion Transport Solver: \n"); @@ -128,13 +132,13 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke switch (BoundaryConditionSolid){ case 0: - if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned"); + if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned\n"); break; case 1: - if (rank==0) printf("LB Ion Solver: solid boundary: Neumann-type surfacen ion concentration is assigned"); + if (rank==0) printf("LB Ion Solver: solid boundary: Neumann-type surfacen ion concentration is assigned\n"); break; default: - if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned"); + if (rank==0) printf("LB Ion Solver: solid boundary: non-flux boundary is assigned\n"); break; } } @@ -375,6 +379,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // *************ODD TIMESTEP*************// timestep++; //Update ion concentration and charge density + if (rank==0) printf("timestep=%i; updating ion concentration and charge density\n",timestep); for (int ic=0; icSendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -386,7 +391,9 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } //LB-Ion collison + if (rank==0) printf("timestep=%i; execute collision step 1/2\n",timestep); for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); } @@ -394,7 +401,9 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // Set boundary conditions /* ... */ + if (rank==0) printf("timestep=%i; execute collision step 2/2\n",timestep); for (int ic=0; icLastExterior(), Np); } @@ -409,6 +418,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // *************EVEN TIMESTEP*************// timestep++; //Update ion concentration and charge density + if (rank==0) printf("timestep=%i; updating ion concentration and charge density\n",timestep); for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -419,6 +429,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } //LB-Ion collison + if (rank==0) printf("timestep=%i; execute collision step 1/2\n",timestep); for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -427,6 +438,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // Set boundary conditions /* ... */ + if (rank==0) printf("timestep=%i; execute collision step 2/2\n",timestep); for (int ic=0; icLastExterior(), Np); diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 27c0cda5..3cf8a6d2 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -22,7 +22,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ domain_db = db->getDatabase( "Domain" ); electric_db = db->getDatabase( "Poisson" ); - k2_inv = 4.5;//the inverse of 2nd-rank moment of D3Q7 lattice + k2_inv = 4.5;//speed of sound for D3Q7 lattice gamma = 0.3;//time step of LB-Poisson equation tau = 0.5+k2_inv*gamma; timestepMax = 100000; @@ -30,7 +30,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ h = 1.0;//resolution; unit: um/lu epsilon0 = 8.85e-12;//electrical permittivity of vaccum; unit:[C/(V*m)] epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] - epsilonR = 78.4;//default dielectric constant for water + epsilonR = 78.4;//default dielectric constant of water epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity analysis_interval = 1000; @@ -50,6 +50,13 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "epsilonR" )){ epsilonR = electric_db->getScalar( "epsilonR" ); } + + // Read solid boundary condition specific to Poisson equation + BoundaryConditionSolid = 1; + if (electric_db->keyExists( "BC_Solid" )){ + BoundaryConditionSolid = electric_db->getScalar( "BC_Solid" ); + } + // Read domain parameters if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); @@ -59,10 +66,6 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } - BoundaryConditionSolid = 1; - if (domain_db->keyExists( "BC_Solid" )){ - BoundaryConditionSolid = domain_db->getScalar( "BC_Solid" ); - } //Re-calcualte model parameters if user updates input epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] @@ -76,13 +79,13 @@ void ScaLBL_Poisson::ReadParams(string filename){ switch (BoundaryConditionSolid){ case 1: - if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned"); + if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned\n"); break; case 2: - if (rank==0) printf("LB-Poisson Solver: solid boundary: Neumann-type surfacen charge density is assigned"); + if (rank==0) printf("LB-Poisson Solver: solid boundary: Neumann-type surfacen charge density is assigned\n"); break; default: - if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned"); + if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned\n"); break; } } diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 46f28f45..1b733745 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -86,7 +86,7 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ // Re-calculate model parameters due to parameter read mu=(tau-0.5)/3.0; - time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] // convert user-input electric field ([V/m]) from physical unit to LB unit Ex = Ex*(h*1.0e-6);//LB electric field: V/lu Ey = Ey*(h*1.0e-6); diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index bdc15ef4..7cf835da 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -78,8 +78,13 @@ int main(int argc, char **argv) while (timestep < Study.timestepMax){ timestep++; + if (rank==0) printf("timestep=%i; running Poisson solver\n",timestep); PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental + + if (rank==0) printf("timestep=%i; running StokesModel\n",timestep); StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity + + if (rank==0) printf("timestep=%i; running Ion model\n",timestep); IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential From d8a1837ba0913ec6361e9d1b8892793cb1aa4462 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 19 Aug 2020 11:31:32 -0400 Subject: [PATCH 212/270] save the work; to be continued to clean up the code --- common/ScaLBL.cpp | 501 ---- common/ScaLBL.h | 164 +- cpu/GreyscaleFE.cpp | 3113 --------------------- cpu/GreyscaleSC.cpp | 3625 ------------------------ gpu/GreyscaleFE.cu | 3338 ---------------------- gpu/GreyscaleSC.cu | 3819 -------------------------- models/GreyscaleColorModel.cpp | 669 ++--- models/GreyscaleColorModel.h | 10 +- models/GreyscaleFEModel.cpp | 1247 --------- models/GreyscaleFEModel.h | 108 - models/GreyscaleModel.cpp | 13 +- models/GreyscaleSCModel.cpp | 1433 ---------- models/GreyscaleSCModel.h | 103 - tests/CMakeLists.txt | 2 - tests/lbpm_greyscaleFE_simulator.cpp | 59 - tests/lbpm_greyscaleSC_simulator.cpp | 58 - 16 files changed, 373 insertions(+), 17889 deletions(-) delete mode 100644 cpu/GreyscaleFE.cpp delete mode 100644 cpu/GreyscaleSC.cpp delete mode 100644 gpu/GreyscaleFE.cu delete mode 100644 gpu/GreyscaleSC.cu delete mode 100644 models/GreyscaleFEModel.cpp delete mode 100644 models/GreyscaleFEModel.h delete mode 100644 models/GreyscaleSCModel.cpp delete mode 100644 models/GreyscaleSCModel.h delete mode 100644 tests/lbpm_greyscaleFE_simulator.cpp delete mode 100644 tests/lbpm_greyscaleSC_simulator.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 4e9f95e7..13508dd4 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1094,368 +1094,6 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){ } -void ScaLBL_Communicator::BiSendD3Q19AA(double *Aq, double *Bq){ - - // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 - if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); - } - else{ - Lock=true; - } - // assign tag of 19 to D3Q19 communication - sendtag = recvtag = 38; - ScaLBL_DeviceBarrier(); - // Pack the distributions - //...Packing for x face(2,8,10,12,14)................................ - ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,0*sendCount_x,sendCount_x,sendbuf_x,Aq,N); - ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,1*sendCount_x,sendCount_x,sendbuf_x,Aq,N); - ScaLBL_D3Q19_Pack(10,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,Aq,N); - ScaLBL_D3Q19_Pack(12,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,Aq,N); - ScaLBL_D3Q19_Pack(14,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,Aq,N); - ScaLBL_D3Q19_Pack(2 ,dvcSendList_x,5*sendCount_x,sendCount_x,sendbuf_x,Bq,N); - ScaLBL_D3Q19_Pack(8 ,dvcSendList_x,6*sendCount_x,sendCount_x,sendbuf_x,Bq,N); - ScaLBL_D3Q19_Pack(10,dvcSendList_x,7*sendCount_x,sendCount_x,sendbuf_x,Bq,N); - ScaLBL_D3Q19_Pack(12,dvcSendList_x,8*sendCount_x,sendCount_x,sendbuf_x,Bq,N); - ScaLBL_D3Q19_Pack(14,dvcSendList_x,9*sendCount_x,sendCount_x,sendbuf_x,Bq,N); - - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, 10*sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, 10*recvCount_X,rank_X,recvtag); - - //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,0*sendCount_X,sendCount_X,sendbuf_X,Aq,N); - ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,1*sendCount_X,sendCount_X,sendbuf_X,Aq,N); - ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,Aq,N); - ScaLBL_D3Q19_Pack(11,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,Aq,N); - ScaLBL_D3Q19_Pack(13,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,Aq,N); - ScaLBL_D3Q19_Pack(1 ,dvcSendList_X,5*sendCount_X,sendCount_X,sendbuf_X,Bq,N); - ScaLBL_D3Q19_Pack(7 ,dvcSendList_X,6*sendCount_X,sendCount_X,sendbuf_X,Bq,N); - ScaLBL_D3Q19_Pack(9 ,dvcSendList_X,7*sendCount_X,sendCount_X,sendbuf_X,Bq,N); - ScaLBL_D3Q19_Pack(11,dvcSendList_X,8*sendCount_X,sendCount_X,sendbuf_X,Bq,N); - ScaLBL_D3Q19_Pack(13,dvcSendList_X,9*sendCount_X,sendCount_X,sendbuf_X,Bq,N); - - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, 10*sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, 10*recvCount_x,rank_x,recvtag); - - //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,0*sendCount_y,sendCount_y,sendbuf_y,Aq,N); - ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,1*sendCount_y,sendCount_y,sendbuf_y,Aq,N); - ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,Aq,N); - ScaLBL_D3Q19_Pack(16,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,Aq,N); - ScaLBL_D3Q19_Pack(18,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,Aq,N); - ScaLBL_D3Q19_Pack(4 ,dvcSendList_y,5*sendCount_y,sendCount_y,sendbuf_y,Bq,N); - ScaLBL_D3Q19_Pack(8 ,dvcSendList_y,6*sendCount_y,sendCount_y,sendbuf_y,Bq,N); - ScaLBL_D3Q19_Pack(9 ,dvcSendList_y,7*sendCount_y,sendCount_y,sendbuf_y,Bq,N); - ScaLBL_D3Q19_Pack(16,dvcSendList_y,8*sendCount_y,sendCount_y,sendbuf_y,Bq,N); - ScaLBL_D3Q19_Pack(18,dvcSendList_y,9*sendCount_y,sendCount_y,sendbuf_y,Bq,N); - - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, 10*sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, 10*recvCount_Y,rank_Y,recvtag); - - //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,0*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); - ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,1*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); - ScaLBL_D3Q19_Pack(10,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); - ScaLBL_D3Q19_Pack(15,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); - ScaLBL_D3Q19_Pack(17,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,Aq,N); - ScaLBL_D3Q19_Pack(3 ,dvcSendList_Y,5*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - ScaLBL_D3Q19_Pack(7 ,dvcSendList_Y,6*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - ScaLBL_D3Q19_Pack(10,dvcSendList_Y,7*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - ScaLBL_D3Q19_Pack(15,dvcSendList_Y,8*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - ScaLBL_D3Q19_Pack(17,dvcSendList_Y,9*sendCount_Y,sendCount_Y,sendbuf_Y,Bq,N); - - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, 10*sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, 10*recvCount_y,rank_y,recvtag); - - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,0*sendCount_z,sendCount_z,sendbuf_z,Aq,N); - ScaLBL_D3Q19_Pack(12,dvcSendList_z,1*sendCount_z,sendCount_z,sendbuf_z,Aq,N); - ScaLBL_D3Q19_Pack(13,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,Aq,N); - ScaLBL_D3Q19_Pack(16,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,Aq,N); - ScaLBL_D3Q19_Pack(17,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,Aq,N); - ScaLBL_D3Q19_Pack(6 ,dvcSendList_z,5*sendCount_z,sendCount_z,sendbuf_z,Bq,N); - ScaLBL_D3Q19_Pack(12,dvcSendList_z,6*sendCount_z,sendCount_z,sendbuf_z,Bq,N); - ScaLBL_D3Q19_Pack(13,dvcSendList_z,7*sendCount_z,sendCount_z,sendbuf_z,Bq,N); - ScaLBL_D3Q19_Pack(16,dvcSendList_z,8*sendCount_z,sendCount_z,sendbuf_z,Bq,N); - ScaLBL_D3Q19_Pack(17,dvcSendList_z,9*sendCount_z,sendCount_z,sendbuf_z,Bq,N); - - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, 10*sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, 10*recvCount_Z,rank_Z,recvtag); - - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,0*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); - ScaLBL_D3Q19_Pack(11,dvcSendList_Z,1*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); - ScaLBL_D3Q19_Pack(14,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); - ScaLBL_D3Q19_Pack(15,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); - ScaLBL_D3Q19_Pack(18,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,Aq,N); - ScaLBL_D3Q19_Pack(5 ,dvcSendList_Z,5*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); - ScaLBL_D3Q19_Pack(11,dvcSendList_Z,6*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); - ScaLBL_D3Q19_Pack(14,dvcSendList_Z,7*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); - ScaLBL_D3Q19_Pack(15,dvcSendList_Z,8*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); - ScaLBL_D3Q19_Pack(18,dvcSendList_Z,9*sendCount_Z,sendCount_Z,sendbuf_Z,Bq,N); - - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, 10*sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, 10*recvCount_z,rank_z,recvtag); - - //...Pack the xy edge (8)................................ - ScaLBL_D3Q19_Pack(8,dvcSendList_xy,0*sendCount_xy,sendCount_xy,sendbuf_xy,Aq,N); - ScaLBL_D3Q19_Pack(8,dvcSendList_xy,1*sendCount_xy,sendCount_xy,sendbuf_xy,Bq,N); - req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy, 2*sendCount_xy,rank_xy,sendtag); - req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY, 2*recvCount_XY,rank_XY,recvtag); - - //...Pack the Xy edge (9)................................ - ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,0*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Aq,N); - ScaLBL_D3Q19_Pack(9,dvcSendList_Xy,1*sendCount_Xy,sendCount_Xy,sendbuf_Xy,Bq,N); - req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy, 2*sendCount_Xy,rank_Xy,sendtag); - req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY, 2*recvCount_xY,rank_xY,recvtag); - - //...Pack the xY edge (10)................................ - ScaLBL_D3Q19_Pack(10,dvcSendList_xY,0*sendCount_xY,sendCount_xY,sendbuf_xY,Aq,N); - ScaLBL_D3Q19_Pack(10,dvcSendList_xY,1*sendCount_xY,sendCount_xY,sendbuf_xY,Bq,N); - req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY, 2*sendCount_xY,rank_xY,sendtag); - req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy, 2*recvCount_Xy,rank_Xy,recvtag); - - //...Pack the XY edge (7)................................ - ScaLBL_D3Q19_Pack(7,dvcSendList_XY,0*sendCount_XY,sendCount_XY,sendbuf_XY,Aq,N); - ScaLBL_D3Q19_Pack(7,dvcSendList_XY,1*sendCount_XY,sendCount_XY,sendbuf_XY,Bq,N); - req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY, 2*sendCount_XY,rank_XY,sendtag); - req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy, 2*recvCount_xy,rank_xy,recvtag); - - //...Pack the xz edge (12)................................ - ScaLBL_D3Q19_Pack(12,dvcSendList_xz,0*sendCount_xz,sendCount_xz,sendbuf_xz,Aq,N); - ScaLBL_D3Q19_Pack(12,dvcSendList_xz,1*sendCount_xz,sendCount_xz,sendbuf_xz,Bq,N); - req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz, 2*sendCount_xz,rank_xz,sendtag); - req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ, 2*recvCount_XZ,rank_XZ,recvtag); - - //...Pack the xZ edge (14)................................ - ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,0*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Aq,N); - ScaLBL_D3Q19_Pack(14,dvcSendList_xZ,1*sendCount_xZ,sendCount_xZ,sendbuf_xZ,Bq,N); - req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ, 2*sendCount_xZ,rank_xZ,sendtag); - req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz, 2*recvCount_Xz,rank_Xz,recvtag); - - //...Pack the Xz edge (13)................................ - ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,0*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Aq,N); - ScaLBL_D3Q19_Pack(13,dvcSendList_Xz,1*sendCount_Xz,sendCount_Xz,sendbuf_Xz,Bq,N); - req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz, 2*sendCount_Xz,rank_Xz,sendtag); - req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ, 2*recvCount_xZ,rank_xZ,recvtag); - - //...Pack the XZ edge (11)................................ - ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,0*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Aq,N); - ScaLBL_D3Q19_Pack(11,dvcSendList_XZ,1*sendCount_XZ,sendCount_XZ,sendbuf_XZ,Bq,N); - req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ, 2*sendCount_XZ,rank_XZ,sendtag); - req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz, 2*recvCount_xz,rank_xz,recvtag); - - //...Pack the yz edge (16)................................ - ScaLBL_D3Q19_Pack(16,dvcSendList_yz,0*sendCount_yz,sendCount_yz,sendbuf_yz,Aq,N); - ScaLBL_D3Q19_Pack(16,dvcSendList_yz,1*sendCount_yz,sendCount_yz,sendbuf_yz,Bq,N); - req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz, 2*sendCount_yz,rank_yz,sendtag); - req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ, 2*recvCount_YZ,rank_YZ,recvtag); - - //...Pack the yZ edge (18)................................ - ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,0*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Aq,N); - ScaLBL_D3Q19_Pack(18,dvcSendList_yZ,1*sendCount_yZ,sendCount_yZ,sendbuf_yZ,Bq,N); - req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ, 2*sendCount_yZ,rank_yZ,sendtag); - req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz, 2*recvCount_Yz,rank_Yz,recvtag); - - //...Pack the Yz edge (17)................................ - ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,0*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Aq,N); - ScaLBL_D3Q19_Pack(17,dvcSendList_Yz,1*sendCount_Yz,sendCount_Yz,sendbuf_Yz,Bq,N); - req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz, 2*sendCount_Yz,rank_Yz,sendtag); - req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ, 2*recvCount_yZ,rank_yZ,recvtag); - - //...Pack the YZ edge (15)................................ - ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,0*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Aq,N); - ScaLBL_D3Q19_Pack(15,dvcSendList_YZ,1*sendCount_YZ,sendCount_YZ,sendbuf_YZ,Bq,N); - req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ, 2*sendCount_YZ,rank_YZ,sendtag); - req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz, 2*recvCount_yz,rank_yz,recvtag); - //................................................................................... -} - -void ScaLBL_Communicator::BiRecvD3Q19AA(double *Aq, double *Bq){ - - // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 - //................................................................................... - // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(18,req1); - MPI_COMM_SCALBL.waitAll(18,req2); - ScaLBL_DeviceBarrier(); - - //................................................................................... - // NOTE: AA Routine writes to opposite - // Unpack the distributions on the device - //................................................................................... - //...Unpacking for x face(2,8,10,12,14)................................ - ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,recvbuf_x,Aq,N); - ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,recvbuf_x,Aq,N); - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,Aq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,Aq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,Aq,N); - ScaLBL_D3Q19_Unpack(2, dvcRecvDist_x,0*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); - ScaLBL_D3Q19_Unpack(8, dvcRecvDist_x,1*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_x,2*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_x,3*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_x,4*recvCount_x,recvCount_x,&recvbuf_x[5*recvCount_x],Bq,N); - //................................................................................... - //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,recvbuf_X,Aq,N); - ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,recvbuf_X,Aq,N); - ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,recvbuf_X,Aq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,recvbuf_X,Aq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,recvbuf_X,Aq,N); - ScaLBL_D3Q19_Unpack(1, dvcRecvDist_X,0*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); - ScaLBL_D3Q19_Unpack(7, dvcRecvDist_X,1*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); - ScaLBL_D3Q19_Unpack(9, dvcRecvDist_X,2*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_X,3*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_X,4*recvCount_X,recvCount_X,&recvbuf_X[5*recvCount_X],Bq,N); - //................................................................................... - //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,recvbuf_y,Aq,N); - ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,recvbuf_y,Aq,N); - ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,recvbuf_y,Aq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,recvbuf_y,Aq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,recvbuf_y,Aq,N); - ScaLBL_D3Q19_Unpack(4, dvcRecvDist_y,0*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); - ScaLBL_D3Q19_Unpack(8, dvcRecvDist_y,1*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); - ScaLBL_D3Q19_Unpack(9, dvcRecvDist_y,2*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_y,3*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_y,4*recvCount_y,recvCount_y,&recvbuf_y[5*recvCount_y],Bq,N); - //................................................................................... - //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); - ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,Aq,N); - ScaLBL_D3Q19_Unpack(3, dvcRecvDist_Y,0*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); - ScaLBL_D3Q19_Unpack(7, dvcRecvDist_Y,1*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,&recvbuf_Y[5*recvCount_Y],Bq,N); - //................................................................................... - - //...Pack the xy edge (8)................................ - ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,Aq,N); - ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,&recvbuf_xy[recvCount_xy],Bq,N); - - //...Pack the Xy edge (9)................................ - ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,recvbuf_Xy,Aq,N); - ScaLBL_D3Q19_Unpack(9,dvcRecvDist_Xy,0,recvCount_Xy,&recvbuf_Xy[recvCount_Xy],Bq,N); - - //...Pack the xY edge (10)................................ - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,Aq,N); - ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,&recvbuf_xY[recvCount_xY],Bq,N); - - //...Pack the XY edge (7)................................ - ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,Aq,N); - ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,&recvbuf_XY[recvCount_XY],Bq,N); - - if (BoundaryCondition > 0 && kproc == 0){ - // don't unpack little z - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - //...Pack the xZ edge (14)................................ - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N); - //...Pack the XZ edge (11)................................ - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N); - //...Pack the yZ edge (18)................................ - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N); - //...Pack the YZ edge (15)................................ - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N); - } - else if (BoundaryCondition > 0 && kproc == nprocz-1){ - // don't unpack big Z - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - //...Pack the xz edge (12)................................ - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N); - //...Pack the Xz edge (13)................................ - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N); - //...Pack the yz edge (16)................................ - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N); - //...Pack the Yz edge (17)................................ - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N); - } - else { - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,Aq,N); - ScaLBL_D3Q19_Unpack(6, dvcRecvDist_z,0*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,1*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,&recvbuf_z[5*recvCount_z],Bq,N); - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,Aq,N); - ScaLBL_D3Q19_Unpack(5, dvcRecvDist_Z,0*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,1*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,&recvbuf_Z[5*recvCount_Z],Bq,N); - //...Pack the xZ edge (14)................................ - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,Aq,N); - ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,&recvbuf_xZ[recvCount_xZ],Bq,N); - //...Pack the XZ edge (11)................................ - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,Aq,N); - ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,&recvbuf_XZ[recvCount_XZ],Bq,N); - //...Pack the yZ edge (18)................................ - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,Aq,N); - ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,&recvbuf_yZ[recvCount_yZ],Bq,N); - //...Pack the YZ edge (15)................................ - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,Aq,N); - ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,&recvbuf_YZ[recvCount_YZ],Bq,N); - //...Pack the xz edge (12)................................ - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,Aq,N); - ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,&recvbuf_xz[recvCount_xz],Bq,N); - //...Pack the Xz edge (13)................................ - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,Aq,N); - ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,&recvbuf_Xz[recvCount_Xz],Bq,N); - //...Pack the yz edge (16)................................ - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,Aq,N); - ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,&recvbuf_yz[recvCount_yz],Bq,N); - //...Pack the Yz edge (17)................................ - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,Aq,N); - ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,&recvbuf_Yz[recvCount_Yz],Bq,N); - } - - //................................................................................... - Lock=false; // unlock the communicator after communications complete - //................................................................................... - -} - void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ // Recieves halo and incorporates into D3Q19 based stencil gradient computation @@ -1539,106 +1177,6 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){ } -void ScaLBL_Communicator::SendD3Q7AA(double *Aq){ - - // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 - if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); - } - else{ - Lock=true; - } - // assign tag of 19 to D3Q19 communication - sendtag = recvtag = 7; - ScaLBL_DeviceBarrier(); - // Pack the distributions - //...Packing for x face(2,8,10,12,14)................................ - ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,Aq,N); - - req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag); - req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag); - - //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,Aq,N); - - req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag); - req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag); - - //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,Aq,N); - - req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag); - req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag); - - //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,Aq,N); - - req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag); - req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag); - - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,Aq,N); - - req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag); - req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag); - - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,Aq,N); - - req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag); - req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag); - //................................................................................... -} - -void ScaLBL_Communicator::RecvD3Q7AA(double *Aq){ - - // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 - //................................................................................... - // Wait for completion of D3Q19 communication - MPI_COMM_SCALBL.waitAll(6,req1); - MPI_COMM_SCALBL.waitAll(6,req2); - ScaLBL_DeviceBarrier(); - - //................................................................................... - // NOTE: AA Routine writes to opposite - // Unpack the distributions on the device - //................................................................................... - //...Unpacking for x face(2,8,10,12,14)................................ - ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,Aq,N); - //................................................................................... - //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,Aq,N); - //................................................................................... - //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,Aq,N); - //................................................................................... - //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N); - //................................................................................... - - if (BoundaryCondition > 0 && kproc == 0){ - // don't unpack little z - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); - } - else if (BoundaryCondition > 0 && kproc == nprocz-1){ - // don't unpack big z - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); - } - else { - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N); - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N); - } - - //................................................................................... - Lock=false; // unlock the communicator after communications complete - //................................................................................... - -} - void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 @@ -2171,42 +1709,3 @@ void ScaLBL_Communicator::PrintD3Q19(){ delete [] TempBuffer; } -void ScaLBL_Communicator::GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB) -{ - if (kproc == 0) { - // Set the density field on the z inlet - ScaLBL_GreyscaleSC_BC_z(dvcSendList_z, Map, DenA, DenB, vA, vB, sendCount_z); - } -} - -void ScaLBL_Communicator::GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB) -{ - if (kproc == nprocz-1){ - // Set the density field on the Z outlet - ScaLBL_GreyscaleSC_BC_Z(dvcSendList_Z, Map, DenA, DenB, vA, vB, sendCount_Z); - } -} - -void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time) -{ - if (kproc == 0) { - if (time%2==0){ - ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N); - } - else{ - ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(neighborList, dvcSendList_z, fqA, fqB, dinA, dinB, sendCount_z, N); - } - } -} - -void ScaLBL_Communicator::GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time) -{ - if (kproc == nprocz-1){ - if (time%2==0){ - ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N); - } - else{ - ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(neighborList, dvcSendList_Z, fqA, fqB, doutA, doutB, sendCount_Z, N); - } - } -} diff --git a/common/ScaLBL.h b/common/ScaLBL.h index f29cfa60..2e8eef1a 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -79,89 +79,89 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist double *Poros,double *Perm, double *Velocity,double Den,double *Pressure); // GREYSCALE FREE-ENERGY MODEL (Two-component) -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure); - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure); - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); - -extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); - -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, - double *Pressure, double rhoA,double rhoB, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, - double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den, +// double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, +// double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure); +// +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, +// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, +// double Gx, double Gy, double Gz, +// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap); +// +//extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np); +// +//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, +// double *Pressure, double rhoA,double rhoB, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, +// double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np); // GREYSCALE SHAN-CHEN MODEL (Two-component) -extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np); - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np); - -extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np); - -extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); - -extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); - -extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); - -extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); - -extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); - -extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); +//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, +// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, +// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, +// int start, int finish, int Np); +// +//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np); +// +//extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); +// +//extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count); +// +//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); +// +//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); +// +//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N); +// +//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N); // GREYSCALE COLOR MODEL (Two-component) //extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np); @@ -298,12 +298,8 @@ public: int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np); void SendD3Q19AA(double *dist); void RecvD3Q19AA(double *dist); - void BiSendD3Q19AA(double *Aq, double *Bq); - void BiRecvD3Q19AA(double *Aq, double *Bq); // void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); // void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd); - void SendD3Q7AA(double *Aq); - void RecvD3Q7AA(double *Aq); void BiSendD3Q7AA(double *Aq, double *Bq); void BiRecvD3Q7AA(double *Aq, double *Bq); void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq); diff --git a/cpu/GreyscaleFE.cpp b/cpu/GreyscaleFE.cpp deleted file mode 100644 index 1b9fd753..00000000 --- a/cpu/GreyscaleFE.cpp +++ /dev/null @@ -1,3113 +0,0 @@ -#include -#include - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Poros,double *Velocity, - double *Pressure, double rhoA,double rhoB, int N){ - - int n; - double ux,uy,uz,u_mag; - double pressure; - double porosity; - double rho0; - double phi; - double nA,nB; - - for (n=0; n0.0)-Gsc*nB*nA_gradx*int(phi<0.0); -// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); -// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - dist[1*Np+n] = fq; - - // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - dist[2*Np+n] = fq; - - // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[3*Np+n] = fq; - - // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[4*Np+n] = fq; - - // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[5*Np+n] = fq; - - // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[6*Np+n] = fq; - - // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - dist[7*Np+n] = fq; - - // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - dist[8*Np+n] = fq; - - // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - dist[9*Np+n] = fq; - - // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - dist[10*Np+n] = fq; - - // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - dist[11*Np+n] = fq; - - // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - dist[12*Np+n] = fq; - - // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - dist[13*Np+n] = fq; - - // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - dist[14*Np+n] = fq; - - // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - dist[15*Np+n] = fq; - - // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - dist[16*Np+n] = fq; - - // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - dist[17*Np+n] = fq; - - // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - dist[18*Np+n] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // Calculate the color gradient - nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); - ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); - nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - double ColorMag = C; - if (C==0.0) ColorMag=1.0; - nx = nx/ColorMag; - ny = ny/ColorMag; - nz = nz/ColorMag; - if (C == 0.0) nx = ny = nz = 0.0; - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - nAB = 1.0/(nA+nB); - Aq[n] = 0.3333333333333333*nA; - Bq[n] = 0.3333333333333333*nB; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nx; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; - - Aq[1*Np+n] = a1; - Bq[1*Np+n] = b1; - Aq[2*Np+n] = a2; - Bq[2*Np+n] = b2; - - //............................................... - // q = 2 - // Cq = {0,1,0} - delta = beta*nA*nB*nAB*0.1111111111111111*ny; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; - - Aq[3*Np+n] = a1; - Bq[3*Np+n] = b1; - Aq[4*Np+n] = a2; - Bq[4*Np+n] = b2; - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nz; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; - - Aq[5*Np+n] = a1; - Bq[5*Np+n] = b1; - Aq[6*Np+n] = a2; - Bq[6*Np+n] = b2; - //............................................... - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure){ - - int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho0; - double phi; - double nx,ny,nz,C; - double nA,nB; - double a1,b1,a2,b2,nAB,delta; - double beta=0.95; - double nA_gradx,nA_grady,nA_gradz; - double nB_gradx,nB_grady,nB_gradz; - double Gff_x,Gff_y,Gff_z; - double Gfs_x,Gfs_y,Gfs_z; - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - for (n=start; n 10Np => odd part of dist) - fq = dist[nr1]; // reading the f1 data into register fq - pressure = fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // q=2 - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = dist[nr2]; // reading the f2 data into register fq - pressure += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - fq = dist[nr3]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - fq = dist[nr4]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - nr5 = neighborList[n+4*Np]; - fq = dist[nr5]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - nr6 = neighborList[n+5*Np]; - fq = dist[nr6]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// - // fluid-fluid force -// Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); -// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); -// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - //nread = neighborList[n+Np]; - dist[nr2] = fq; - - // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - //nread = neighborList[n]; - dist[nr1] = fq; - - // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+3*Np]; - dist[nr4] = fq; - - // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+2*Np]; - dist[nr3] = fq; - - // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+5*Np]; - dist[nr6] = fq; - - // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+4*Np]; - dist[nr5] = fq; - - // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - nread = neighborList[n+7*Np]; - dist[nread] = fq; - - // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - nread = neighborList[n+6*Np]; - dist[nread] = fq; - - // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - nread = neighborList[n+9*Np]; - dist[nread] = fq; - - // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - nread = neighborList[n+8*Np]; - dist[nread] = fq; - - // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - nread = neighborList[n+11*Np]; - dist[nread] = fq; - - // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - nread = neighborList[n+10*Np]; - dist[nread]= fq; - - // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - nread = neighborList[n+13*Np]; - dist[nread] = fq; - - // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - nread = neighborList[n+12*Np]; - dist[nread] = fq; - - // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - nread = neighborList[n+15*Np]; - dist[nread] = fq; - - // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - nread = neighborList[n+14*Np]; - dist[nread] = fq; - - // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - nread = neighborList[n+17*Np]; - dist[nread] = fq; - - // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - nread = neighborList[n+16*Np]; - dist[nread] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // Calculate the color gradient - nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); - ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); - nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - double ColorMag = C; - if (C==0.0) ColorMag=1.0; - nx = nx/ColorMag; - ny = ny/ColorMag; - nz = nz/ColorMag; - if (C == 0.0) nx = ny = nz = 0.0; - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - nAB = 1.0/(nA+nB); - Aq[n] = 0.3333333333333333*nA; - Bq[n] = 0.3333333333333333*nB; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nx; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; - - // q = 1 - //nread = neighborList[n+Np]; - Aq[nr2] = a1; - Bq[nr2] = b1; - // q=2 - //nread = neighborList[n]; - Aq[nr1] = a2; - Bq[nr1] = b2; - - //............................................... - // Cq = {0,1,0} - delta = beta*nA*nB*nAB*0.1111111111111111*ny; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; - - // q = 3 - //nread = neighborList[n+3*Np]; - Aq[nr4] = a1; - Bq[nr4] = b1; - // q = 4 - //nread = neighborList[n+2*Np]; - Aq[nr3] = a2; - Bq[nr3] = b2; - - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nz; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; - - // q = 5 - //nread = neighborList[n+5*Np]; - Aq[nr6] = a1; - Bq[nr6] = b1; - // q = 6 - //nread = neighborList[n+4*Np]; - Aq[nr5] = a2; - Bq[nr5] = b2; - //............................................... - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - - int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho,rho0; - double phi; - double phi_lap;//laplacian of phase field - double nA,nB; - double Gfs_x,Gfs_y,Gfs_z; - double Gff_x,Gff_y,Gff_z; - double chem; - double rlx_phi; - double a1,a2;//PDF of phase field - // *---------------------------------Pressure Tensor Gradient------------------------------------*// - double Pxx_x,Pyy_y,Pzz_z; - double Pxy_x,Pxy_y; - double Pyz_y,Pyz_z; - double Pxz_x,Pxz_z; - double px,py,pz; //pressure gradient - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - for (n=start; n0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - //Load pressure gradient - px=PressureGrad[0*Np+n]; - py=PressureGrad[1*Np+n]; - pz=PressureGrad[2*Np+n]; - - //Load pressure tensor gradient - //For reference full list of PressTensorGrad - //PressTensorGrad[n+0*Np] = Pxx_x - //PressTensorGrad[n+1*Np] = Pxx_y - //PressTensorGrad[n+2*Np] = Pxx_z - //PressTensorGrad[n+3*Np] = Pyy_x - //PressTensorGrad[n+4*Np] = Pyy_y - //PressTensorGrad[n+5*Np] = Pyy_z - //PressTensorGrad[n+6*Np] = Pzz_x - //PressTensorGrad[n+7*Np] = Pzz_y - //PressTensorGrad[n+8*Np] = Pzz_z - //PressTensorGrad[n+9*Np] = Pxy_x - //PressTensorGrad[n+10*Np] = Pxy_y - //PressTensorGrad[n+11*Np] = Pxy_z - //PressTensorGrad[n+12*Np] = Pyz_x - //PressTensorGrad[n+13*Np] = Pyz_y - //PressTensorGrad[n+14*Np] = Pyz_z - //PressTensorGrad[n+15*Np] = Pxz_x - //PressTensorGrad[n+16*Np] = Pxz_y - //PressTensorGrad[n+17*Np] = Pxz_z - Pxx_x = PressTensorGrad[0*Np+n]; - Pyy_y = PressTensorGrad[4*Np+n]; - Pzz_z = PressTensorGrad[8*Np+n]; - Pxy_x = PressTensorGrad[9*Np+n]; - Pxz_x = PressTensorGrad[15*Np+n]; - Pxy_y = PressTensorGrad[10*Np+n]; - Pyz_y = PressTensorGrad[13*Np+n]; - Pyz_z = PressTensorGrad[14*Np+n]; - Pxz_z = PressTensorGrad[17*Np+n]; - //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... - //TODO double check if you need porosity as a fre-factor - Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); - Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); - Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - // local density - rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - // local relaxation time - tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); - rlx_setA = 1.f/tau; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); - mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity - - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = dist[n]; - rho = fq; - m1 = -30.0*fq; - m2 = 12.0*fq; - - // q=1 - nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist) - fq = dist[nr1]; // reading the f1 data into register fq - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // q=2 - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = dist[nr2]; // reading the f2 data into register fq - rho += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - fq = dist[nr3]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - fq = dist[nr4]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - nr5 = neighborList[n+4*Np]; - fq = dist[nr5]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - nr6 = neighborList[n+5*Np]; - fq = dist[nr6]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - pressure=rho/3.0; - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - //nread = neighborList[n+Np]; - dist[nr2] = fq; - - // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - //nread = neighborList[n]; - dist[nr1] = fq; - - // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+3*Np]; - dist[nr4] = fq; - - // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+2*Np]; - dist[nr3] = fq; - - // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+5*Np]; - dist[nr6] = fq; - - // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+4*Np]; - dist[nr5] = fq; - - // q = 7 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - nread = neighborList[n+7*Np]; - dist[nread] = fq; - - // q = 8 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - nread = neighborList[n+6*Np]; - dist[nread] = fq; - - // q = 9 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - nread = neighborList[n+9*Np]; - dist[nread] = fq; - - // q = 10 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - nread = neighborList[n+8*Np]; - dist[nread] = fq; - - // q = 11 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - nread = neighborList[n+11*Np]; - dist[nread] = fq; - - // q = 12 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - nread = neighborList[n+10*Np]; - dist[nread]= fq; - - // q = 13 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - nread = neighborList[n+13*Np]; - dist[nread] = fq; - - // q= 14 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - nread = neighborList[n+12*Np]; - dist[nread] = fq; - - // q = 15 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - nread = neighborList[n+15*Np]; - dist[nread] = fq; - - // q = 16 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - nread = neighborList[n+14*Np]; - dist[nread] = fq; - - // q = 17 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - nread = neighborList[n+17*Np]; - dist[nread] = fq; - - // q = 18 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - nread = neighborList[n+16*Np]; - dist[nread] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // calcuale chemical potential - chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; - //rlx_phi = 3.f-sqrt(3.f); - rlx_phi = 1.0; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - //a1 = Cq[nr2]; - //a2 = Cq[nr1]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); - - // q = 1 - //nread = neighborList[n+Np]; - Cq[nr2] = a1; - // q=2 - //nread = neighborList[n]; - Cq[nr1] = a2; - - //............................................... - // Cq = {0,1,0} - //a1 = Cq[nr4]; - //a2 = Cq[nr3]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); - - // q = 3 - //nread = neighborList[n+3*Np]; - Cq[nr4] = a1; - // q = 4 - //nread = neighborList[n+2*Np]; - Cq[nr3] = a2; - - //............................................... - // q = 4 - // Cq = {0,0,1} - //a1 = Cq[nr6]; - //a2 = Cq[nr5]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); - - // q = 5 - //nread = neighborList[n+5*Np]; - Cq[nr6] = a1; - // q = 6 - //nread = neighborList[n+4*Np]; - Cq[nr5] = a2; - //............................................... - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - //a1=Cq[n]; - //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); - Cq[n] = phi-3.0*gamma*chem; - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - - int n; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho,rho0; - double phi; - double phi_lap;//laplacian of phase field - double nA,nB; - double Gfs_x,Gfs_y,Gfs_z; - double Gff_x,Gff_y,Gff_z; - double chem; - double rlx_phi; - double a1,a2;//PDF of phase field - // *---------------------------------Pressure Tensor Gradient------------------------------------*// - double Pxx_x,Pyy_y,Pzz_z; - double Pxy_x,Pxy_y; - double Pyz_y,Pyz_z; - double Pxz_x,Pxz_z; - double px,py,pz; //pressure gradient - - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - - for (n=start; n0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - //Load pressure gradient - px=PressureGrad[0*Np+n]; - py=PressureGrad[1*Np+n]; - pz=PressureGrad[2*Np+n]; - - //Load pressure tensor gradient - //For reference full list of PressTensorGrad - //PressTensorGrad[n+0*Np] = Pxx_x - //PressTensorGrad[n+1*Np] = Pxx_y - //PressTensorGrad[n+2*Np] = Pxx_z - //PressTensorGrad[n+3*Np] = Pyy_x - //PressTensorGrad[n+4*Np] = Pyy_y - //PressTensorGrad[n+5*Np] = Pyy_z - //PressTensorGrad[n+6*Np] = Pzz_x - //PressTensorGrad[n+7*Np] = Pzz_y - //PressTensorGrad[n+8*Np] = Pzz_z - //PressTensorGrad[n+9*Np] = Pxy_x - //PressTensorGrad[n+10*Np] = Pxy_y - //PressTensorGrad[n+11*Np] = Pxy_z - //PressTensorGrad[n+12*Np] = Pyz_x - //PressTensorGrad[n+13*Np] = Pyz_y - //PressTensorGrad[n+14*Np] = Pyz_z - //PressTensorGrad[n+15*Np] = Pxz_x - //PressTensorGrad[n+16*Np] = Pxz_y - //PressTensorGrad[n+17*Np] = Pxz_z - Pxx_x = PressTensorGrad[0*Np+n]; - Pyy_y = PressTensorGrad[4*Np+n]; - Pzz_z = PressTensorGrad[8*Np+n]; - Pxy_x = PressTensorGrad[9*Np+n]; - Pxz_x = PressTensorGrad[15*Np+n]; - Pxy_y = PressTensorGrad[10*Np+n]; - Pyz_y = PressTensorGrad[13*Np+n]; - Pyz_z = PressTensorGrad[14*Np+n]; - Pxz_z = PressTensorGrad[17*Np+n]; - //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... - //TODO double check if you need porosity as a fre-factor - Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); - Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); - Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - // local density - rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - // local relaxation time - tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); - rlx_setA = 1.f/tau; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); - mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity - - - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = dist[n]; - rho = fq; - m1 = -30.0*fq; - m2 = 12.0*fq; - - // q=1 - fq = dist[2*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // f2 = dist[10*Np+n]; - fq = dist[1*Np+n]; - rho += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - fq = dist[4*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - fq = dist[3*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - fq = dist[6*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - fq = dist[5*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - fq = dist[8*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - fq = dist[7*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - fq = dist[10*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - fq = dist[9*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - fq = dist[12*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - fq = dist[11*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - fq = dist[14*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - fq = dist[13*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - fq = dist[16*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - fq = dist[15*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - fq = dist[18*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - fq = dist[17*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - pressure=rho/3.0; - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - dist[1*Np+n] = fq; - - // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - dist[2*Np+n] = fq; - - // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[3*Np+n] = fq; - - // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[4*Np+n] = fq; - - // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[5*Np+n] = fq; - - // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[6*Np+n] = fq; - - // q = 7 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - dist[7*Np+n] = fq; - - // q = 8 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - dist[8*Np+n] = fq; - - // q = 9 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - dist[9*Np+n] = fq; - - // q = 10 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - dist[10*Np+n] = fq; - - // q = 11 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - dist[11*Np+n] = fq; - - // q = 12 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - dist[12*Np+n] = fq; - - // q = 13 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - dist[13*Np+n] = fq; - - // q= 14 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - dist[14*Np+n] = fq; - - // q = 15 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - dist[15*Np+n] = fq; - - // q = 16 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - dist[16*Np+n] = fq; - - // q = 17 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - dist[17*Np+n] = fq; - - // q = 18 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - dist[18*Np+n] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // calcuale chemical potential - chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; - //rlx_phi = 3.f-sqrt(3.f); - rlx_phi = 1.0; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - //a1 = Cq[1*Np+n]; - //a2 = Cq[2*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); - - Cq[1*Np+n] = a1; - Cq[2*Np+n] = a2; - - //............................................... - // q = 2 - // Cq = {0,1,0} - //a1 = Cq[3*Np+n]; - //a2 = Cq[4*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); - - Cq[3*Np+n] = a1; - Cq[4*Np+n] = a2; - //............................................... - // q = 4 - // Cq = {0,0,1} - //a1 = Cq[5*Np+n]; - //a2 = Cq[6*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); - - Cq[5*Np+n] = a1; - Cq[6*Np+n] = a2; - //............................................... - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - //a1=Cq[n]; - //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); - Cq[n] = phi-3.0*gamma*chem; - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ - int n; - double phi; - double nA,nB; - double Den0; - for (n=0; n -#include - -//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList,int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, -// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, -// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, -// int start, int finish, int Np){ -// -// int ijk; -// int n, nread; -// double vx,vy,vz,v_mag; -// double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; -// double ux,uy,uz; -// // conserved momemnts -// double jxA,jyA,jzA; -// double jxB,jyB,jzB; -// double rhoA,rhoB; -// double nA,nB; -// // non-conserved moments -// double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; -// double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; -// double fq; -// //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; -// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) -// double porosity; -// double perm;//voxel permeability -// double permA,permB;//effective relative perm -// double c0, c1; //Guo's model parameters -// double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity -// double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity -// double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) -// double FxB, FyB, FzB; -// double rlx_setA,rlx_setB; -// double nA_gradx,nA_grady,nA_gradz; -// double nB_gradx,nB_grady,nB_gradz; -// double GffA_x,GffA_y,GffA_z; -// double GfsA_x,GfsA_y,GfsA_z; -// double GffB_x,GffB_y,GffB_z; -// double GfsB_x,GfsB_y,GfsB_z; -// -// const double mrt_V1=0.05263157894736842; -// const double mrt_V2=0.012531328320802; -// const double mrt_V3=0.04761904761904762; -// const double mrt_V4=0.004594820384294068; -// const double mrt_V5=0.01587301587301587; -// const double mrt_V6=0.0555555555555555555555555; -// const double mrt_V7=0.02777777777777778; -// const double mrt_V8=0.08333333333333333; -// const double mrt_V9=0.003341687552213868; -// const double mrt_V10=0.003968253968253968; -// const double mrt_V11=0.01388888888888889; -// const double mrt_V12=0.04166666666666666; -// -// int S = Np/NBLOCKS/NTHREADS + 1; -// for (int s=0; s even part of dist) -// fq = distA[nread]; // reading the f2 data into register fq -// rhoA += fq; -// m1A -= 11.0*(fq); -// m2A -= 4.0*(fq); -// jxA -= fq; -// m4A += 4.0*(fq); -// m9A += 2.0*(fq); -// m10A -= 4.0*(fq); -// -// // q=3 -// nread = neighborList[n+2*Np]; // neighbor 4 -// fq = distA[nread]; -// rhoA += fq; -// m1A -= 11.0*fq; -// m2A -= 4.0*fq; -// jyA = fq; -// m6A = -4.0*fq; -// m9A -= fq; -// m10A += 2.0*fq; -// m11A = fq; -// m12A = -2.0*fq; -// -// // q = 4 -// nread = neighborList[n+3*Np]; // neighbor 3 -// fq = distA[nread]; -// rhoA += fq; -// m1A -= 11.0*fq; -// m2A -= 4.0*fq; -// jyA -= fq; -// m6A += 4.0*fq; -// m9A -= fq; -// m10A += 2.0*fq; -// m11A += fq; -// m12A -= 2.0*fq; -// -// // q=5 -// nread = neighborList[n+4*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A -= 11.0*fq; -// m2A -= 4.0*fq; -// jzA = fq; -// m8A = -4.0*fq; -// m9A -= fq; -// m10A += 2.0*fq; -// m11A -= fq; -// m12A += 2.0*fq; -// -// -// // q = 6 -// nread = neighborList[n+5*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A -= 11.0*fq; -// m2A -= 4.0*fq; -// jzA -= fq; -// m8A += 4.0*fq; -// m9A -= fq; -// m10A += 2.0*fq; -// m11A -= fq; -// m12A += 2.0*fq; -// -// // q=7 -// nread = neighborList[n+6*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA += fq; -// m4A += fq; -// jyA += fq; -// m6A += fq; -// m9A += fq; -// m10A += fq; -// m11A += fq; -// m12A += fq; -// m13A = fq; -// m16A = fq; -// m17A = -fq; -// -// // q = 8 -// nread = neighborList[n+7*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA -= fq; -// m4A -= fq; -// jyA -= fq; -// m6A -= fq; -// m9A += fq; -// m10A += fq; -// m11A += fq; -// m12A += fq; -// m13A += fq; -// m16A -= fq; -// m17A += fq; -// -// // q=9 -// nread = neighborList[n+8*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA += fq; -// m4A += fq; -// jyA -= fq; -// m6A -= fq; -// m9A += fq; -// m10A += fq; -// m11A += fq; -// m12A += fq; -// m13A -= fq; -// m16A += fq; -// m17A += fq; -// -// // q = 10 -// nread = neighborList[n+9*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA -= fq; -// m4A -= fq; -// jyA += fq; -// m6A += fq; -// m9A += fq; -// m10A += fq; -// m11A += fq; -// m12A += fq; -// m13A -= fq; -// m16A -= fq; -// m17A -= fq; -// -// // q=11 -// nread = neighborList[n+10*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA += fq; -// m4A += fq; -// jzA += fq; -// m8A += fq; -// m9A += fq; -// m10A += fq; -// m11A -= fq; -// m12A -= fq; -// m15A = fq; -// m16A -= fq; -// m18A = fq; -// -// // q=12 -// nread = neighborList[n+11*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA -= fq; -// m4A -= fq; -// jzA -= fq; -// m8A -= fq; -// m9A += fq; -// m10A += fq; -// m11A -= fq; -// m12A -= fq; -// m15A += fq; -// m16A += fq; -// m18A -= fq; -// -// // q=13 -// nread = neighborList[n+12*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA += fq; -// m4A += fq; -// jzA -= fq; -// m8A -= fq; -// m9A += fq; -// m10A += fq; -// m11A -= fq; -// m12A -= fq; -// m15A -= fq; -// m16A -= fq; -// m18A -= fq; -// -// // q=14 -// nread = neighborList[n+13*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jxA -= fq; -// m4A -= fq; -// jzA += fq; -// m8A += fq; -// m9A += fq; -// m10A += fq; -// m11A -= fq; -// m12A -= fq; -// m15A -= fq; -// m16A += fq; -// m18A += fq; -// -// // q=15 -// nread = neighborList[n+14*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jyA += fq; -// m6A += fq; -// jzA += fq; -// m8A += fq; -// m9A -= 2.0*fq; -// m10A -= 2.0*fq; -// m14A = fq; -// m17A += fq; -// m18A -= fq; -// -// // q=16 -// nread = neighborList[n+15*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jyA -= fq; -// m6A -= fq; -// jzA -= fq; -// m8A -= fq; -// m9A -= 2.0*fq; -// m10A -= 2.0*fq; -// m14A += fq; -// m17A -= fq; -// m18A += fq; -// -// // q=17 -// nread = neighborList[n+16*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jyA += fq; -// m6A += fq; -// jzA -= fq; -// m8A -= fq; -// m9A -= 2.0*fq; -// m10A -= 2.0*fq; -// m14A -= fq; -// m17A += fq; -// m18A += fq; -// -// // q=18 -// nread = neighborList[n+17*Np]; -// fq = distA[nread]; -// rhoA += fq; -// m1A += 8.0*fq; -// m2A += fq; -// jyA -= fq; -// m6A -= fq; -// jzA += fq; -// m8A += fq; -// m9A -= 2.0*fq; -// m10A -= 2.0*fq; -// m14A -= fq; -// m17A -= fq; -// m18A -= fq; -// //---------------------------------------------------------------------// -// -// // ------------------- Fluid component B ---------------------------------// -// //........................................................................ -// // READ THE DISTRIBUTIONS -// // (read from opposite array due to previous swap operation) -// //........................................................................ -// // q=0 -// fq = distB[n]; -// rhoB = fq; -// m1B = -30.0*fq; -// m2B = 12.0*fq; -// -// // q=1 -// nread = neighborList[n]; // neighbor 2 -// fq = distB[nread]; // reading the f1 data into register fq -// rhoB += fq; -// m1B -= 11.0*fq; -// m2B -= 4.0*fq; -// jxB = fq; -// m4B = -4.0*fq; -// m9B = 2.0*fq; -// m10B = -4.0*fq; -// -// // q=2 -// nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) -// fq = distB[nread]; // reading the f2 data into register fq -// rhoB += fq; -// m1B -= 11.0*(fq); -// m2B -= 4.0*(fq); -// jxB -= fq; -// m4B += 4.0*(fq); -// m9B += 2.0*(fq); -// m10B -= 4.0*(fq); -// -// // q=3 -// nread = neighborList[n+2*Np]; // neighbor 4 -// fq = distB[nread]; -// rhoB += fq; -// m1B -= 11.0*fq; -// m2B -= 4.0*fq; -// jyB = fq; -// m6B = -4.0*fq; -// m9B -= fq; -// m10B += 2.0*fq; -// m11B = fq; -// m12B = -2.0*fq; -// -// // q = 4 -// nread = neighborList[n+3*Np]; // neighbor 3 -// fq = distB[nread]; -// rhoB += fq; -// m1B -= 11.0*fq; -// m2B -= 4.0*fq; -// jyB -= fq; -// m6B += 4.0*fq; -// m9B -= fq; -// m10B += 2.0*fq; -// m11B += fq; -// m12B -= 2.0*fq; -// -// // q=5 -// nread = neighborList[n+4*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B -= 11.0*fq; -// m2B -= 4.0*fq; -// jzB = fq; -// m8B = -4.0*fq; -// m9B -= fq; -// m10B += 2.0*fq; -// m11B -= fq; -// m12B += 2.0*fq; -// -// -// // q = 6 -// nread = neighborList[n+5*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B -= 11.0*fq; -// m2B -= 4.0*fq; -// jzB -= fq; -// m8B += 4.0*fq; -// m9B -= fq; -// m10B += 2.0*fq; -// m11B -= fq; -// m12B += 2.0*fq; -// -// // q=7 -// nread = neighborList[n+6*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB += fq; -// m4B += fq; -// jyB += fq; -// m6B += fq; -// m9B += fq; -// m10B += fq; -// m11B += fq; -// m12B += fq; -// m13B = fq; -// m16B = fq; -// m17B = -fq; -// -// // q = 8 -// nread = neighborList[n+7*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB -= fq; -// m4B -= fq; -// jyB -= fq; -// m6B -= fq; -// m9B += fq; -// m10B += fq; -// m11B += fq; -// m12B += fq; -// m13B += fq; -// m16B -= fq; -// m17B += fq; -// -// // q=9 -// nread = neighborList[n+8*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB += fq; -// m4B += fq; -// jyB -= fq; -// m6B -= fq; -// m9B += fq; -// m10B += fq; -// m11B += fq; -// m12B += fq; -// m13B -= fq; -// m16B += fq; -// m17B += fq; -// -// // q = 10 -// nread = neighborList[n+9*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB -= fq; -// m4B -= fq; -// jyB += fq; -// m6B += fq; -// m9B += fq; -// m10B += fq; -// m11B += fq; -// m12B += fq; -// m13B -= fq; -// m16B -= fq; -// m17B -= fq; -// -// // q=11 -// nread = neighborList[n+10*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB += fq; -// m4B += fq; -// jzB += fq; -// m8B += fq; -// m9B += fq; -// m10B += fq; -// m11B -= fq; -// m12B -= fq; -// m15B = fq; -// m16B -= fq; -// m18B = fq; -// -// // q=12 -// nread = neighborList[n+11*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB -= fq; -// m4B -= fq; -// jzB -= fq; -// m8B -= fq; -// m9B += fq; -// m10B += fq; -// m11B -= fq; -// m12B -= fq; -// m15B += fq; -// m16B += fq; -// m18B -= fq; -// -// // q=13 -// nread = neighborList[n+12*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB += fq; -// m4B += fq; -// jzB -= fq; -// m8B -= fq; -// m9B += fq; -// m10B += fq; -// m11B -= fq; -// m12B -= fq; -// m15B -= fq; -// m16B -= fq; -// m18B -= fq; -// -// // q=14 -// nread = neighborList[n+13*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jxB -= fq; -// m4B -= fq; -// jzB += fq; -// m8B += fq; -// m9B += fq; -// m10B += fq; -// m11B -= fq; -// m12B -= fq; -// m15B -= fq; -// m16B += fq; -// m18B += fq; -// -// // q=15 -// nread = neighborList[n+14*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jyB += fq; -// m6B += fq; -// jzB += fq; -// m8B += fq; -// m9B -= 2.0*fq; -// m10B -= 2.0*fq; -// m14B = fq; -// m17B += fq; -// m18B -= fq; -// -// // q=16 -// nread = neighborList[n+15*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jyB -= fq; -// m6B -= fq; -// jzB -= fq; -// m8B -= fq; -// m9B -= 2.0*fq; -// m10B -= 2.0*fq; -// m14B += fq; -// m17B -= fq; -// m18B += fq; -// -// // q=17 -// nread = neighborList[n+16*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jyB += fq; -// m6B += fq; -// jzB -= fq; -// m8B -= fq; -// m9B -= 2.0*fq; -// m10B -= 2.0*fq; -// m14B -= fq; -// m17B += fq; -// m18B += fq; -// -// // q=18 -// nread = neighborList[n+17*Np]; -// fq = distB[nread]; -// rhoB += fq; -// m1B += 8.0*fq; -// m2B += fq; -// jyB -= fq; -// m6B -= fq; -// jzB += fq; -// m8B += fq; -// m9B -= 2.0*fq; -// m10B -= 2.0*fq; -// m14B -= fq; -// m17B -= fq; -// m18B -= fq; -// //---------------------------------------------------------------------// -// -// -// // Compute SC fluid-fluid interaction force -// GffA_x = -Gsc*nB_gradx; -// GffA_y = -Gsc*nB_grady; -// GffA_z = -Gsc*nB_gradz; -// GffB_x = -Gsc*nA_gradx; -// GffB_y = -Gsc*nA_grady; -// GffB_z = -Gsc*nA_gradz; -// // Compute SC fluid-solid force -// GfsA_x = SolidForceA[n+0*Np]; -// GfsA_y = SolidForceA[n+1*Np]; -// GfsA_z = SolidForceA[n+2*Np]; -// GfsB_x = SolidForceB[n+0*Np]; -// GfsB_y = SolidForceB[n+1*Np]; -// GfsB_z = SolidForceB[n+2*Np]; -// -// // Compute greyscale related parameters -// // ------------------- Fluid Component A -----------------------// -// c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); -// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes -// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); -// c1 = porosity*0.5*GeoFun/sqrt(permA); -// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes -// -// vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); -// vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); -// vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); -// v_mag=sqrt(vx*vx+vy*vy+vz*vz); -// ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); -// uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); -// uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); -// u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); -// -// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium -// FxA = nA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); -// FyA = nA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); -// FzA = nA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); -// if (porosity==1.0){ -// FxA=nA*(Gx + GffA_x + GfsA_x); -// FyA=nA*(Gy + GffA_y + GfsA_y); -// FzA=nA*(Gz + GffA_z + GfsA_z); -// } -// // ------------------- Fluid Component B -----------------------// -// // Compute greyscale related parameters -// c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); -// if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes -// //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); -// c1 = porosity*0.5*GeoFun/sqrt(permB); -// if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes -// -// vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); -// vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); -// vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); -// v_mag=sqrt(vx*vx+vy*vy+vz*vz); -// ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); -// uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); -// uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); -// u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); -// -// //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium -// FxB = nB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); -// FyB = nB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); -// FzB = nB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); -// if (porosity==1.0){ -// FxB=nB*(Gx + GffB_x + GfsB_x); -// FyB=nB*(Gy + GffB_y + GfsB_y); -// FzB=nB*(Gz + GffB_z + GfsB_z); -// } -// -// // Calculate barycentric velocity of the fluid mixture -// ux = (nA*ux_A+nB*ux_B)/(nA+nB); -// uy = (nA*uy_A+nB*uy_B)/(nA+nB); -// uz = (nA*uz_A+nB*uz_B)/(nA+nB); -// -// // ------------------- Fluid Component A -----------------------// -// rlx_setA = 1.0/tauA; -// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); -// //-------------------- MRT collison where body force has NO higher-order terms -------------// -// //..............carry out relaxation process............................................... -// //TODO need to incoporate porosity -// m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA) - m1A) -// + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); -// m2A = m2A + rlx_setA*((3*rhoA - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) -// + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); -// jxA = jxA + FxA; -// m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); -// jyA = jyA + FyA; -// m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA)- m6A) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); -// jzA = jzA + FzA; -// m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA)- m8A) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); -// m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) -// + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); -// //m10A = m10A + rlx_setA*( - m10A) -// // + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); -// m10A = m10A + rlx_setA*( -0.5*(rhoA*(2*ux*ux-uy*uy-uz*uz))- m10A) -// + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); -// m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) -// + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); -// //m12A = m12A + rlx_setA*( - m12A) -// // + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); -// m12A = m12A + rlx_setA*( -0.5*(rhoA*(uy*uy-uz*uz))- m12A) -// + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); -// m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) -// + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); -// m14A = m14A + rlx_setA*( rhoA*(uy*uz) - m14A) -// + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); -// m15A = m15A + rlx_setA*( rhoA*(ux*uz) - m15A) -// + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); -// m16A = m16A + rlx_setB*( - m16A); -// m17A = m17A + rlx_setB*( - m17A); -// m18A = m18A + rlx_setB*( - m18A); -// //....................................................................................................... -// -// -// // ------------------- Fluid Component A -----------------------// -// //.................inverse transformation...................................................... -// // q=0 -// fq = mrt_V1*rhoA-mrt_V2*m1A+mrt_V3*m2A; -// distA[n] = fq; -// -// // q = 1 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); -// nread = neighborList[n+Np]; -// distA[nread] = fq; -// -// // q=2 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); -// nread = neighborList[n]; -// distA[nread] = fq; -// -// // q = 3 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); -// nread = neighborList[n+3*Np]; -// distA[nread] = fq; -// -// // q = 4 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); -// nread = neighborList[n+2*Np]; -// distA[nread] = fq; -// -// // q = 5 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); -// nread = neighborList[n+5*Np]; -// distA[nread] = fq; -// -// // q = 6 -// fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); -// nread = neighborList[n+4*Np]; -// distA[nread] = fq; -// -// // q = 7 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); -// nread = neighborList[n+7*Np]; -// distA[nread] = fq; -// -// // q = 8 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); -// nread = neighborList[n+6*Np]; -// distA[nread] = fq; -// -// // q = 9 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); -// nread = neighborList[n+9*Np]; -// distA[nread] = fq; -// -// // q = 10 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); -// nread = neighborList[n+8*Np]; -// distA[nread] = fq; -// -// // q = 11 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); -// nread = neighborList[n+11*Np]; -// distA[nread] = fq; -// -// // q = 12 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); -// nread = neighborList[n+10*Np]; -// distA[nread]= fq; -// -// // q = 13 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); -// nread = neighborList[n+13*Np]; -// distA[nread] = fq; -// -// // q= 14 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); -// nread = neighborList[n+12*Np]; -// distA[nread] = fq; -// -// // q = 15 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); -// nread = neighborList[n+15*Np]; -// distA[nread] = fq; -// -// // q = 16 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); -// nread = neighborList[n+14*Np]; -// distA[nread] = fq; -// -// // q = 17 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); -// nread = neighborList[n+17*Np]; -// distA[nread] = fq; -// -// // q = 18 -// fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); -// nread = neighborList[n+16*Np]; -// distA[nread] = fq; -// //........................................................................ -// -// // ------------------- Fluid Component B -----------------------// -// rlx_setA = 1.0/tauB; -// rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); -// //-------------------- MRT collison where body force has NO higher-order terms -------------// -// //..............carry out relaxation process............................................... -// //TODO need to incoporate porosity -// m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB) - m1B) -// + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); -// m2B = m2B + rlx_setA*((3*rhoB - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) -// + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); -// jxB = jxB + FxB; -// m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); -// jyB = jyB + FyB; -// m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB)- m6B) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); -// jzB = jzB + FzB; -// m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB)- m8B) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); -// m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) -// + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); -// //m10B = m10B + rlx_setA*( - m10B) -// // + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); -// m10B = m10B + rlx_setA*( -0.5*(rhoB*(2*ux*ux-uy*uy-uz*uz))- m10B) -// + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); -// m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) -// + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); -// //m12B = m12B + rlx_setA*( - m12B) -// // + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); -// m12B = m12B + rlx_setA*( -0.5*(rhoB*(uy*uy-uz*uz))- m12B) -// + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); -// m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) -// + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); -// m14B = m14B + rlx_setA*( rhoB*(uy*uz) - m14B) -// + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); -// m15B = m15B + rlx_setA*( rhoB*(ux*uz) - m15B) -// + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); -// m16B = m16B + rlx_setB*( - m16B); -// m17B = m17B + rlx_setB*( - m17B); -// m18B = m18B + rlx_setB*( - m18B); -// //....................................................................................................... -// -// -// // ------------------- Fluid Component B -----------------------// -// //.................inverse transformation...................................................... -// // q=0 -// fq = mrt_V1*rhoB-mrt_V2*m1B+mrt_V3*m2B; -// distB[n] = fq; -// -// // q = 1 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); -// nread = neighborList[n+Np]; -// distB[nread] = fq; -// -// // q=2 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); -// nread = neighborList[n]; -// distB[nread] = fq; -// -// // q = 3 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); -// nread = neighborList[n+3*Np]; -// distB[nread] = fq; -// -// // q = 4 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); -// nread = neighborList[n+2*Np]; -// distB[nread] = fq; -// -// // q = 5 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); -// nread = neighborList[n+5*Np]; -// distB[nread] = fq; -// -// // q = 6 -// fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); -// nread = neighborList[n+4*Np]; -// distB[nread] = fq; -// -// // q = 7 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); -// nread = neighborList[n+7*Np]; -// distB[nread] = fq; -// -// // q = 8 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); -// nread = neighborList[n+6*Np]; -// distB[nread] = fq; -// -// // q = 9 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); -// nread = neighborList[n+9*Np]; -// distB[nread] = fq; -// -// // q = 10 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); -// nread = neighborList[n+8*Np]; -// distB[nread] = fq; -// -// // q = 11 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); -// nread = neighborList[n+11*Np]; -// distB[nread] = fq; -// -// // q = 12 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); -// nread = neighborList[n+10*Np]; -// distB[nread]= fq; -// -// // q = 13 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); -// nread = neighborList[n+13*Np]; -// distB[nread] = fq; -// -// // q= 14 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); -// nread = neighborList[n+12*Np]; -// distB[nread] = fq; -// -// // q = 15 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); -// nread = neighborList[n+15*Np]; -// distB[nread] = fq; -// -// // q = 16 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); -// nread = neighborList[n+14*Np]; -// distB[nread] = fq; -// -// // q = 17 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); -// nread = neighborList[n+17*Np]; -// distB[nread] = fq; -// -// // q = 18 -// fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); -// nread = neighborList[n+16*Np]; -// distB[nread] = fq; -// //........................................................................ -// -// //Update velocity on device -// Velocity[0*Np+n] = ux; -// Velocity[1*Np+n] = uy; -// Velocity[2*Np+n] = uz; -// //Update pressure on device -// Pressure[n] = (nA+nB+Gsc*nA*nB)/3.0; -// } -// } -//} - -//__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, -// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, -// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, -// int start, int finish, int Np){ -// -// int ijk; -// int n; -// double vx,vy,vz,v_mag; -// double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; -// double ux,uy,uz; -// // conserved momemnts -// double jxA,jyA,jzA; -// double jxB,jyB,jzB; -// double rhoA,rhoB; -// double nA,nB; -// // non-conserved moments -// double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; -// double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; -// double fq; -// //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; -// double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) -// double porosity; -// double perm;//voxel permeability -// double permA,permB;//effective relative perm -// double c0, c1; //Guo's model parameters -// double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity -// double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity -// double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) -// double FxB, FyB, FzB; -// double rlx_setA,rlx_setB; -// double nA_gradx,nA_grady,nA_gradz; -// double nB_gradx,nB_grady,nB_gradz; -// double GffA_x,GffA_y,GffA_z; -// double GfsA_x,GfsA_y,GfsA_z; -// double GffB_x,GffB_y,GffB_z; -// double GfsB_x,GfsB_y,GfsB_z; -// -// const double mrt_V1=0.05263157894736842; -// const double mrt_V2=0.012531328320802; -// const double mrt_V3=0.04761904761904762; -// const double mrt_V4=0.004594820384294068; -// const double mrt_V5=0.01587301587301587; -// const double mrt_V6=0.0555555555555555555555555; -// const double mrt_V7=0.02777777777777778; -// const double mrt_V8=0.08333333333333333; -// const double mrt_V9=0.003341687552213868; -// const double mrt_V10=0.003968253968253968; -// const double mrt_V11=0.01388888888888889; -// const double mrt_V12=0.04166666666666666; -// -// int S = Np/NBLOCKS/NTHREADS + 1; -// for (int s=0; s 10Np => odd part of dist) - f1A = distA[nr1]; // reading the f1 data into register fq - f1B = distB[nr1]; // reading the f1 data into register fq - - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - f2A = distA[nr2]; // reading the f2 data into register fq - f2B = distB[nr2]; // reading the f2 data into register fq - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - f3A = distA[nr3]; - f3B = distB[nr3]; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - f4A = distA[nr4]; - f4B = distB[nr4]; - - // q=5 - nr5 = neighborList[n+4*Np]; - f5A = distA[nr5]; - f5B = distB[nr5]; - - // q = 6 - nr6 = neighborList[n+5*Np]; - f6A = distA[nr6]; - f6B = distB[nr6]; - - // q=7 - nr7 = neighborList[n+6*Np]; - f7A = distA[nr7]; - f7B = distB[nr7]; - - // q = 8 - nr8 = neighborList[n+7*Np]; - f8A = distA[nr8]; - f8B = distB[nr8]; - - // q=9 - nr9 = neighborList[n+8*Np]; - f9A = distA[nr9]; - f9B = distB[nr9]; - - // q = 10 - nr10 = neighborList[n+9*Np]; - f10A = distA[nr10]; - f10B = distB[nr10]; - - // q=11 - nr11 = neighborList[n+10*Np]; - f11A = distA[nr11]; - f11B = distB[nr11]; - - // q=12 - nr12 = neighborList[n+11*Np]; - f12A = distA[nr12]; - f12B = distB[nr12]; - - // q=13 - nr13 = neighborList[n+12*Np]; - f13A = distA[nr13]; - f13B = distB[nr13]; - - // q=14 - nr14 = neighborList[n+13*Np]; - f14A = distA[nr14]; - f14B = distB[nr14]; - - // q=15 - nr15 = neighborList[n+14*Np]; - f15A = distA[nr15]; - f15B = distB[nr15]; - - // q=16 - nr16 = neighborList[n+15*Np]; - f16A = distA[nr16]; - f16B = distB[nr16]; - - // q=17 - //fq = dist[18*Np+n]; - nr17 = neighborList[n+16*Np]; - f17A = distA[nr17]; - f17B = distB[nr17]; - - // q=18 - nr18 = neighborList[n+17*Np]; - f18A = distA[nr18]; - f18B = distB[nr18]; - //---------------------------------------------------------------------// - - // Compute SC fluid-fluid interaction force - GffA_x = -Gsc*rhoB_gradx; - GffA_y = -Gsc*rhoB_grady; - GffA_z = -Gsc*rhoB_gradz; - GffB_x = -Gsc*rhoA_gradx; - GffB_y = -Gsc*rhoA_grady; - GffB_z = -Gsc*rhoA_gradz; - // Compute SC fluid-solid force - GfsA_x = SolidForceA[n+0*Np]; - GfsA_y = SolidForceA[n+1*Np]; - GfsA_z = SolidForceA[n+2*Np]; - GfsB_x = SolidForceB[n+0*Np]; - GfsB_y = SolidForceB[n+1*Np]; - GfsB_z = SolidForceB[n+2*Np]; - - // Compute greyscale related parameters - // ------------------- Fluid Component A -----------------------// - jxA = f1A-f2A+f7A-f8A+f9A-f10A+f11A-f12A+f13A-f14A; - jyA = f3A-f4A+f7A-f8A-f9A+f10A+f15A-f16A+f17A-f18A; - jzA = f5A-f6A+f11A-f12A-f13A+f14A+f15A-f16A-f17A+f18A; - - c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permA); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); - vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); - vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); - FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); - FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); - if (porosity==1.0){ - FxA=rhoA*(Gx + GffA_x + GfsA_x); - FyA=rhoA*(Gy + GffA_y + GfsA_y); - FzA=rhoA*(Gz + GffA_z + GfsA_z); - } - // ------------------- Fluid Component B -----------------------// - // Compute greyscale related parameters - jxB = f1B-f2B+f7B-f8B+f9B-f10B+f11B-f12B+f13B-f14B; - jyB = f3B-f4B+f7B-f8B-f9B+f10B+f15B-f16B+f17B-f18B; - jzB = f5B-f6B+f11B-f12B-f13B+f14B+f15B-f16B-f17B+f18B; - - c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permB); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); - vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); - vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); - FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); - FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); - if (porosity==1.0){ - FxB=rhoB*(Gx + GffB_x + GfsB_x); - FyB=rhoB*(Gy + GffB_y + GfsB_y); - FzB=rhoB*(Gz + GffB_z + GfsB_z); - } - - // Calculate barycentric velocity of the fluid mixture - ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); - uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); - uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); - - //..............carry out relaxation process............................................... - // ------------------- Fluid Component A -----------------------// - // q=0 - distA[n] = f0A*(1.0-rlx) + rlx*0.3333333333333333*rhoA*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 1 - distA[nr2] = f1A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q=2 - distA[nr1] = f2A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(-3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 3 - distA[nr4] = f3A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 4 - distA[nr3] = f4A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 5 - distA[nr6] = f5A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(3. + (6.*uz)/porosity)); - - // q = 6 - distA[nr5] = f6A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(-3. + (6.*uz)/porosity)); - - // q = 7 - distA[nr8] = f7A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 8 - distA[nr7] = f8A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyA*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 9 - distA[nr10] = f9A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyA*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 10 - distA[nr9] = f10A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 11 - distA[nr12] = f11A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); - - // q = 12 - distA[nr11] = f12A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - FzA*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); - - // q = 13 - distA[nr14] = f13A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - FzA*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); - - // q= 14 - distA[nr13] = f14A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); - - // q = 15 - distA[nr16] = f15A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); - - // q = 16 - distA[nr15] = f16A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - FzA*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 17 - distA[nr18] = f17A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - FzA*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 18 - distA[nr17] = f18A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - - - // ------------------- Fluid Component B -----------------------// - // q=0 - distB[n] = f0B*(1.0-rlx) + rlx*0.3333333333333333*rhoB*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 1 - distB[nr2] = f1B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q=2 - distB[nr1] = f2B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(-3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 3 - distB[nr4] = f3B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 4 - distB[nr3] = f4B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 5 - distB[nr6] = f5B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(3. + (6.*uz)/porosity)); - - // q = 6 - distB[nr5] = f6B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(-3. + (6.*uz)/porosity)); - - // q = 7 - distB[nr8] = f7B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 8 - distB[nr7] = f8B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyB*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 9 - distB[nr10] = f9B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyB*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 10 - distB[nr9] = f10B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 11 - distB[nr12] = f11B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); - - // q = 12 - distB[nr11] = f12B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - FzB*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); - - // q = 13 - distB[nr14] = f13B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - FzB*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); - - // q= 14 - distB[nr13] = f14B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); - - // q = 15 - distB[nr16] = f15B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); - - // q = 16 - distB[nr15] = f16B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - FzB*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 17 - distB[nr18] = f17B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - FzB*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 18 - distB[nr17] = f18B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map,double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - int n; - int ijk; - double vx,vy,vz,v_mag; - double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; - double ux,uy,uz; - double rhoA,rhoB; - double jxA,jyA,jzA; - double jxB,jyB,jzB; - // distribution functions - double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A,f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; - double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B,f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double permA,permB;//effective relative perm - double c0, c1; //Guo's model parameters - double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity - double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity - double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double FxB, FyB, FzB; - double tau,rlx; - double phi;//phase field indicator - double rhoA_gradx,rhoA_grady,rhoA_gradz; - double rhoB_gradx,rhoB_grady,rhoB_gradz; - double GffA_x,GffA_y,GffA_z; - double GfsA_x,GfsA_y,GfsA_z; - double GffB_x,GffB_y,GffB_z; - double GfsB_x,GfsB_y,GfsB_z; - - for (n=start; n - -#define NBLOCKS 1024 -#define NTHREADS 256 - -__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Poros,double *Velocity, - double *Pressure, double rhoA,double rhoB, int N){ - - int n; - double ux,uy,uz,u_mag; - double pressure; - double porosity; - double rho0; - double phi; - double nA,nB; - - int S = N/NBLOCKS/NTHREADS + 1; - for (int s=0; s0.0)-Gsc*nB*nA_gradx*int(phi<0.0); -// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); -// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - dist[1*Np+n] = fq; - - // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - dist[2*Np+n] = fq; - - // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[3*Np+n] = fq; - - // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[4*Np+n] = fq; - - // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[5*Np+n] = fq; - - // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[6*Np+n] = fq; - - // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - dist[7*Np+n] = fq; - - // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - dist[8*Np+n] = fq; - - // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - dist[9*Np+n] = fq; - - // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - dist[10*Np+n] = fq; - - // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - dist[11*Np+n] = fq; - - // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - dist[12*Np+n] = fq; - - // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - dist[13*Np+n] = fq; - - // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - dist[14*Np+n] = fq; - - // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - dist[15*Np+n] = fq; - - // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - dist[16*Np+n] = fq; - - // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - dist[17*Np+n] = fq; - - // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - dist[18*Np+n] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // Calculate the color gradient - nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); - ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); - nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - double ColorMag = C; - if (C==0.0) ColorMag=1.0; - nx = nx/ColorMag; - ny = ny/ColorMag; - nz = nz/ColorMag; - if (C == 0.0) nx = ny = nz = 0.0; - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - nAB = 1.0/(nA+nB); - Aq[n] = 0.3333333333333333*nA; - Bq[n] = 0.3333333333333333*nB; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nx; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; - - Aq[1*Np+n] = a1; - Bq[1*Np+n] = b1; - Aq[2*Np+n] = a2; - Bq[2*Np+n] = b2; - - //............................................... - // q = 2 - // Cq = {0,1,0} - delta = beta*nA*nB*nAB*0.1111111111111111*ny; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; - - Aq[3*Np+n] = a1; - Bq[3*Np+n] = b1; - Aq[4*Np+n] = a2; - Bq[4*Np+n] = b2; - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nz; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; - - Aq[5*Np+n] = a1; - Bq[5*Np+n] = b1; - Aq[6*Np+n] = a2; - Bq[6*Np+n] = b2; - //............................................... - - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure){ - - int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho0; - double phi; - double nx,ny,nz,C; - double nA,nB; - double a1,b1,a2,b2,nAB,delta; - double beta=0.95; - double nA_gradx,nA_grady,nA_gradz; - double nB_gradx,nB_grady,nB_gradz; - double Gff_x,Gff_y,Gff_z; - double Gfs_x,Gfs_y,Gfs_z; - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s 10Np => odd part of dist) - fq = dist[nr1]; // reading the f1 data into register fq - pressure = fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // q=2 - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = dist[nr2]; // reading the f2 data into register fq - pressure += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - fq = dist[nr3]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - fq = dist[nr4]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - nr5 = neighborList[n+4*Np]; - fq = dist[nr5]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - nr6 = neighborList[n+5*Np]; - fq = dist[nr6]; - pressure += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = dist[nread]; - pressure += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - //---------------- Calculate SC fluid-fluid and fluid-solid forces ---------------// - // fluid-fluid force -// Gff_x = -Gsc*nA*nB_gradx*int(phi>0.0)-Gsc*nB*nA_gradx*int(phi<0.0); -// Gff_y = -Gsc*nA*nB_grady*int(phi>0.0)-Gsc*nB*nA_grady*int(phi<0.0); -// Gff_z = -Gsc*nA*nB_gradz*int(phi>0.0)-Gsc*nB*nA_gradz*int(phi<0.0); - Gff_x = -Gsc*(nA*nB_gradx+nB*nA_gradx); - Gff_y = -Gsc*(nA*nB_grady+nB*nA_grady); - Gff_z = -Gsc*(nA*nB_gradz+nB*nA_gradz); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - porosity = Poros[n]; - // use local saturation as an estimation of effective relperm values - perm = Perm[n]*nA/(nA+nB)*int(phi>0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - -// //..............carry out relaxation process............................................... -// m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1) -// + (1-0.5*rlx_setA)*38*(Fx*ux+Fy*uy+Fz*uz)/porosity; -// m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2) -// + (1-0.5*rlx_setA)*11*(-Fx*ux-Fy*uy-Fz*uz)/porosity; -// jx = jx + Fx; -// m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); -// jy = jy + Fy; -// m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); -// jz = jz + Fz; -// m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) -// + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); -// m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9) -// + (1-0.5*rlx_setA)*(4*Fx*ux-2*Fy*uy-2*Fz*uz)/porosity; -// m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10) -// + (1-0.5*rlx_setA)*(-2*Fx*ux+Fy*uy+Fz*uz)/porosity; -// m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11) -// + (1-0.5*rlx_setA)*(2*Fy*uy-2*Fz*uz)/porosity; -// m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12) -// + (1-0.5*rlx_setA)*(-Fy*uy+Fz*uz)/porosity; -// m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13) -// + (1-0.5*rlx_setA)*(Fy*ux+Fx*uy)/porosity; -// m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14) -// + (1-0.5*rlx_setA)*(Fz*uy+Fy*uz)/porosity; -// m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15) -// + (1-0.5*rlx_setA)*(Fz*ux+Fx*uz)/porosity; -// m16 = m16 + rlx_setB*( - m16); -// m17 = m17 + rlx_setB*( - m17); -// m18 = m18 + rlx_setB*( - m18); -// //....................................................................................................... - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((-30*rho0+19*(ux*ux+uy*uy+uz*uz)/porosity + 57*pressure*porosity) - m1); - m2 = m2 + rlx_setA*((12*rho0 - 5.5*(ux*ux+uy*uy+uz*uz)/porosity-27*pressure*porosity) - m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*ux*rho0) - m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*uy*rho0) - m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*uz*rho0) - m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*((rho0*(2*ux*ux-uy*uy-uz*uz)/porosity) - m9); - m10 = m10 + rlx_setA*(-0.5*rho0*((2*ux*ux-uy*uy-uz*uz)/porosity)- m10); - m11 = m11 + rlx_setA*((rho0*(uy*uy-uz*uz)/porosity) - m11); - m12 = m12 + rlx_setA*(-0.5*(rho0*(uy*uy-uz*uz)/porosity)- m12); - m13 = m13 + rlx_setA*((rho0*ux*uy/porosity) - m13); - m14 = m14 + rlx_setA*((rho0*uy*uz/porosity) - m14); - m15 = m15 + rlx_setA*((rho0*ux*uz/porosity) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho0-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - //nread = neighborList[n+Np]; - dist[nr2] = fq; - - // q=2 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - //nread = neighborList[n]; - dist[nr1] = fq; - - // q = 3 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+3*Np]; - dist[nr4] = fq; - - // q = 4 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+2*Np]; - dist[nr3] = fq; - - // q = 5 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+5*Np]; - dist[nr6] = fq; - - // q = 6 - fq = mrt_V1*rho0-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+4*Np]; - dist[nr5] = fq; - - // q = 7 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - nread = neighborList[n+7*Np]; - dist[nread] = fq; - - // q = 8 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - nread = neighborList[n+6*Np]; - dist[nread] = fq; - - // q = 9 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - nread = neighborList[n+9*Np]; - dist[nread] = fq; - - // q = 10 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - nread = neighborList[n+8*Np]; - dist[nread] = fq; - - // q = 11 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - nread = neighborList[n+11*Np]; - dist[nread] = fq; - - // q = 12 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - nread = neighborList[n+10*Np]; - dist[nread]= fq; - - // q = 13 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - nread = neighborList[n+13*Np]; - dist[nread] = fq; - - // q= 14 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - nread = neighborList[n+12*Np]; - dist[nread] = fq; - - // q = 15 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - nread = neighborList[n+15*Np]; - dist[nread] = fq; - - // q = 16 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - nread = neighborList[n+14*Np]; - dist[nread] = fq; - - // q = 17 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - nread = neighborList[n+17*Np]; - dist[nread] = fq; - - // q = 18 - fq = mrt_V1*rho0+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - nread = neighborList[n+16*Np]; - dist[nread] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // Calculate the color gradient - nx = (2*nB*nA_gradx-2*nA*nB_gradx)/(nA+nB)/(nA+nB); - ny = (2*nB*nA_grady-2*nA*nB_grady)/(nA+nB)/(nA+nB); - nz = (2*nB*nA_gradz-2*nA*nB_gradz)/(nA+nB)/(nA+nB); - //...........Normalize the Color Gradient................................. - C = sqrt(nx*nx+ny*ny+nz*nz); - double ColorMag = C; - if (C==0.0) ColorMag=1.0; - nx = nx/ColorMag; - ny = ny/ColorMag; - nz = nz/ColorMag; - if (C == 0.0) nx = ny = nz = 0.0; - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - nAB = 1.0/(nA+nB); - Aq[n] = 0.3333333333333333*nA; - Bq[n] = 0.3333333333333333*nB; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nx; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; - - // q = 1 - //nread = neighborList[n+Np]; - Aq[nr2] = a1; - Bq[nr2] = b1; - // q=2 - //nread = neighborList[n]; - Aq[nr1] = a2; - Bq[nr1] = b2; - - //............................................... - // Cq = {0,1,0} - delta = beta*nA*nB*nAB*0.1111111111111111*ny; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta; - - // q = 3 - //nread = neighborList[n+3*Np]; - Aq[nr4] = a1; - Bq[nr4] = b1; - // q = 4 - //nread = neighborList[n+2*Np]; - Aq[nr3] = a2; - Bq[nr3] = b2; - - //............................................... - // q = 4 - // Cq = {0,0,1} - delta = beta*nA*nB*nAB*0.1111111111111111*nz; - if (!(nA*nB*nAB>0)) delta=0; - a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta; - b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta; - a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta; - b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta; - - // q = 5 - //nread = neighborList[n+5*Np]; - Aq[nr6] = a1; - Bq[nr6] = b1; - // q = 6 - //nread = neighborList[n+4*Np]; - Aq[nr5] = a2; - Bq[nr5] = b2; - //............................................... - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - - int n, nread, nr1,nr2,nr3,nr4,nr5,nr6; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho,rho0; - double phi; - double phi_lap;//laplacian of phase field - double nA,nB; - double Gfs_x,Gfs_y,Gfs_z; - double Gff_x,Gff_y,Gff_z; - double chem; - double rlx_phi; - double a1,a2;//PDF of phase field - // *---------------------------------Pressure Tensor Gradient------------------------------------*// - double Pxx_x,Pyy_y,Pzz_z; - double Pxy_x,Pxy_y; - double Pyz_y,Pyz_z; - double Pxz_x,Pxz_z; - double px,py,pz; //pressure gradient - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - //Load pressure gradient - px=PressureGrad[0*Np+n]; - py=PressureGrad[1*Np+n]; - pz=PressureGrad[2*Np+n]; - - //Load pressure tensor gradient - //For reference full list of PressTensorGrad - //PressTensorGrad[n+0*Np] = Pxx_x - //PressTensorGrad[n+1*Np] = Pxx_y - //PressTensorGrad[n+2*Np] = Pxx_z - //PressTensorGrad[n+3*Np] = Pyy_x - //PressTensorGrad[n+4*Np] = Pyy_y - //PressTensorGrad[n+5*Np] = Pyy_z - //PressTensorGrad[n+6*Np] = Pzz_x - //PressTensorGrad[n+7*Np] = Pzz_y - //PressTensorGrad[n+8*Np] = Pzz_z - //PressTensorGrad[n+9*Np] = Pxy_x - //PressTensorGrad[n+10*Np] = Pxy_y - //PressTensorGrad[n+11*Np] = Pxy_z - //PressTensorGrad[n+12*Np] = Pyz_x - //PressTensorGrad[n+13*Np] = Pyz_y - //PressTensorGrad[n+14*Np] = Pyz_z - //PressTensorGrad[n+15*Np] = Pxz_x - //PressTensorGrad[n+16*Np] = Pxz_y - //PressTensorGrad[n+17*Np] = Pxz_z - Pxx_x = PressTensorGrad[0*Np+n]; - Pyy_y = PressTensorGrad[4*Np+n]; - Pzz_z = PressTensorGrad[8*Np+n]; - Pxy_x = PressTensorGrad[9*Np+n]; - Pxz_x = PressTensorGrad[15*Np+n]; - Pxy_y = PressTensorGrad[10*Np+n]; - Pyz_y = PressTensorGrad[13*Np+n]; - Pyz_z = PressTensorGrad[14*Np+n]; - Pxz_z = PressTensorGrad[17*Np+n]; - //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... - //TODO double check if you need porosity as a fre-factor - Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); - Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); - Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - // local density - rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - // local relaxation time - tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); - rlx_setA = 1.f/tau; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); - mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity - - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = dist[n]; - rho = fq; - m1 = -30.0*fq; - m2 = 12.0*fq; - - // q=1 - nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist) - fq = dist[nr1]; // reading the f1 data into register fq - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // q=2 - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = dist[nr2]; // reading the f2 data into register fq - rho += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - fq = dist[nr3]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - fq = dist[nr4]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - nr5 = neighborList[n+4*Np]; - fq = dist[nr5]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - nr6 = neighborList[n+5*Np]; - fq = dist[nr6]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = dist[nread]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - pressure=rho/3.0; - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - //nread = neighborList[n+Np]; - dist[nr2] = fq; - - // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - //nread = neighborList[n]; - dist[nr1] = fq; - - // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+3*Np]; - dist[nr4] = fq; - - // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - //nread = neighborList[n+2*Np]; - dist[nr3] = fq; - - // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+5*Np]; - dist[nr6] = fq; - - // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - //nread = neighborList[n+4*Np]; - dist[nr5] = fq; - - // q = 7 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - nread = neighborList[n+7*Np]; - dist[nread] = fq; - - // q = 8 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - nread = neighborList[n+6*Np]; - dist[nread] = fq; - - // q = 9 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - nread = neighborList[n+9*Np]; - dist[nread] = fq; - - // q = 10 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - nread = neighborList[n+8*Np]; - dist[nread] = fq; - - // q = 11 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - nread = neighborList[n+11*Np]; - dist[nread] = fq; - - // q = 12 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - nread = neighborList[n+10*Np]; - dist[nread]= fq; - - // q = 13 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - nread = neighborList[n+13*Np]; - dist[nread] = fq; - - // q= 14 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - nread = neighborList[n+12*Np]; - dist[nread] = fq; - - // q = 15 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - nread = neighborList[n+15*Np]; - dist[nread] = fq; - - // q = 16 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - nread = neighborList[n+14*Np]; - dist[nread] = fq; - - // q = 17 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - nread = neighborList[n+17*Np]; - dist[nread] = fq; - - // q = 18 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - nread = neighborList[n+16*Np]; - dist[nread] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // calcuale chemical potential - chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; - //rlx_phi = 3.f-sqrt(3.f); - rlx_phi = 1.0; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - //a1 = Cq[nr2]; - //a2 = Cq[nr1]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); - - // q = 1 - //nread = neighborList[n+Np]; - Cq[nr2] = a1; - // q=2 - //nread = neighborList[n]; - Cq[nr1] = a2; - - //............................................... - // Cq = {0,1,0} - //a1 = Cq[nr4]; - //a2 = Cq[nr3]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); - - // q = 3 - //nread = neighborList[n+3*Np]; - Cq[nr4] = a1; - // q = 4 - //nread = neighborList[n+2*Np]; - Cq[nr3] = a2; - - //............................................... - // q = 4 - // Cq = {0,0,1} - //a1 = Cq[nr6]; - //a2 = Cq[nr5]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); - - // q = 5 - //nread = neighborList[n+5*Np]; - Cq[nr6] = a1; - // q = 6 - //nread = neighborList[n+4*Np]; - Cq[nr5] = a2; - //............................................... - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - //a1=Cq[n]; - //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); - Cq[n] = phi-3.0*gamma*chem; - - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - int n; - double vx,vy,vz,v_mag; - double ux,uy,uz,u_mag; - double pressure;//defined for this incompressible model - // conserved momemnts - double jx,jy,jz; - // non-conserved moments - double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; - double fq; - // currently disable 'GeoFun' - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double c0, c1; //Guo's model parameters - double Fx, Fy, Fz;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double tau,tau_eff,rlx_setA,rlx_setB; - double mu_eff;//effective kinematic viscosity for Darcy term - double rho,rho0; - double phi; - double phi_lap;//laplacian of phase field - double nA,nB; - double Gfs_x,Gfs_y,Gfs_z; - double Gff_x,Gff_y,Gff_z; - double chem; - double rlx_phi; - double a1,a2;//PDF of phase field - // *---------------------------------Pressure Tensor Gradient------------------------------------*// - double Pxx_x,Pyy_y,Pzz_z; - double Pxy_x,Pxy_y; - double Pyz_y,Pyz_z; - double Pxz_x,Pxz_z; - double px,py,pz; //pressure gradient - - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s0.0)+Perm[n]*nB/(nA+nB)*int(phi<0.0); - - //Load pressure gradient - px=PressureGrad[0*Np+n]; - py=PressureGrad[1*Np+n]; - pz=PressureGrad[2*Np+n]; - - //Load pressure tensor gradient - //For reference full list of PressTensorGrad - //PressTensorGrad[n+0*Np] = Pxx_x - //PressTensorGrad[n+1*Np] = Pxx_y - //PressTensorGrad[n+2*Np] = Pxx_z - //PressTensorGrad[n+3*Np] = Pyy_x - //PressTensorGrad[n+4*Np] = Pyy_y - //PressTensorGrad[n+5*Np] = Pyy_z - //PressTensorGrad[n+6*Np] = Pzz_x - //PressTensorGrad[n+7*Np] = Pzz_y - //PressTensorGrad[n+8*Np] = Pzz_z - //PressTensorGrad[n+9*Np] = Pxy_x - //PressTensorGrad[n+10*Np] = Pxy_y - //PressTensorGrad[n+11*Np] = Pxy_z - //PressTensorGrad[n+12*Np] = Pyz_x - //PressTensorGrad[n+13*Np] = Pyz_y - //PressTensorGrad[n+14*Np] = Pyz_z - //PressTensorGrad[n+15*Np] = Pxz_x - //PressTensorGrad[n+16*Np] = Pxz_y - //PressTensorGrad[n+17*Np] = Pxz_z - Pxx_x = PressTensorGrad[0*Np+n]; - Pyy_y = PressTensorGrad[4*Np+n]; - Pzz_z = PressTensorGrad[8*Np+n]; - Pxy_x = PressTensorGrad[9*Np+n]; - Pxz_x = PressTensorGrad[15*Np+n]; - Pxy_y = PressTensorGrad[10*Np+n]; - Pyz_y = PressTensorGrad[13*Np+n]; - Pyz_z = PressTensorGrad[14*Np+n]; - Pxz_z = PressTensorGrad[17*Np+n]; - //............Compute the fluid-fluid force (gfx,gfy,gfz)................................... - //TODO double check if you need porosity as a fre-factor - Gff_x = porosity*px-(Pxx_x+Pxy_y+Pxz_z); - Gff_y = porosity*py-(Pxy_x+Pyy_y+Pyz_z); - Gff_z = porosity*pz-(Pxz_x+Pyz_y+Pzz_z); - // fluid-solid force - Gfs_x = (nA-nB)*SolidForce[n+0*Np]; - Gfs_y = (nA-nB)*SolidForce[n+1*Np]; - Gfs_z = (nA-nB)*SolidForce[n+2*Np]; - - // local density - rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA); - // local relaxation time - tau=tauA + 0.5*(1.0-phi)*(tauB-tauA); - rlx_setA = 1.f/tau; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff); - mu_eff = (tau_eff-0.5)/3.f;//kinematic viscosity - - - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = dist[n]; - rho = fq; - m1 = -30.0*fq; - m2 = 12.0*fq; - - // q=1 - fq = dist[2*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jx = fq; - m4 = -4.0*fq; - m9 = 2.0*fq; - m10 = -4.0*fq; - - // f2 = dist[10*Np+n]; - fq = dist[1*Np+n]; - rho += fq; - m1 -= 11.0*(fq); - m2 -= 4.0*(fq); - jx -= fq; - m4 += 4.0*(fq); - m9 += 2.0*(fq); - m10 -= 4.0*(fq); - - // q=3 - fq = dist[4*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy = fq; - m6 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 = fq; - m12 = -2.0*fq; - - // q = 4 - fq = dist[3*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jy -= fq; - m6 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 += fq; - m12 -= 2.0*fq; - - // q=5 - fq = dist[6*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz = fq; - m8 = -4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q = 6 - fq = dist[5*Np+n]; - rho += fq; - m1 -= 11.0*fq; - m2 -= 4.0*fq; - jz -= fq; - m8 += 4.0*fq; - m9 -= fq; - m10 += 2.0*fq; - m11 -= fq; - m12 += 2.0*fq; - - // q=7 - fq = dist[8*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 = fq; - m16 = fq; - m17 = -fq; - - // q = 8 - fq = dist[7*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 += fq; - m16 -= fq; - m17 += fq; - - // q=9 - fq = dist[10*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jy -= fq; - m6 -= fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 += fq; - m17 += fq; - - // q = 10 - fq = dist[9*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jy += fq; - m6 += fq; - m9 += fq; - m10 += fq; - m11 += fq; - m12 += fq; - m13 -= fq; - m16 -= fq; - m17 -= fq; - - // q=11 - fq = dist[12*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 = fq; - m16 -= fq; - m18 = fq; - - // q=12 - fq = dist[11*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 += fq; - m16 += fq; - m18 -= fq; - - // q=13 - fq = dist[14*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx += fq; - m4 += fq; - jz -= fq; - m8 -= fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 -= fq; - m18 -= fq; - - // q=14 - fq = dist[13*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jx -= fq; - m4 -= fq; - jz += fq; - m8 += fq; - m9 += fq; - m10 += fq; - m11 -= fq; - m12 -= fq; - m15 -= fq; - m16 += fq; - m18 += fq; - - // q=15 - fq = dist[16*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 = fq; - m17 += fq; - m18 -= fq; - - // q=16 - fq = dist[15*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 += fq; - m17 -= fq; - m18 += fq; - - // q=17 - fq = dist[18*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy += fq; - m6 += fq; - jz -= fq; - m8 -= fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 += fq; - m18 += fq; - - // q=18 - fq = dist[17*Np+n]; - rho += fq; - m1 += 8.0*fq; - m2 += fq; - jy -= fq; - m6 -= fq; - jz += fq; - m8 += fq; - m9 -= 2.0*fq; - m10 -= 2.0*fq; - m14 -= fq; - m17 -= fq; - m18 -= fq; - //---------------------------------------------------------------------// - - c0 = 0.5*(1.0+porosity*0.5*mu_eff/perm); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(perm); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jx/rho0+0.5*(porosity*Gx+Gff_x+Gfs_x); - vy = jy/rho0+0.5*(porosity*Gy+Gff_y+Gfs_y); - vz = jz/rho0+0.5*(porosity*Gz+Gff_z+Gfs_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux*ux+uy*uy+uz*uz); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - Fx = rho0*(-porosity*mu_eff/perm*ux - porosity*GeoFun/sqrt(perm)*u_mag*ux + porosity*Gx + Gff_x + Gfs_x); - Fy = rho0*(-porosity*mu_eff/perm*uy - porosity*GeoFun/sqrt(perm)*u_mag*uy + porosity*Gy + Gff_y + Gfs_y); - Fz = rho0*(-porosity*mu_eff/perm*uz - porosity*GeoFun/sqrt(perm)*u_mag*uz + porosity*Gz + Gff_z + Gfs_z); - if (porosity==1.0){ - Fx=rho0*(Gx + Gff_x + Gfs_x); - Fy=rho0*(Gy + Gff_y + Gfs_y); - Fz=rho0*(Gz + Gff_z + Gfs_z); - } - - //Calculate pressure for Incompressible-MRT model - //pressure=0.5/porosity*(pressure-0.5*rho0*u_mag*u_mag/porosity); - pressure=rho/3.0; - - //-------------------- IMRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2); - jx = jx + Fx; - m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fx); - jy = jy + Fy; - m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fy); - jz = jz + Fz; - m8 = m8 + rlx_setB*((-0.6666666666666666*jz)- m8) - + (1-0.5*rlx_setB)*(-0.6666666666666666*Fz); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); - m10 = m10 + rlx_setA*( - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); - m12 = m12 + rlx_setA*( - m12); - m13 = m13 + rlx_setA*( (jx*jy/rho0) - m13); - m14 = m14 + rlx_setA*( (jy*jz/rho0) - m14); - m15 = m15 + rlx_setA*( (jx*jz/rho0) - m15); - m16 = m16 + rlx_setB*( - m16); - m17 = m17 + rlx_setB*( - m17); - m18 = m18 + rlx_setB*( - m18); - //....................................................................................................... - - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; - dist[n] = fq; - - // q = 1 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10); - dist[1*Np+n] = fq; - - // q=2 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10); - dist[2*Np+n] = fq; - - // q = 3 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[3*Np+n] = fq; - - // q = 4 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12); - dist[4*Np+n] = fq; - - // q = 5 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[5*Np+n] = fq; - - // q = 6 - fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11); - dist[6*Np+n] = fq; - - // q = 7 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m16-m17); - dist[7*Np+n] = fq; - - // q = 8 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12+0.25*m13+0.125*(m17-m16); - dist[8*Np+n] = fq; - - // q = 9 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13+0.125*(m16+m17); - dist[9*Np+n] = fq; - - // q = 10 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11+mrt_V12*m12-0.25*m13-0.125*(m16+m17); - dist[10*Np+n] = fq; - - // q = 11 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m18-m16); - dist[11*Np+n] = fq; - - // q = 12 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12+0.25*m15+0.125*(m16-m18); - dist[12*Np+n] = fq; - - // q = 13 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15-0.125*(m16+m18); - dist[13*Np+n] = fq; - - // q= 14 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11-mrt_V12*m12-0.25*m15+0.125*(m16+m18); - dist[14*Np+n] = fq; - - // q = 15 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18); - dist[15*Np+n] = fq; - - // q = 16 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17); - dist[16*Np+n] = fq; - - // q = 17 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18); - dist[17*Np+n] = fq; - - // q = 18 - fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18); - dist[18*Np+n] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = pressure; - - //-----------------------Mass transport------------------------// - // calcuale chemical potential - chem = 0.125*(lambdaA+lambdaB)*(-phi+phi*phi*phi)-0.25*(kappaA+kappaB)*phi_lap; - //rlx_phi = 3.f-sqrt(3.f); - rlx_phi = 1.0; - - //............................................... - // q = 0,2,4 - // Cq = {1,0,0}, {0,1,0}, {0,0,1} - //a1 = Cq[1*Np+n]; - //a2 = Cq[2*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*ux)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*ux)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*ux); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*ux); - - Cq[1*Np+n] = a1; - Cq[2*Np+n] = a2; - - //............................................... - // q = 2 - // Cq = {0,1,0} - //a1 = Cq[3*Np+n]; - //a2 = Cq[4*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uy)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uy)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uy); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uy); - - Cq[3*Np+n] = a1; - Cq[4*Np+n] = a2; - //............................................... - // q = 4 - // Cq = {0,0,1} - //a1 = Cq[5*Np+n]; - //a2 = Cq[6*Np+n]; - //a1 = (1.0-rlx_phi)*a1+rlx_phi*(0.1111111111111111*4.5*(gamma*chem+phi*uz)); - //a2 = (1.0-rlx_phi)*a2+rlx_phi*(0.1111111111111111*4.5*(gamma*chem-phi*uz)); - a1 = 0.1111111111111111*4.5*(gamma*chem+phi*uz); - a2 = 0.1111111111111111*4.5*(gamma*chem-phi*uz); - - Cq[5*Np+n] = a1; - Cq[6*Np+n] = a2; - //............................................... - - // Instantiate mass transport distributions - // Stationary value - distribution 0 - //a1=Cq[n]; - //Cq[n] = (1.0-rlx_phi)*a1+rlx_phi*(phi-3.0*gamma*chem); - Cq[n] = phi-3.0*gamma*chem; - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ - int n; - int S = Np/NBLOCKS/NTHREADS + 1; - double phi; - double nA,nB; - double Den0; - for (int s=0; s>>(dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleFE: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, - double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure){ - - dvc_ScaLBL_D3Q19_AAodd_GreyscaleFE<<>>(neighborList, dist, Aq, Bq, Den, DenGradA, DenGradB, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, Gsc, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleFE: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - - dvc_ScaLBL_D3Q19_AAeven_GreyscaleFEChem<<>>(dist, Cq, Phi, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleFEChem: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np, - double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB, - double Gx, double Gy, double Gz, - double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap){ - - dvc_ScaLBL_D3Q19_AAodd_GreyscaleFEChem<<>>(neighborList, dist, Cq, Phi, SolidForce, start, finish, Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Gx, Gy, Gz, - Poros, Perm, Velocity, Pressure,PressureGrad,PressTensorGrad,PhiLap); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleFEChem: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Phi, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q7_GreyscaleFE_Init<<>>(Phi, Cq, PhiLap,gamma,kappaA,kappaB,lambdaA,lambdaB, start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_GreyscaleFE_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleFE_IMRT_Init<<>>(dist,Den,rhoA,rhoB,Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_IMRT_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEDensity<<>>(NeighborList, Aq, Bq, Den, Phi, start, finish, Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleFEDensity: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEDensity<<>>(Aq, Bq, Den, Phi, start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleFEDensity: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q7_AAodd_GreyscaleFEPhi<<>>(NeighborList, Cq, Phi, start, finish, Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAodd_GreyscaleFEPhi: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q7_AAeven_GreyscaleFEPhi<<>>(Cq, Phi, start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q7_AAeven_GreyscaleFEPhi: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_GreyscaleFE_Gradient<<>>(neighborList, Den, DenGrad, start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Gradient: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleFE_Laplacian<<>>(neighborList, Den, DenLap, start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Laplacian: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity, - double *Pressure, double rhoA,double rhoB, int Np){ - - dvc_ScaLBL_D3Q19_GreyscaleFE_Pressure<<>>(dist, Den, Porosity, Velocity, Pressure, rhoA, rhoB, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_Pressure: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap, - double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np){ - dvc_ScaLBL_D3Q19_GreyscaleFE_PressureTensor<<>>(neighborList,Phi,Pressure,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,start,finish,Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleFE_PressureTensor: %s \n",cudaGetErrorString(err)); - } -} diff --git a/gpu/GreyscaleSC.cu b/gpu/GreyscaleSC.cu deleted file mode 100644 index 309a9371..00000000 --- a/gpu/GreyscaleSC.cu +++ /dev/null @@ -1,3819 +0,0 @@ -#include - -#define NBLOCKS 1024 -#define NTHREADS 256 - -__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList,int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - int ijk; - int n, nread; - double vx,vy,vz,v_mag; - double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; - double ux,uy,uz; - // conserved momemnts - double jxA,jyA,jzA; - double jxB,jyB,jzB; - double rhoA,rhoB; - double nA,nB; - // non-conserved moments - double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; - double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; - double fq; - //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double permA,permB;//effective relative perm - double c0, c1; //Guo's model parameters - double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity - double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity - double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double FxB, FyB, FzB; - double rlx_setA,rlx_setB; - double nA_gradx,nA_grady,nA_gradz; - double nB_gradx,nB_grady,nB_gradz; - double GffA_x,GffA_y,GffA_z; - double GfsA_x,GfsA_y,GfsA_z; - double GffB_x,GffB_y,GffB_z; - double GfsB_x,GfsB_y,GfsB_z; - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s even part of dist) - fq = distA[nread]; // reading the f2 data into register fq - rhoA += fq; - m1A -= 11.0*(fq); - m2A -= 4.0*(fq); - jxA -= fq; - m4A += 4.0*(fq); - m9A += 2.0*(fq); - m10A -= 4.0*(fq); - - // q=3 - nread = neighborList[n+2*Np]; // neighbor 4 - fq = distA[nread]; - rhoA += fq; - m1A -= 11.0*fq; - m2A -= 4.0*fq; - jyA = fq; - m6A = -4.0*fq; - m9A -= fq; - m10A += 2.0*fq; - m11A = fq; - m12A = -2.0*fq; - - // q = 4 - nread = neighborList[n+3*Np]; // neighbor 3 - fq = distA[nread]; - rhoA += fq; - m1A -= 11.0*fq; - m2A -= 4.0*fq; - jyA -= fq; - m6A += 4.0*fq; - m9A -= fq; - m10A += 2.0*fq; - m11A += fq; - m12A -= 2.0*fq; - - // q=5 - nread = neighborList[n+4*Np]; - fq = distA[nread]; - rhoA += fq; - m1A -= 11.0*fq; - m2A -= 4.0*fq; - jzA = fq; - m8A = -4.0*fq; - m9A -= fq; - m10A += 2.0*fq; - m11A -= fq; - m12A += 2.0*fq; - - - // q = 6 - nread = neighborList[n+5*Np]; - fq = distA[nread]; - rhoA += fq; - m1A -= 11.0*fq; - m2A -= 4.0*fq; - jzA -= fq; - m8A += 4.0*fq; - m9A -= fq; - m10A += 2.0*fq; - m11A -= fq; - m12A += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA += fq; - m4A += fq; - jyA += fq; - m6A += fq; - m9A += fq; - m10A += fq; - m11A += fq; - m12A += fq; - m13A = fq; - m16A = fq; - m17A = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA -= fq; - m4A -= fq; - jyA -= fq; - m6A -= fq; - m9A += fq; - m10A += fq; - m11A += fq; - m12A += fq; - m13A += fq; - m16A -= fq; - m17A += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA += fq; - m4A += fq; - jyA -= fq; - m6A -= fq; - m9A += fq; - m10A += fq; - m11A += fq; - m12A += fq; - m13A -= fq; - m16A += fq; - m17A += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA -= fq; - m4A -= fq; - jyA += fq; - m6A += fq; - m9A += fq; - m10A += fq; - m11A += fq; - m12A += fq; - m13A -= fq; - m16A -= fq; - m17A -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA += fq; - m4A += fq; - jzA += fq; - m8A += fq; - m9A += fq; - m10A += fq; - m11A -= fq; - m12A -= fq; - m15A = fq; - m16A -= fq; - m18A = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA -= fq; - m4A -= fq; - jzA -= fq; - m8A -= fq; - m9A += fq; - m10A += fq; - m11A -= fq; - m12A -= fq; - m15A += fq; - m16A += fq; - m18A -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA += fq; - m4A += fq; - jzA -= fq; - m8A -= fq; - m9A += fq; - m10A += fq; - m11A -= fq; - m12A -= fq; - m15A -= fq; - m16A -= fq; - m18A -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jxA -= fq; - m4A -= fq; - jzA += fq; - m8A += fq; - m9A += fq; - m10A += fq; - m11A -= fq; - m12A -= fq; - m15A -= fq; - m16A += fq; - m18A += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jyA += fq; - m6A += fq; - jzA += fq; - m8A += fq; - m9A -= 2.0*fq; - m10A -= 2.0*fq; - m14A = fq; - m17A += fq; - m18A -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jyA -= fq; - m6A -= fq; - jzA -= fq; - m8A -= fq; - m9A -= 2.0*fq; - m10A -= 2.0*fq; - m14A += fq; - m17A -= fq; - m18A += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jyA += fq; - m6A += fq; - jzA -= fq; - m8A -= fq; - m9A -= 2.0*fq; - m10A -= 2.0*fq; - m14A -= fq; - m17A += fq; - m18A += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = distA[nread]; - rhoA += fq; - m1A += 8.0*fq; - m2A += fq; - jyA -= fq; - m6A -= fq; - jzA += fq; - m8A += fq; - m9A -= 2.0*fq; - m10A -= 2.0*fq; - m14A -= fq; - m17A -= fq; - m18A -= fq; - //---------------------------------------------------------------------// - - // ------------------- Fluid component B ---------------------------------// - //........................................................................ - // READ THE DISTRIBUTIONS - // (read from opposite array due to previous swap operation) - //........................................................................ - // q=0 - fq = distB[n]; - rhoB = fq; - m1B = -30.0*fq; - m2B = 12.0*fq; - - // q=1 - nread = neighborList[n]; // neighbor 2 - fq = distB[nread]; // reading the f1 data into register fq - rhoB += fq; - m1B -= 11.0*fq; - m2B -= 4.0*fq; - jxB = fq; - m4B = -4.0*fq; - m9B = 2.0*fq; - m10B = -4.0*fq; - - // q=2 - nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - fq = distB[nread]; // reading the f2 data into register fq - rhoB += fq; - m1B -= 11.0*(fq); - m2B -= 4.0*(fq); - jxB -= fq; - m4B += 4.0*(fq); - m9B += 2.0*(fq); - m10B -= 4.0*(fq); - - // q=3 - nread = neighborList[n+2*Np]; // neighbor 4 - fq = distB[nread]; - rhoB += fq; - m1B -= 11.0*fq; - m2B -= 4.0*fq; - jyB = fq; - m6B = -4.0*fq; - m9B -= fq; - m10B += 2.0*fq; - m11B = fq; - m12B = -2.0*fq; - - // q = 4 - nread = neighborList[n+3*Np]; // neighbor 3 - fq = distB[nread]; - rhoB += fq; - m1B -= 11.0*fq; - m2B -= 4.0*fq; - jyB -= fq; - m6B += 4.0*fq; - m9B -= fq; - m10B += 2.0*fq; - m11B += fq; - m12B -= 2.0*fq; - - // q=5 - nread = neighborList[n+4*Np]; - fq = distB[nread]; - rhoB += fq; - m1B -= 11.0*fq; - m2B -= 4.0*fq; - jzB = fq; - m8B = -4.0*fq; - m9B -= fq; - m10B += 2.0*fq; - m11B -= fq; - m12B += 2.0*fq; - - - // q = 6 - nread = neighborList[n+5*Np]; - fq = distB[nread]; - rhoB += fq; - m1B -= 11.0*fq; - m2B -= 4.0*fq; - jzB -= fq; - m8B += 4.0*fq; - m9B -= fq; - m10B += 2.0*fq; - m11B -= fq; - m12B += 2.0*fq; - - // q=7 - nread = neighborList[n+6*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB += fq; - m4B += fq; - jyB += fq; - m6B += fq; - m9B += fq; - m10B += fq; - m11B += fq; - m12B += fq; - m13B = fq; - m16B = fq; - m17B = -fq; - - // q = 8 - nread = neighborList[n+7*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB -= fq; - m4B -= fq; - jyB -= fq; - m6B -= fq; - m9B += fq; - m10B += fq; - m11B += fq; - m12B += fq; - m13B += fq; - m16B -= fq; - m17B += fq; - - // q=9 - nread = neighborList[n+8*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB += fq; - m4B += fq; - jyB -= fq; - m6B -= fq; - m9B += fq; - m10B += fq; - m11B += fq; - m12B += fq; - m13B -= fq; - m16B += fq; - m17B += fq; - - // q = 10 - nread = neighborList[n+9*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB -= fq; - m4B -= fq; - jyB += fq; - m6B += fq; - m9B += fq; - m10B += fq; - m11B += fq; - m12B += fq; - m13B -= fq; - m16B -= fq; - m17B -= fq; - - // q=11 - nread = neighborList[n+10*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB += fq; - m4B += fq; - jzB += fq; - m8B += fq; - m9B += fq; - m10B += fq; - m11B -= fq; - m12B -= fq; - m15B = fq; - m16B -= fq; - m18B = fq; - - // q=12 - nread = neighborList[n+11*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB -= fq; - m4B -= fq; - jzB -= fq; - m8B -= fq; - m9B += fq; - m10B += fq; - m11B -= fq; - m12B -= fq; - m15B += fq; - m16B += fq; - m18B -= fq; - - // q=13 - nread = neighborList[n+12*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB += fq; - m4B += fq; - jzB -= fq; - m8B -= fq; - m9B += fq; - m10B += fq; - m11B -= fq; - m12B -= fq; - m15B -= fq; - m16B -= fq; - m18B -= fq; - - // q=14 - nread = neighborList[n+13*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jxB -= fq; - m4B -= fq; - jzB += fq; - m8B += fq; - m9B += fq; - m10B += fq; - m11B -= fq; - m12B -= fq; - m15B -= fq; - m16B += fq; - m18B += fq; - - // q=15 - nread = neighborList[n+14*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jyB += fq; - m6B += fq; - jzB += fq; - m8B += fq; - m9B -= 2.0*fq; - m10B -= 2.0*fq; - m14B = fq; - m17B += fq; - m18B -= fq; - - // q=16 - nread = neighborList[n+15*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jyB -= fq; - m6B -= fq; - jzB -= fq; - m8B -= fq; - m9B -= 2.0*fq; - m10B -= 2.0*fq; - m14B += fq; - m17B -= fq; - m18B += fq; - - // q=17 - nread = neighborList[n+16*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jyB += fq; - m6B += fq; - jzB -= fq; - m8B -= fq; - m9B -= 2.0*fq; - m10B -= 2.0*fq; - m14B -= fq; - m17B += fq; - m18B += fq; - - // q=18 - nread = neighborList[n+17*Np]; - fq = distB[nread]; - rhoB += fq; - m1B += 8.0*fq; - m2B += fq; - jyB -= fq; - m6B -= fq; - jzB += fq; - m8B += fq; - m9B -= 2.0*fq; - m10B -= 2.0*fq; - m14B -= fq; - m17B -= fq; - m18B -= fq; - //---------------------------------------------------------------------// - - - // Compute SC fluid-fluid interaction force - GffA_x = -Gsc*nB_gradx; - GffA_y = -Gsc*nB_grady; - GffA_z = -Gsc*nB_gradz; - GffB_x = -Gsc*nA_gradx; - GffB_y = -Gsc*nA_grady; - GffB_z = -Gsc*nA_gradz; - // Compute SC fluid-solid force - GfsA_x = SolidForceA[n+0*Np]; - GfsA_y = SolidForceA[n+1*Np]; - GfsA_z = SolidForceA[n+2*Np]; - GfsB_x = SolidForceB[n+0*Np]; - GfsB_y = SolidForceB[n+1*Np]; - GfsB_z = SolidForceB[n+2*Np]; - - // Compute greyscale related parameters - // ------------------- Fluid Component A -----------------------// - c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permA); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); - vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); - vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxA = nA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); - FyA = nA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); - FzA = nA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); - if (porosity==1.0){ - FxA=nA*(Gx + GffA_x + GfsA_x); - FyA=nA*(Gy + GffA_y + GfsA_y); - FzA=nA*(Gz + GffA_z + GfsA_z); - } - // ------------------- Fluid Component B -----------------------// - // Compute greyscale related parameters - c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permB); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); - vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); - vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxB = nB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); - FyB = nB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); - FzB = nB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); - if (porosity==1.0){ - FxB=nB*(Gx + GffB_x + GfsB_x); - FyB=nB*(Gy + GffB_y + GfsB_y); - FzB=nB*(Gz + GffB_z + GfsB_z); - } - - // Calculate barycentric velocity of the fluid mixture - ux = (nA*ux_A+nB*ux_B)/(nA+nB); - uy = (nA*uy_A+nB*uy_B)/(nA+nB); - uz = (nA*uz_A+nB*uz_B)/(nA+nB); - - // ------------------- Fluid Component A -----------------------// - rlx_setA = 1.0/tauA; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - //-------------------- MRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - //TODO need to incoporate porosity - m1A = m1A + rlx_setA*((19*rhoA*(ux*ux+uy*uy+uz*uz) - 11*rhoA) - m1A) - + (1-0.5*rlx_setA)*38*(FxA*ux+FyA*uy+FzA*uz); - m2A = m2A + rlx_setA*((3*rhoA - 5.5*rhoA*(ux*ux+uy*uy+uz*uz))- m2A) - + (1-0.5*rlx_setA)*11*(-FxA*ux-FyA*uy-FzA*uz); - jxA = jxA + FxA; - m4A = m4A + rlx_setB*((-0.6666666666666666*ux*rhoA)- m4A) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FxA); - jyA = jyA + FyA; - m6A = m6A + rlx_setB*((-0.6666666666666666*uy*rhoA)- m6A) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FyA); - jzA = jzA + FzA; - m8A = m8A + rlx_setB*((-0.6666666666666666*uz*rhoA)- m8A) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FzA); - m9A = m9A + rlx_setA*((rhoA*(2*ux*ux-uy*uy-uz*uz)) - m9A) - + (1-0.5*rlx_setA)*(4*FxA*ux-2*FyA*uy-2*FzA*uz); - //m10A = m10A + rlx_setA*( - m10A) - // + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); - m10A = m10A + rlx_setA*( -0.5*(rhoA*(2*ux*ux-uy*uy-uz*uz))- m10A) - + (1-0.5*rlx_setA)*(-2*FxA*ux+FyA*uy+FzA*uz); - m11A = m11A + rlx_setA*((rhoA*(uy*uy-uz*uz)) - m11A) - + (1-0.5*rlx_setA)*(2*FyA*uy-2*FzA*uz); - //m12A = m12A + rlx_setA*( - m12A) - // + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); - m12A = m12A + rlx_setA*( -0.5*(rhoA*(uy*uy-uz*uz))- m12A) - + (1-0.5*rlx_setA)*(-FyA*uy+FzA*uz); - m13A = m13A + rlx_setA*( rhoA*(ux*uy) - m13A) - + (1-0.5*rlx_setA)*(FyA*ux+FxA*uy); - m14A = m14A + rlx_setA*( rhoA*(uy*uz) - m14A) - + (1-0.5*rlx_setA)*(FzA*uy+FyA*uz); - m15A = m15A + rlx_setA*( rhoA*(ux*uz) - m15A) - + (1-0.5*rlx_setA)*(FzA*ux+FxA*uz); - m16A = m16A + rlx_setB*( - m16A); - m17A = m17A + rlx_setB*( - m17A); - m18A = m18A + rlx_setB*( - m18A); - //....................................................................................................... - - - // ------------------- Fluid Component A -----------------------// - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rhoA-mrt_V2*m1A+mrt_V3*m2A; - distA[n] = fq; - - // q = 1 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jxA-m4A)+mrt_V6*(m9A-m10A); - nread = neighborList[n+Np]; - distA[nread] = fq; - - // q=2 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m4A-jxA)+mrt_V6*(m9A-m10A); - nread = neighborList[n]; - distA[nread] = fq; - - // q = 3 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jyA-m6A)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - nread = neighborList[n+3*Np]; - distA[nread] = fq; - - // q = 4 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m6A-jyA)+mrt_V7*(m10A-m9A)+mrt_V8*(m11A-m12A); - nread = neighborList[n+2*Np]; - distA[nread] = fq; - - // q = 5 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(jzA-m8A)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - nread = neighborList[n+5*Np]; - distA[nread] = fq; - - // q = 6 - fq = mrt_V1*rhoA-mrt_V4*m1A-mrt_V5*m2A+0.1*(m8A-jzA)+mrt_V7*(m10A-m9A)+mrt_V8*(m12A-m11A); - nread = neighborList[n+4*Np]; - distA[nread] = fq; - - // q = 7 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jyA)+0.025*(m4A+m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m16A-m17A); - nread = neighborList[n+7*Np]; - distA[nread] = fq; - - // q = 8 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jyA)-0.025*(m4A+m6A) +mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A+0.25*m13A+0.125*(m17A-m16A); - nread = neighborList[n+6*Np]; - distA[nread] = fq; - - // q = 9 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jyA)+0.025*(m4A-m6A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A+0.125*(m16A+m17A); - nread = neighborList[n+9*Np]; - distA[nread] = fq; - - // q = 10 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jxA)+0.025*(m6A-m4A)+mrt_V7*m9A+mrt_V11*m10A+mrt_V8*m11A+mrt_V12*m12A-0.25*m13A-0.125*(m16A+m17A); - nread = neighborList[n+8*Np]; - distA[nread] = fq; - - // q = 11 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA+jzA)+0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m18A-m16A); - nread = neighborList[n+11*Np]; - distA[nread] = fq; - - // q = 12 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jxA+jzA)-0.025*(m4A+m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A+0.25*m15A+0.125*(m16A-m18A); - nread = neighborList[n+10*Np]; - distA[nread]= fq; - - // q = 13 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jxA-jzA)+0.025*(m4A-m8A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A-0.125*(m16A+m18A); - nread = neighborList[n+13*Np]; - distA[nread] = fq; - - // q= 14 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jxA)+0.025*(m8A-m4A)+mrt_V7*m9A+mrt_V11*m10A-mrt_V8*m11A-mrt_V12*m12A-0.25*m15A+0.125*(m16A+m18A); - nread = neighborList[n+12*Np]; - distA[nread] = fq; - - // q = 15 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA+jzA)+0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m17A-m18A); - nread = neighborList[n+15*Np]; - distA[nread] = fq; - - // q = 16 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A-0.1*(jyA+jzA)-0.025*(m6A+m8A)-mrt_V6*m9A-mrt_V7*m10A+0.25*m14A+0.125*(m18A-m17A); - nread = neighborList[n+14*Np]; - distA[nread] = fq; - - // q = 17 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jyA-jzA)+0.025*(m6A-m8A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A+0.125*(m17A+m18A); - nread = neighborList[n+17*Np]; - distA[nread] = fq; - - // q = 18 - fq = mrt_V1*rhoA+mrt_V9*m1A+mrt_V10*m2A+0.1*(jzA-jyA)+0.025*(m8A-m6A)-mrt_V6*m9A-mrt_V7*m10A-0.25*m14A-0.125*(m17A+m18A); - nread = neighborList[n+16*Np]; - distA[nread] = fq; - //........................................................................ - - // ------------------- Fluid Component B -----------------------// - rlx_setA = 1.0/tauB; - rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - //-------------------- MRT collison where body force has NO higher-order terms -------------// - //..............carry out relaxation process............................................... - //TODO need to incoporate porosity - m1B = m1B + rlx_setA*((19*rhoB*(ux*ux+uy*uy+uz*uz) - 11*rhoB) - m1B) - + (1-0.5*rlx_setA)*38*(FxB*ux+FyB*uy+FzB*uz); - m2B = m2B + rlx_setA*((3*rhoB - 5.5*rhoB*(ux*ux+uy*uy+uz*uz))- m2B) - + (1-0.5*rlx_setA)*11*(-FxB*ux-FyB*uy-FzB*uz); - jxB = jxB + FxB; - m4B = m4B + rlx_setB*((-0.6666666666666666*ux*rhoB)- m4B) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FxB); - jyB = jyB + FyB; - m6B = m6B + rlx_setB*((-0.6666666666666666*uy*rhoB)- m6B) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FyB); - jzB = jzB + FzB; - m8B = m8B + rlx_setB*((-0.6666666666666666*uz*rhoB)- m8B) - + (1-0.5*rlx_setB)*(-0.6666666666666666*FzB); - m9B = m9B + rlx_setA*((rhoB*(2*ux*ux-uy*uy-uz*uz)) - m9B) - + (1-0.5*rlx_setA)*(4*FxB*ux-2*FyB*uy-2*FzB*uz); - //m10B = m10B + rlx_setA*( - m10B) - // + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); - m10B = m10B + rlx_setA*( -0.5*(rhoB*(2*ux*ux-uy*uy-uz*uz))- m10B) - + (1-0.5*rlx_setA)*(-2*FxB*ux+FyB*uy+FzB*uz); - m11B = m11B + rlx_setA*((rhoB*(uy*uy-uz*uz)) - m11B) - + (1-0.5*rlx_setA)*(2*FyB*uy-2*FzB*uz); - //m12B = m12B + rlx_setA*( - m12B) - // + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); - m12B = m12B + rlx_setA*( -0.5*(rhoB*(uy*uy-uz*uz))- m12B) - + (1-0.5*rlx_setA)*(-FyB*uy+FzB*uz); - m13B = m13B + rlx_setA*( rhoB*(ux*uy) - m13B) - + (1-0.5*rlx_setA)*(FyB*ux+FxB*uy); - m14B = m14B + rlx_setA*( rhoB*(uy*uz) - m14B) - + (1-0.5*rlx_setA)*(FzB*uy+FyB*uz); - m15B = m15B + rlx_setA*( rhoB*(ux*uz) - m15B) - + (1-0.5*rlx_setA)*(FzB*ux+FxB*uz); - m16B = m16B + rlx_setB*( - m16B); - m17B = m17B + rlx_setB*( - m17B); - m18B = m18B + rlx_setB*( - m18B); - //....................................................................................................... - - - // ------------------- Fluid Component B -----------------------// - //.................inverse transformation...................................................... - // q=0 - fq = mrt_V1*rhoB-mrt_V2*m1B+mrt_V3*m2B; - distB[n] = fq; - - // q = 1 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jxB-m4B)+mrt_V6*(m9B-m10B); - nread = neighborList[n+Np]; - distB[nread] = fq; - - // q=2 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m4B-jxB)+mrt_V6*(m9B-m10B); - nread = neighborList[n]; - distB[nread] = fq; - - // q = 3 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jyB-m6B)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - nread = neighborList[n+3*Np]; - distB[nread] = fq; - - // q = 4 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m6B-jyB)+mrt_V7*(m10B-m9B)+mrt_V8*(m11B-m12B); - nread = neighborList[n+2*Np]; - distB[nread] = fq; - - // q = 5 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(jzB-m8B)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - nread = neighborList[n+5*Np]; - distB[nread] = fq; - - // q = 6 - fq = mrt_V1*rhoB-mrt_V4*m1B-mrt_V5*m2B+0.1*(m8B-jzB)+mrt_V7*(m10B-m9B)+mrt_V8*(m12B-m11B); - nread = neighborList[n+4*Np]; - distB[nread] = fq; - - // q = 7 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jyB)+0.025*(m4B+m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m16B-m17B); - nread = neighborList[n+7*Np]; - distB[nread] = fq; - - // q = 8 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jyB)-0.025*(m4B+m6B) +mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B+0.25*m13B+0.125*(m17B-m16B); - nread = neighborList[n+6*Np]; - distB[nread] = fq; - - // q = 9 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jyB)+0.025*(m4B-m6B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B+0.125*(m16B+m17B); - nread = neighborList[n+9*Np]; - distB[nread] = fq; - - // q = 10 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jxB)+0.025*(m6B-m4B)+mrt_V7*m9B+mrt_V11*m10B+mrt_V8*m11B+mrt_V12*m12B-0.25*m13B-0.125*(m16B+m17B); - nread = neighborList[n+8*Np]; - distB[nread] = fq; - - // q = 11 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB+jzB)+0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m18B-m16B); - nread = neighborList[n+11*Np]; - distB[nread] = fq; - - // q = 12 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jxB+jzB)-0.025*(m4B+m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B+0.25*m15B+0.125*(m16B-m18B); - nread = neighborList[n+10*Np]; - distB[nread]= fq; - - // q = 13 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jxB-jzB)+0.025*(m4B-m8B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B-0.125*(m16B+m18B); - nread = neighborList[n+13*Np]; - distB[nread] = fq; - - // q= 14 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jxB)+0.025*(m8B-m4B)+mrt_V7*m9B+mrt_V11*m10B-mrt_V8*m11B-mrt_V12*m12B-0.25*m15B+0.125*(m16B+m18B); - nread = neighborList[n+12*Np]; - distB[nread] = fq; - - // q = 15 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB+jzB)+0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m17B-m18B); - nread = neighborList[n+15*Np]; - distB[nread] = fq; - - // q = 16 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B-0.1*(jyB+jzB)-0.025*(m6B+m8B)-mrt_V6*m9B-mrt_V7*m10B+0.25*m14B+0.125*(m18B-m17B); - nread = neighborList[n+14*Np]; - distB[nread] = fq; - - // q = 17 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jyB-jzB)+0.025*(m6B-m8B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B+0.125*(m17B+m18B); - nread = neighborList[n+17*Np]; - distB[nread] = fq; - - // q = 18 - fq = mrt_V1*rhoB+mrt_V9*m1B+mrt_V10*m2B+0.1*(jzB-jyB)+0.025*(m8B-m6B)-mrt_V6*m9B-mrt_V7*m10B-0.25*m14B-0.125*(m17B+m18B); - nread = neighborList[n+16*Np]; - distB[nread] = fq; - //........................................................................ - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = (nA+nB+Gsc*nA*nB)/3.0; - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - int ijk; - int n; - double vx,vy,vz,v_mag; - double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; - double ux,uy,uz; - // conserved momemnts - double jxA,jyA,jzA; - double jxB,jyB,jzB; - double rhoA,rhoB; - double nA,nB; - // non-conserved moments - double m1A,m2A,m4A,m6A,m8A,m9A,m10A,m11A,m12A,m13A,m14A,m15A,m16A,m17A,m18A; - double m1B,m2B,m4B,m6B,m8B,m9B,m10B,m11B,m12B,m13B,m14B,m15B,m16B,m17B,m18B; - double fq; - //double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18; - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double permA,permB;//effective relative perm - double c0, c1; //Guo's model parameters - double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity - double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity - double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double FxB, FyB, FzB; - double rlx_setA,rlx_setB; - double nA_gradx,nA_grady,nA_gradz; - double nB_gradx,nB_grady,nB_gradz; - double GffA_x,GffA_y,GffA_z; - double GfsA_x,GfsA_y,GfsA_z; - double GffB_x,GffB_y,GffB_z; - double GfsB_x,GfsB_y,GfsB_z; - - const double mrt_V1=0.05263157894736842; - const double mrt_V2=0.012531328320802; - const double mrt_V3=0.04761904761904762; - const double mrt_V4=0.004594820384294068; - const double mrt_V5=0.01587301587301587; - const double mrt_V6=0.0555555555555555555555555; - const double mrt_V7=0.02777777777777778; - const double mrt_V8=0.08333333333333333; - const double mrt_V9=0.003341687552213868; - const double mrt_V10=0.003968253968253968; - const double mrt_V11=0.01388888888888889; - const double mrt_V12=0.04166666666666666; - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s 10Np => odd part of dist) - f1A = distA[nr1]; // reading the f1 data into register fq - f1B = distB[nr1]; // reading the f1 data into register fq - - nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) - f2A = distA[nr2]; // reading the f2 data into register fq - f2B = distB[nr2]; // reading the f2 data into register fq - - // q=3 - nr3 = neighborList[n+2*Np]; // neighbor 4 - f3A = distA[nr3]; - f3B = distB[nr3]; - - // q = 4 - nr4 = neighborList[n+3*Np]; // neighbor 3 - f4A = distA[nr4]; - f4B = distB[nr4]; - - // q=5 - nr5 = neighborList[n+4*Np]; - f5A = distA[nr5]; - f5B = distB[nr5]; - - // q = 6 - nr6 = neighborList[n+5*Np]; - f6A = distA[nr6]; - f6B = distB[nr6]; - - // q=7 - nr7 = neighborList[n+6*Np]; - f7A = distA[nr7]; - f7B = distB[nr7]; - - // q = 8 - nr8 = neighborList[n+7*Np]; - f8A = distA[nr8]; - f8B = distB[nr8]; - - // q=9 - nr9 = neighborList[n+8*Np]; - f9A = distA[nr9]; - f9B = distB[nr9]; - - // q = 10 - nr10 = neighborList[n+9*Np]; - f10A = distA[nr10]; - f10B = distB[nr10]; - - // q=11 - nr11 = neighborList[n+10*Np]; - f11A = distA[nr11]; - f11B = distB[nr11]; - - // q=12 - nr12 = neighborList[n+11*Np]; - f12A = distA[nr12]; - f12B = distB[nr12]; - - // q=13 - nr13 = neighborList[n+12*Np]; - f13A = distA[nr13]; - f13B = distB[nr13]; - - // q=14 - nr14 = neighborList[n+13*Np]; - f14A = distA[nr14]; - f14B = distB[nr14]; - - // q=15 - nr15 = neighborList[n+14*Np]; - f15A = distA[nr15]; - f15B = distB[nr15]; - - // q=16 - nr16 = neighborList[n+15*Np]; - f16A = distA[nr16]; - f16B = distB[nr16]; - - // q=17 - //fq = dist[18*Np+n]; - nr17 = neighborList[n+16*Np]; - f17A = distA[nr17]; - f17B = distB[nr17]; - - // q=18 - nr18 = neighborList[n+17*Np]; - f18A = distA[nr18]; - f18B = distB[nr18]; - //---------------------------------------------------------------------// - - // Compute SC fluid-fluid interaction force - GffA_x = -Gsc*rhoB_gradx; - GffA_y = -Gsc*rhoB_grady; - GffA_z = -Gsc*rhoB_gradz; - GffB_x = -Gsc*rhoA_gradx; - GffB_y = -Gsc*rhoA_grady; - GffB_z = -Gsc*rhoA_gradz; - // Compute SC fluid-solid force - GfsA_x = SolidForceA[n+0*Np]; - GfsA_y = SolidForceA[n+1*Np]; - GfsA_z = SolidForceA[n+2*Np]; - GfsB_x = SolidForceB[n+0*Np]; - GfsB_y = SolidForceB[n+1*Np]; - GfsB_z = SolidForceB[n+2*Np]; - - // Compute greyscale related parameters - // ------------------- Fluid Component A -----------------------// - jxA = f1A-f2A+f7A-f8A+f9A-f10A+f11A-f12A+f13A-f14A; - jyA = f3A-f4A+f7A-f8A-f9A+f10A+f15A-f16A+f17A-f18A; - jzA = f5A-f6A+f11A-f12A-f13A+f14A+f15A-f16A-f17A+f18A; - - c0 = 0.5*(1.0+porosity*0.5*muA_eff/permA); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permA); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxA/rhoA+0.5*(porosity*Gx+GffA_x+GfsA_x); - vy = jyA/rhoA+0.5*(porosity*Gy+GffA_y+GfsA_y); - vz = jzA/rhoA+0.5*(porosity*Gz+GffA_z+GfsA_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_A = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_A = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_A = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_A*ux_A+uy_A*uy_A+uz_A*uz_A); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxA = rhoA*(-porosity*muA_eff/permA*ux_A - porosity*GeoFun/sqrt(permA)*u_mag*ux_A + porosity*Gx + GffA_x + GfsA_x); - FyA = rhoA*(-porosity*muA_eff/permA*uy_A - porosity*GeoFun/sqrt(permA)*u_mag*uy_A + porosity*Gy + GffA_y + GfsA_y); - FzA = rhoA*(-porosity*muA_eff/permA*uz_A - porosity*GeoFun/sqrt(permA)*u_mag*uz_A + porosity*Gz + GffA_z + GfsA_z); - if (porosity==1.0){ - FxA=rhoA*(Gx + GffA_x + GfsA_x); - FyA=rhoA*(Gy + GffA_y + GfsA_y); - FzA=rhoA*(Gz + GffA_z + GfsA_z); - } - // ------------------- Fluid Component B -----------------------// - // Compute greyscale related parameters - jxB = f1B-f2B+f7B-f8B+f9B-f10B+f11B-f12B+f13B-f14B; - jyB = f3B-f4B+f7B-f8B-f9B+f10B+f15B-f16B+f17B-f18B; - jzB = f5B-f6B+f11B-f12B-f13B+f14B+f15B-f16B-f17B+f18B; - - c0 = 0.5*(1.0+porosity*0.5*muB_eff/permB); - if (porosity==1.0) c0 = 0.5;//i.e. apparent pore nodes - //GeoFun = 1.75/sqrt(150.0*porosity*porosity*porosity); - c1 = porosity*0.5*GeoFun/sqrt(permB); - if (porosity==1.0) c1 = 0.0;//i.e. apparent pore nodes - - vx = jxB/rhoB+0.5*(porosity*Gx+GffB_x+GfsB_x); - vy = jyB/rhoB+0.5*(porosity*Gy+GffB_y+GfsB_y); - vz = jzB/rhoB+0.5*(porosity*Gz+GffB_z+GfsB_z); - v_mag=sqrt(vx*vx+vy*vy+vz*vz); - ux_B = vx/(c0+sqrt(c0*c0+c1*v_mag)); - uy_B = vy/(c0+sqrt(c0*c0+c1*v_mag)); - uz_B = vz/(c0+sqrt(c0*c0+c1*v_mag)); - u_mag=sqrt(ux_B*ux_B+uy_B*uy_B+uz_B*uz_B); - - //Update the total force to include linear (Darcy) and nonlinear (Forchheimer) drags due to the porous medium - FxB = rhoB*(-porosity*muB_eff/permB*ux_B - porosity*GeoFun/sqrt(permB)*u_mag*ux_B + porosity*Gx + GffB_x + GfsB_x); - FyB = rhoB*(-porosity*muB_eff/permB*uy_B - porosity*GeoFun/sqrt(permB)*u_mag*uy_B + porosity*Gy + GffB_y + GfsB_y); - FzB = rhoB*(-porosity*muB_eff/permB*uz_B - porosity*GeoFun/sqrt(permB)*u_mag*uz_B + porosity*Gz + GffB_z + GfsB_z); - if (porosity==1.0){ - FxB=rhoB*(Gx + GffB_x + GfsB_x); - FyB=rhoB*(Gy + GffB_y + GfsB_y); - FzB=rhoB*(Gz + GffB_z + GfsB_z); - } - - // Calculate barycentric velocity of the fluid mixture - ux = (rhoA*ux_A+rhoB*ux_B)/(rhoA+rhoB); - uy = (rhoA*uy_A+rhoB*uy_B)/(rhoA+rhoB); - uz = (rhoA*uz_A+rhoB*uz_B)/(rhoA+rhoB); - - //..............carry out relaxation process............................................... - // ------------------- Fluid Component A -----------------------// - // q=0 - distA[n] = f0A*(1.0-rlx) + rlx*0.3333333333333333*rhoA*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 1 - distA[nr2] = f1A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q=2 - distA[nr1] = f2A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(-3. + (6.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 3 - distA[nr4] = f3A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 4 - distA[nr3] = f4A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. + (6.*uy)/porosity) + FzA*(0. - (3.*uz)/porosity)); - - // q = 5 - distA[nr6] = f5A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(3. + (6.*uz)/porosity)); - - // q = 6 - distA[nr5] = f6A*(1.0-rlx) + rlx*0.05555555555555555*rhoA*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(0. - (3.*uy)/porosity) + FzA*(-3. + (6.*uz)/porosity)); - - // q = 7 - distA[nr8] = f7A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 8 - distA[nr7] = f8A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyA*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 9 - distA[nr10] = f9A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyA*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 10 - distA[nr9] = f10A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - FzA*(0. - (3.*uz)/porosity)); - - // q = 11 - distA[nr12] = f11A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); - - // q = 12 - distA[nr11] = f12A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - FzA*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); - - // q = 13 - distA[nr14] = f13A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - FzA*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); - - // q= 14 - distA[nr13] = f14A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyA*(0. - (3.*uy)/porosity) + FxA*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); - - // q = 15 - distA[nr16] = f15A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); - - // q = 16 - distA[nr15] = f16A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - FzA*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 17 - distA[nr18] = f17A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - FzA*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 18 - distA[nr17] = f18A*(1.0-rlx) + rlx*0.027777777777777776*rhoA*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxA*(0. - (3.*ux)/porosity) + FyA*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - FzA*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - - - // ------------------- Fluid Component B -----------------------// - // q=0 - distB[n] = f0B*(1.0-rlx) + rlx*0.3333333333333333*rhoB*(1. - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - + 0.3333333333333333*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 1 - distB[nr2] = f1B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q=2 - distB[nr1] = f2B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*ux + (4.5*ux*ux)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(-3. + (6.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 3 - distB[nr4] = f3B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 4 - distB[nr3] = f4B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uy + (4.5*uy*uy)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. + (6.*uy)/porosity) + FzB*(0. - (3.*uz)/porosity)); - - // q = 5 - distB[nr6] = f5B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 + 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(3. + (6.*uz)/porosity)); - - // q = 6 - distB[nr5] = f6B*(1.0-rlx) + rlx*0.05555555555555555*rhoB*(1 - 3.*uz + (4.5*uz*uz)/porosity - (1.5*(ux*ux+ uy*uy + uz*uz))/porosity) - +0.05555555555555555*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(0. - (3.*uy)/porosity) + FzB*(-3. + (6.*uz)/porosity)); - - // q = 7 - distB[nr8] = f7B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uy) + (4.5*(ux + uy)*(ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(ux + uy))/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 8 - distB[nr7] = f8B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uy) + (4.5*(-ux - uy)*(-ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uy))/porosity) + FyB*(-3. - (9.*(-ux - uy))/porosity - (3.*uy)/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 9 - distB[nr10] = f9B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uy) + (4.5*(ux - uy)*(ux - uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(3. - (3.*ux)/porosity + (9.*(ux - uy))/porosity) + FyB*(-3. - (9.*(ux - uy))/porosity - (3.*uy)/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 10 - distB[nr9] = f10B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uy) + (4.5*(-ux + uy)*(-ux + uy))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uy))/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(-ux + uy))/porosity) + - FzB*(0. - (3.*uz)/porosity)); - - // q = 11 - distB[nr12] = f11B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux + uz) + (4.5*(ux + uz)*(ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(ux + uz))/porosity)); - - // q = 12 - distB[nr11] = f12B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux - uz) + (4.5*(-ux - uz)*(-ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux - uz))/porosity) + - FzB*(-3. - (9.*(-ux - uz))/porosity - (3.*uz)/porosity)); - - // q = 13 - distB[nr14] = f13B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(ux - uz) + (4.5*(ux - uz)*(ux - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(3. - (3.*ux)/porosity + (9.*(ux - uz))/porosity) + - FzB*(-3. - (9.*(ux - uz))/porosity - (3.*uz)/porosity)); - - // q= 14 - distB[nr13] = f14B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-ux + uz) + (4.5*(-ux + uz)*(-ux + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FyB*(0. - (3.*uy)/porosity) + FxB*(-3. - (3.*ux)/porosity - (9.*(-ux + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(-ux + uz))/porosity)); - - // q = 15 - distB[nr16] = f15B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy + uz) + (4.5*(uy + uz)*(uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(uy + uz))/porosity)); - - // q = 16 - distB[nr15] = f16B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy - uz) + (4.5*(-uy - uz)*(-uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy - uz))/porosity) + - FzB*(-3. - (9.*(-uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 17 - distB[nr18] = f17B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(uy - uz) + (4.5*(uy - uz)*(uy - uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(3. - (3.*uy)/porosity + (9.*(uy - uz))/porosity) + - FzB*(-3. - (9.*(uy - uz))/porosity - (3.*uz)/porosity)); - - // q = 18 - distB[nr17] = f18B*(1.0-rlx) + rlx*0.027777777777777776*rhoB*(1 + 3.*(-uy + uz) + (4.5*(-uy + uz)*(-uy + uz))/porosity - (1.5*(ux*ux + uy*uy + uz*uz))/porosity) - +0.027777777777777776*(1. - 0.5*rlx)*(FxB*(0. - (3.*ux)/porosity) + FyB*(-3. - (3.*uy)/porosity - (9.*(-uy + uz))/porosity) + - FzB*(3. - (3.*uz)/porosity + (9.*(-uy + uz))/porosity)); - - - //Update velocity on device - Velocity[0*Np+n] = ux; - Velocity[1*Np+n] = uy; - Velocity[2*Np+n] = uz; - //Update pressure on device - Pressure[n] = (rhoA+rhoB+Gsc*rhoA*rhoB)/3.0; - } - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - int n; - int ijk; - double vx,vy,vz,v_mag; - double ux_A,uy_A,uz_A,ux_B,uy_B,uz_B,u_mag; - double ux,uy,uz; - double rhoA,rhoB; - double jxA,jyA,jzA; - double jxB,jyB,jzB; - // distribution functions - double f0A,f1A,f2A,f3A,f4A,f5A,f6A,f7A,f8A,f9A,f10A,f11A,f12A,f13A,f14A,f15A,f16A,f17A,f18A; - double f0B,f1B,f2B,f3B,f4B,f5B,f6B,f7B,f8B,f9B,f10B,f11B,f12B,f13B,f14B,f15B,f16B,f17B,f18B; - double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002) - double porosity; - double perm;//voxel permeability - double permA,permB;//effective relative perm - double c0, c1; //Guo's model parameters - double muA_eff = (tauA_eff-0.5)/3.0;//kinematic viscosity - double muB_eff = (tauB_eff-0.5)/3.0;//kinematic viscosity - double FxA, FyA, FzA;//The total body force including Brinkman force and user-specified (Gx,Gy,Gz) - double FxB, FyB, FzB; - double tau,rlx; - double phi;//phase field indicator - double rhoA_gradx,rhoA_grady,rhoA_gradz; - double rhoB_gradx,rhoB_grady,rhoB_gradz; - double GffA_x,GffA_y,GffA_z; - double GfsA_x,GfsA_y,GfsA_z; - double GffB_x,GffB_y,GffB_z; - double GfsB_x,GfsB_y,GfsB_z; - - int S = Np/NBLOCKS/NTHREADS + 1; - for (int s=0; s>>(Map,distA,distB,DenA,DenB,Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_Density<<>>(NeighborList, Map, distA, distB, DenA, DenB, start, finish, Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_Density: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_Density<<>>(Map,distA, distB, DenA, DenB, start, finish, Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_Density: %s \n",cudaGetErrorString(err)); - } -} - - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Map, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT<<>>(neighborList,Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, - tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT<<>>(Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, - tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK<<>>(neighborList,Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, - tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map,double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, - double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, - double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz, - int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK<<>>(Map,distA,distB,DenA,DenB,DenGradA,DenGradB,SolidForceA,SolidForceB,Poros,Perm,Velocity,Pressure, - tauA,tauB,tauA_eff,tauB_eff,Gsc,Gx,Gy,Gz,start,finish,Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np){ - - dvc_ScaLBL_D3Q19_GreyscaleSC_Gradient<<>>(neighborList, Map, Den, DenGrad, strideY, strideZ,start, finish, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyscaleSC_Gradient: %s \n",cudaGetErrorString(err)); - } -} - - -extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_BC_z<<>>(list, Map, DenA, DenB, vA, vB, count); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_BC_z: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_BC_Z<<>>(list, Map, DenA, DenB, vA, vB, count); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_BC_Z: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z<<>>(list, distA, distB, dinA, dinB, count, N); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z<<>>(list, distA, distB, doutA, doutB, count, N); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z<<>>(neighborList, list, distA, distB, dinA, dinB, count, N); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N){ - int GRID = count / 512 + 1; - dvc_ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z<<>>(neighborList, list, distA, distB, doutA, doutB, count, N); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); - } -} diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 49f1e14d..6525c09c 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -1,5 +1,5 @@ /* -color lattice boltzmann model +Two-fluid greyscale color lattice boltzmann model */ #include "models/GreyscaleColorModel.h" #include "analysis/distance.h" @@ -9,7 +9,7 @@ color lattice boltzmann model #include #include -ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM): +ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),greyMode(0),Lx(0),Ly(0),Lz(0),comm(COMM) @@ -56,8 +56,6 @@ void ScaLBL_GreyscaleColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, do File.close(); } */ - - void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ // read the input database db = std::make_shared( filename ); @@ -139,21 +137,21 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ // Override user-specified boundary condition for specific protocols auto protocol = greyscaleColor_db->getWithDefault( "protocol", "none" ); if (protocol == "seed water"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n"); } domain_db->putScalar( "BC", BoundaryCondition ); } else if (protocol == "open connected oil"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n"); } domain_db->putScalar( "BC", BoundaryCondition ); } else if (protocol == "shell aggregation"){ - if (BoundaryCondition != 0 ){ + if (BoundaryCondition != 0 && BoundaryCondition != 5){ BoundaryCondition = 0; if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n"); } @@ -176,9 +174,9 @@ void ScaLBL_GreyscaleColorModel::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object - comm.barrier(); + MPI_Barrier(comm); Dm->CommInit(); - comm.barrier(); + MPI_Barrier(comm); // Read domain parameters rank = Dm->rank(); nprocx = Dm->nprocx(); @@ -201,12 +199,12 @@ void ScaLBL_GreyscaleColorModel::ReadInput(){ } else if (domain_db->keyExists( "GridFile" )){ // Read the local domain data - auto input_id = readMicroCT( *domain_db, comm ); + auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD ); // Fill the halo (assuming GCW of 1) array size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) }; ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz }; ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 ); - fillHalo fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); + fillHalo fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 ); Array id_view; id_view.viewRaw( size1, Mask->id ); fill.copy( input_id, id_view ); @@ -315,7 +313,7 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels() for (int i=0; iid[i] = Mask->id[i]; for (size_t idx=0; idxComm.sumReduce( label_count[idx] ); + label_count_global[idx] = sumReduce( Dm->Comm, label_count[idx]); if (rank==0){ printf("Number of component labels: %lu \n",NLABELS); @@ -329,6 +327,7 @@ void ScaLBL_GreyscaleColorModel::AssignComponentLabels() ScaLBL_CopyToDevice(Phi, phase, N*sizeof(double)); ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); delete [] phase; } @@ -586,7 +585,7 @@ void ScaLBL_GreyscaleColorModel::AssignGreyPoroPermLabels() // Set Dm to match Mask for (int i=0; iid[i] = Mask->id[i]; - for (int idx=0; idxComm.sumReduce(label_count[idx]); + for (int idx=0; idxComm, label_count[idx]); //Initialize a weighted porosity after considering grey voxels GreyPorosity=0.0; @@ -597,7 +596,7 @@ void ScaLBL_GreyscaleColorModel::AssignGreyPoroPermLabels() if (rank==0){ printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); - printf("Component labels: %lu \n",NLABELS); + printf("Number of component labels: %lu \n",NLABELS); for (unsigned int idx=0; idxMemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - comm.barrier(); + MPI_Barrier(comm); //........................................................................... // MAIN VARIABLES ALLOCATED HERE @@ -707,8 +705,8 @@ void ScaLBL_GreyscaleColorModel::Create(){ // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - - // initialize phi based on PhaseLabel (include solid component labels) + + // initialize phi based on PhaseLabel (include solid component labels) AssignComponentLabels();//do open/black/grey nodes initialization if (greyMode==true){ AssignGreySolidLabels(); @@ -716,7 +714,6 @@ void ScaLBL_GreyscaleColorModel::Create(){ } } - void ScaLBL_GreyscaleColorModel::Initialize(){ if (rank==0) printf ("Initializing distributions \n"); @@ -782,14 +779,15 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_CopyToDevice(Phi,cPhi,N*sizeof(double)); ScaLBL_DeviceBarrier(); - comm.barrier(); + MPI_Barrier(comm); } if (rank==0) printf ("Initializing phase field \n"); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + // establish reservoirs for external bC + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4 ){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -812,6 +810,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ int IMAGE_COUNT = 0; std::vector ImageList; bool SET_CAPILLARY_NUMBER = false; + bool RESCALE_FORCE = false; bool MORPH_ADAPT = false; bool USE_MORPH = false; bool USE_SEED = false; @@ -820,6 +819,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine int MIN_STEADY_TIMESTEPS = 100000; int MAX_STEADY_TIMESTEPS = 200000; + int RESCALE_FORCE_AFTER_TIMESTEP = 0; int RAMP_TIMESTEPS = 0;//50000; // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in) int CURRENT_MORPH_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) int CURRENT_STEADY_TIMESTEPS=0; // counter for number of timesteps spent in morphological adaptation routine (reset each time) @@ -834,17 +834,13 @@ void ScaLBL_GreyscaleColorModel::Run(){ double initial_volume = 0.0; double delta_volume = 0.0; double delta_volume_target = 0.0; - double RESIDUAL_ENDPOINT_THRESHOLD = 0.04; - double NOISE_THRESHOLD = 0.0; - double BUMP_RATE = 2.0; - bool USE_BUMP_RATE = false; /* history for morphological algoirthm */ - double KRA_MORPH_FACTOR=0.8; + double KRA_MORPH_FACTOR=0.5; double volA_prev = 0.0; double log_krA_prev = 1.0; double log_krA_target = 1.0; - double log_krA = 0.0; + double log_krA = 1.0; double slope_krA_volume = 0.0; if (greyscaleColor_db->keyExists( "vol_A_previous" )){ volA_prev = greyscaleColor_db->getScalar( "vol_A_previous" ); @@ -868,46 +864,33 @@ void ScaLBL_GreyscaleColorModel::Run(){ USE_MORPH = true; } else if (protocol == "seed water"){ - morph_delta = 0.05; + morph_delta = -0.05; seed_water = 0.01; USE_SEED = true; USE_MORPH = true; } else if (protocol == "open connected oil"){ - morph_delta = 0.05; + morph_delta = -0.05; USE_MORPH = true; USE_MORPHOPEN_OIL = true; } else if (protocol == "shell aggregation"){ - morph_delta = 0.05; + morph_delta = -0.05; USE_MORPH = true; } - - if (greyscaleColor_db->keyExists( "residual_endpoint_threshold" )){ - RESIDUAL_ENDPOINT_THRESHOLD = greyscaleColor_db->getScalar( "residual_endpoint_threshold" ); - } - NULL_USE( RESIDUAL_ENDPOINT_THRESHOLD ); - if (greyscaleColor_db->keyExists( "noise_threshold" )){ - NOISE_THRESHOLD = greyscaleColor_db->getScalar( "noise_threshold" ); - USE_BUMP_RATE = true; - } - if (greyscaleColor_db->keyExists( "bump_rate" )){ - BUMP_RATE = greyscaleColor_db->getScalar( "bump_rate" ); - USE_BUMP_RATE = true; - } if (greyscaleColor_db->keyExists( "capillary_number" )){ capillary_number = greyscaleColor_db->getScalar( "capillary_number" ); SET_CAPILLARY_NUMBER=true; - //RESCALE_FORCE_MAX = 1; } -// if (analysis_db->keyExists( "rescale_force_count" )){ -// RESCALE_FORCE_MAX = analysis_db->getScalar( "rescale_force_count" ); -// } + if (greyscaleColor_db->keyExists( "rescale_force_after_timestep" )){ + RESCALE_FORCE_AFTER_TIMESTEP = greyscaleColor_db->getScalar( "rescale_force_after_timestep" ); + RESCALE_FORCE = true; + } if (greyscaleColor_db->keyExists( "timestep" )){ timestep = greyscaleColor_db->getScalar( "timestep" ); } - if (BoundaryCondition != 0 && SET_CAPILLARY_NUMBER==true){ - if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 \n"); + if (BoundaryCondition != 0 && BoundaryCondition != 5 && SET_CAPILLARY_NUMBER==true){ + if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 or 5 \n"); SET_CAPILLARY_NUMBER=false; } if (analysis_db->keyExists( "seed_water" )){ @@ -984,8 +967,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); - comm.barrier(); - starttime = Utilities::MPI::time(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ @@ -1010,7 +993,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Perform the collision operation ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - if (BoundaryCondition > 0){ + if (BoundaryCondition > 0 && BoundaryCondition < 5){ ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } @@ -1042,6 +1025,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } if (greyMode==true){ //Model-1&4 ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, @@ -1056,8 +1043,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); // *************EVEN TIMESTEP************* timestep++; @@ -1071,7 +1058,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ // Perform the collision operation ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL // Halo exchange for phase field - if (BoundaryCondition > 0){ + if (BoundaryCondition > 0 && BoundaryCondition < 5){ ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } @@ -1102,6 +1089,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } + else if (BoundaryCondition == 5){ + ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); + ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); + } if (greyMode==true){ //Model-1&4 ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, @@ -1116,19 +1107,17 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } - ScaLBL_DeviceBarrier(); - comm.barrier(); + ScaLBL_DeviceBarrier(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ - PROFILE_STOP("Update"); - if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){ + if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ printf("%i %f \n",timestep,din); } // Run the analysis analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); - // allow initial ramp-up to get closer to steady state if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ analysis.finish(); @@ -1138,7 +1127,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ double volA = Averages->gnb.V; volA /= Dm->Volume; volB /= Dm->Volume;; - initial_volume = volA*Dm->Volume; + //initial_volume = volA*Dm->Volume; double vA_x = Averages->gnb.Px/Averages->gnb.M; double vA_y = Averages->gnb.Py/Averages->gnb.M; double vA_z = Averages->gnb.Pz/Averages->gnb.M; @@ -1163,20 +1152,6 @@ void ScaLBL_GreyscaleColorModel::Run(){ double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); - if (SET_CAPILLARY_NUMBER && CURRENT_STEADY_TIMESTEPS%MIN_STEADY_TIMESTEPS < analysis_interval ){ - Fx *= capillary_number / Ca; - Fy *= capillary_number / Ca; - Fz *= capillary_number / Ca; - if (force_mag > 1e-3){ - Fx *= 1e-3/force_mag; // impose ceiling for stability - Fy *= 1e-3/force_mag; - Fz *= 1e-3/force_mag; - } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); - greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); - } - if ( morph_timesteps > morph_interval ){ bool isSteady = false; @@ -1184,20 +1159,49 @@ void ScaLBL_GreyscaleColorModel::Run(){ isSteady = true; if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS) isSteady = true; - + if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){ + RESCALE_FORCE = false; + double RESCALE_FORCE_FACTOR = capillary_number / Ca; + if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0; + if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5; + Fx *= RESCALE_FORCE_FACTOR; + Fy *= RESCALE_FORCE_FACTOR; + Fz *= RESCALE_FORCE_FACTOR; + force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + if (force_mag > 1e-3){ + Fx *= 1e-3/force_mag; // impose ceiling for stability + Fy *= 1e-3/force_mag; + Fz *= 1e-3/force_mag; + } + if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); + } if ( isSteady ){ MORPH_ADAPT = true; CURRENT_MORPH_TIMESTEPS=0; - //delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change - /** morphological target based on relative permeability for A **/ + delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change + //****** ENDPOINT ADAPTATION ********/ double krA_TMP= fabs(muA*flow_rate_A / force_mag); + double krB_TMP= fabs(muB*flow_rate_B / force_mag); log_krA = log(krA_TMP); - log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); - slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); - delta_volume_target=Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume); + if (krA_TMP < 0.0){ + // cannot do endpoint adaptation if kr is negative + log_krA = log_krA_prev; + } + else if (krA_TMP < krB_TMP && morph_delta > 0.0){ + /** morphological target based on relative permeability for A **/ + log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP)); + slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); + delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); + if (rank==0){ + printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); + printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + } + } log_krA_prev = log_krA; volA_prev = volA; - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + //******************************** **/ /** compute averages & write data **/ Averages->Full(); Averages->Write(timestep); @@ -1213,8 +1217,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ double pB = Averages->gwb.p; double pAc = Averages->gnc.p; double pBc = Averages->gwc.p; - double pAB = (pA-pB)/(h*5.796*alpha); - double pAB_connected = (pAc-pBc)/(h*5.796*alpha); + double pAB = (pA-pB)/(h*6.0*alpha); + double pAB_connected = (pAc-pBc)/(h*6.0*alpha); // connected contribution double Vol_nc = Averages->gnc.V/Dm->Volume; double Vol_wc = Averages->gwc.V/Dm->Volume; @@ -1277,16 +1281,6 @@ void ScaLBL_GreyscaleColorModel::Run(){ Fy *= 1e-3/force_mag; Fz *= 1e-3/force_mag; } - if (flow_rate_A < NOISE_THRESHOLD && USE_BUMP_RATE){ - if (rank==0) printf("Hit noise threshold (%f): bumping capillary number by %f X \n",NOISE_THRESHOLD,BUMP_RATE); - Fx *= BUMP_RATE; // impose bump condition - Fy *= BUMP_RATE; - Fz *= BUMP_RATE; - capillary_number *= BUMP_RATE; - greyscaleColor_db->putScalar("capillary_number",capillary_number); - current_db->putDatabase("Color", greyscaleColor_db); - MORPH_ADAPT = false; // re-run current point if below noise threshold - } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); @@ -1314,7 +1308,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ std::string next_image = ImageList[IMAGE_INDEX]; if (rank==0) printf("***Loading next image in sequence (%i) ***\n",IMAGE_INDEX); greyscaleColor_db->putScalar("image_index",IMAGE_INDEX); - //ImageInit(next_image); + ImageInit(next_image); } else{ if (rank==0) printf("Finished simulating image sequence \n"); @@ -1343,42 +1337,30 @@ void ScaLBL_GreyscaleColorModel::Run(){ CURRENT_STEADY_TIMESTEPS=0; initial_volume = volA*Dm->Volume; delta_volume = 0.0; - if (USE_DIRECT){ - //BoundaryCondition = 0; - //ScaLBL_Comm->BoundaryCondition = 0; - //ScaLBL_Comm_Regular->BoundaryCondition = 0; - //Fx = capillary_number*dir_x*force_mag / Ca; - //Fy = capillary_number*dir_y*force_mag / Ca; - //Fz = capillary_number*dir_z*force_mag / Ca; - } + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; } else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) { MORPH_ADAPT = false; CURRENT_STEADY_TIMESTEPS=0; initial_volume = volA*Dm->Volume; delta_volume = 0.0; + RESCALE_FORCE = true; + if (RESCALE_FORCE_AFTER_TIMESTEP > 0) + RESCALE_FORCE = true; } - if ( REVERSE_FLOW_DIRECTION ){ - //if (rank==0) printf("*****REVERSE FLOW DIRECTION***** \n"); - delta_volume = 0.0; - // flow direction will reverse after next steady point - MORPH_ADAPT = false; - CURRENT_STEADY_TIMESTEPS=0; - //morph_delta *= (-1.0); - REVERSE_FLOW_DIRECTION = false; - } - comm.barrier(); } morph_timesteps += analysis_interval; } + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ ScaLBL_DeviceBarrier(); - comm.barrier(); - stoptime = Utilities::MPI::time(); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + stoptime = MPI_Wtime(); if (rank==0) printf("-------------------------------------------------------------------\n"); // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; @@ -1395,52 +1377,47 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ************************************************************************ } -//double ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ -// -// if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); -// Mask->Decomp(Filename); -// for (int i=0; iid[i]; // save what was read -// for (int i=0; iid[i] = Mask->id[i]; // save what was read -// -// //double *PhaseLabel; -// //PhaseLabel = new double[Nx*Ny*Nz]; -// //AssignComponentLabels(PhaseLabel); -// AssignComponentLabels(); -// -// double Count = 0.0; -// double PoreCount = 0.0; -// for (int k=1; kComm.sumReduce( Count ); -// PoreCount = Dm->Comm.sumReduce( PoreCount ); -// -// if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); -// ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double)); -// comm.barrier(); -// -// ScaLBL_D3Q19_Init(fq, Np); -// ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); -// ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); -// comm.barrier(); -// -// ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); -// -// double saturation = Count/PoreCount; -// return saturation; -// -//} +double ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ + + if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); + Mask->Decomp(Filename); + for (int i=0; iid[i]; // save what was read + for (int i=0; iid[i] = Mask->id[i]; // save what was read + + AssignComponentLabels(); + + double Count = 0.0; + double PoreCount = 0.0; + for (int k=1; kComm, Count); + PoreCount=sumReduce( Dm->Comm, PoreCount); + + if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); + + ScaLBL_D3Q19_Init(fq, Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); + + ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); + + double saturation = Count/PoreCount; + return saturation; + +} double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_change){ @@ -1465,7 +1442,7 @@ double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_chang BlobIDstruct new_index; double vF=0.0; double vS=0.0; ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); - Dm->Comm.barrier(); + MPI_Barrier(Dm->Comm); long long count_connected=0; long long count_porespace=0; @@ -1487,9 +1464,9 @@ double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_chang } } } - count_connected = Dm->Comm.sumReduce( count_connected); - count_porespace = Dm->Comm.sumReduce( count_porespace); - count_water = Dm->Comm.sumReduce( count_water); + count_connected=sumReduce( Dm->Comm, count_connected); + count_porespace=sumReduce( Dm->Comm, count_porespace); + count_water=sumReduce( Dm->Comm, count_water); for (int k=0; kComm.sumReduce( count_morphopen); + count_morphopen=sumReduce( Dm->Comm, count_morphopen); volume_change = double(count_morphopen - count_connected); if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); @@ -1569,7 +1546,7 @@ double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_chang ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -1584,194 +1561,89 @@ double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_chang } return(volume_change); } - double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ - srand(time(NULL)); - double mass_loss =0.f; - double count =0.f; - double *Aq_tmp, *Bq_tmp; - double *Vel_tmp; - - Aq_tmp = new double [7*Np]; - Bq_tmp = new double [7*Np]; - Vel_tmp = new double [3*Np]; + srand(time(NULL)); + double mass_loss =0.f; + double count =0.f; + double *Aq_tmp, *Bq_tmp; + + Aq_tmp = new double [7*Np]; + Bq_tmp = new double [7*Np]; - ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); - ScaLBL_CopyToHost(Vel_tmp, Velocity, 7*Np*sizeof(double)); - - //Extract averged velocity - double vx_glb = (Averages->gnb.Px+Averages->gwb.Px)/(Averages->gnb.M+Averages->gwb.M); - double vy_glb = (Averages->gnb.Py+Averages->gwb.Py)/(Averages->gnb.M+Averages->gwb.M); - double vz_glb = (Averages->gnb.Pz+Averages->gwb.Pz)/(Averages->gnb.M+Averages->gwb.M); - double v_mag_glb = sqrt(vx_glb*vx_glb+vy_glb*vy_glb+vz_glb*vz_glb); + ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); + ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); + + + for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } - for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ - double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); - double weight = (v_mag_local 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - } - mass_loss += random_value*seed_water_in_oil; - } + for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ + double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; + double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; + double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; + double phase_id = (dA - dB) / (dA + dB); + if (phase_id > 0.0){ + Aq_tmp[n] -= 0.3333333333333333*random_value; + Aq_tmp[n+Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; + Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; + + Bq_tmp[n] += 0.3333333333333333*random_value; + Bq_tmp[n+Np] += 0.1111111111111111*random_value; + Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; + Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; + } + mass_loss += random_value*seed_water_in_oil; + } - for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ - double v_mag_local = sqrt(Vel_tmp[n]*Vel_tmp[n]+Vel_tmp[n+1*Np]*Vel_tmp[n+1*Np]+Vel_tmp[n+2*Np]*Vel_tmp[n+2*Np]); - double weight = (v_mag_local 0.0){ - Aq_tmp[n] -= 0.3333333333333333*random_value; - Aq_tmp[n+Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; - Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; - - Bq_tmp[n] += 0.3333333333333333*random_value; - Bq_tmp[n+Np] += 0.1111111111111111*random_value; - Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; - Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; - } - mass_loss += random_value*seed_water_in_oil; - } + count= sumReduce( Dm->Comm, count); + mass_loss= sumReduce( Dm->Comm, mass_loss); + if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); - count = Dm->Comm.sumReduce( count ); - mass_loss = Dm->Comm.sumReduce( mass_loss ); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + // Need to initialize Aq, Bq, Den, Phi directly + //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); + ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); - // Need to initialize Aq, Bq, Den, Phi directly - //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); - ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); - - return(mass_loss); + return(mass_loss); } -//double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ -// srand(time(NULL)); -// double mass_loss =0.f; -// double count =0.f; -// double *Aq_tmp, *Bq_tmp; -// -// Aq_tmp = new double [7*Np]; -// Bq_tmp = new double [7*Np]; -// -// ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double)); -// ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double)); -// -///* for (int k=1; kSDs(i,j,k) < 0.f){ -// // skip -// } -// else if (phase(i,j,k) > 0.f ){ -// phase(i,j,k) -= random_value*seed_water_in_oil; -// mass_loss += random_value*seed_water_in_oil; -// count++; -// } -// else { -// -// } -// } -// } -// } -// */ -// for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){ -// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; -// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; -// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; -// double phase_id = (dA - dB) / (dA + dB); -// if (phase_id > 0.0){ -// Aq_tmp[n] -= 0.3333333333333333*random_value; -// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; -// -// Bq_tmp[n] += 0.3333333333333333*random_value; -// Bq_tmp[n+Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; -// } -// mass_loss += random_value*seed_water_in_oil; -// } -// -// for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){ -// double random_value = seed_water_in_oil*double(rand())/ RAND_MAX; -// double dA = Aq_tmp[n] + Aq_tmp[n+Np] + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np]; -// double dB = Bq_tmp[n] + Bq_tmp[n+Np] + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np]; -// double phase_id = (dA - dB) / (dA + dB); -// if (phase_id > 0.0){ -// Aq_tmp[n] -= 0.3333333333333333*random_value; -// Aq_tmp[n+Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value; -// Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value; -// -// Bq_tmp[n] += 0.3333333333333333*random_value; -// Bq_tmp[n+Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+2*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+3*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+4*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+5*Np] += 0.1111111111111111*random_value; -// Bq_tmp[n+6*Np] += 0.1111111111111111*random_value; -// } -// mass_loss += random_value*seed_water_in_oil; -// } -// -// count = Dm->Comm.sumReduce( count ); -// mass_loss = Dm->Comm.sumReduce( mass_loss ); -// if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); -// -// // Need to initialize Aq, Bq, Den, Phi directly -// //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double)); -// -// return(mass_loss); -//} - double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double target_delta_volume){ const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); double vF = 0.f; double vS = 0.f; double delta_volume; + double WallFactor = 0.0; + bool USE_CONNECTED_NWP = false; DoubleArray phase(Nx,Ny,Nz); IntArray phase_label(Nx,Ny,Nz);; @@ -1792,7 +1664,7 @@ double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double tar } } } - double volume_initial = Dm->Comm.sumReduce( count); + double volume_initial = sumReduce( Dm->Comm, count); /* sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); FILE *INPUT = fopen(LocalRankFilename,"wb"); @@ -1800,32 +1672,56 @@ double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double tar fclose(INPUT); */ // 2. Identify connected components of phase field -> phase_label - BlobIDstruct new_index; - ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - comm.barrier(); - // only operate on component "0" - count = 0.0; - double second_biggest = 0.0; + double volume_connected = 0.0; + double second_biggest = 0.0; + if (USE_CONNECTED_NWP){ + BlobIDstruct new_index; + ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); + MPI_Barrier(Dm->Comm); - for (int k=0; kComm, count); + second_biggest = sumReduce( Dm->Comm, second_biggest); + } + else { + // use the whole NWP + for (int k=0; kSDs(i,j,k) > 0.f){ + if (phase(i,j,k) > 0.f ){ + phase_id(i,j,k) = 0; + } + else { + phase_id(i,j,k) = 1; + } + } + else { + phase_id(i,j,k) = 1; + } } } } - } - double volume_connected = Dm->Comm.sumReduce( count ); - second_biggest = Dm->Comm.sumReduce( second_biggest ); + } /*int reach_x, reach_y, reach_z; for (int k=0; k 0.01*volume_initial) target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental); + delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); for (int k=0; kComm.sumReduce( count ); + double volume_final= sumReduce( Dm->Comm, count); delta_volume = (volume_final-volume_initial); if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); @@ -1935,7 +1834,7 @@ double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double tar // 7. Re-initialize phase field and density ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition >0 ){ + if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); @@ -2039,27 +1938,27 @@ void ScaLBL_GreyscaleColorModel::WriteDebug(){ fwrite(PhaseField.data(),8,N,GreySG_Z_FILE); fclose(GreySG_Z_FILE); -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); -// FILE *CGX_FILE; -// sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); -// CGX_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGX_FILE); -// fclose(CGX_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); -// FILE *CGY_FILE; -// sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); -// CGY_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGY_FILE); -// fclose(CGY_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); -// FILE *CGZ_FILE; -// sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); -// CGZ_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,CGZ_FILE); -// fclose(CGZ_FILE); +/* ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField); + FILE *CGX_FILE; + sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank); + CGX_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGX_FILE); + fclose(CGX_FILE); + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField); + FILE *CGY_FILE; + sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank); + CGY_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGY_FILE); + fclose(CGY_FILE); + + ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField); + FILE *CGZ_FILE; + sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank); + CGZ_FILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,CGZ_FILE); + fclose(CGZ_FILE); +*/ } //void ScaLBL_GreyscaleColorModel::AssignGreySolidLabels()//Model-1 diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 9d5a6a65..d7dde941 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -1,5 +1,5 @@ /* -Implementation of color lattice boltzmann model +Implementation of two-fluid greyscale color lattice boltzmann model */ #include #include @@ -12,13 +12,13 @@ Implementation of color lattice boltzmann model #include "common/Communication.h" #include "analysis/TwoPhase.h" #include "analysis/runAnalysis.h" -#include "common/MPI.h" +#include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" class ScaLBL_GreyscaleColorModel{ public: - ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM); + ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM); ~ScaLBL_GreyscaleColorModel(); // functions in they should be run @@ -67,7 +67,7 @@ public: double *fq, *Aq, *Bq; double *Den, *Phi; //double *GreySolidPhi; //Model 2 & 3 - double *GreySolidGrad;//Model 1 + double *GreySolidGrad;//Model 1 & 4 //double *ColorGrad; double *Velocity; double *Pressure; @@ -75,7 +75,7 @@ public: double *Permeability_dvc; private: - Utilities::MPI comm; + MPI_Comm comm; int dist_mem_size; int neighborSize; diff --git a/models/GreyscaleFEModel.cpp b/models/GreyscaleFEModel.cpp deleted file mode 100644 index 97ed0f98..00000000 --- a/models/GreyscaleFEModel.cpp +++ /dev/null @@ -1,1247 +0,0 @@ -/* -Greyscale lattice boltzmann model - */ -#include "models/GreyscaleFEModel.h" -#include "analysis/distance.h" -#include "analysis/morphology.h" -#include -#include - -template -void DeleteArray( const TYPE *p ) -{ - delete [] p; -} - -ScaLBL_GreyscaleFEModel::ScaLBL_GreyscaleFEModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0), -rhoA(0),rhoB(0),gamma(0),kappaA(0),kappaB(0),lambdaA(0),lambdaB(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) -{ - SignDist.resize(Nx,Ny,Nz); - SignDist.fill(0); - -} -ScaLBL_GreyscaleFEModel::~ScaLBL_GreyscaleFEModel(){ - -} - -void ScaLBL_GreyscaleFEModel::ReadParams(string filename){ - // read the input database - db = std::make_shared( filename ); - domain_db = db->getDatabase( "Domain" ); - greyscaleFE_db = db->getDatabase( "GreyscaleFE" ); - analysis_db = db->getDatabase( "Analysis" ); - vis_db = db->getDatabase( "Visualization" ); - - // set defaults - timestepMax = 100000; - tauA = 1.0; - tauB = 1.0; - tauA_eff = tauA;//the effective viscosity of the Darcy term - tauB_eff = tauB; - rhoA = 1.0;//constant molecular mass (after LB scaling) - rhoB = 1.0; - tolerance = 0.01; - Fx = Fy = Fz = 0.0; - Restart=false; - din=dout=1.0; - flux=0.0; - dp = 10.0; //unit of 'dp': voxel - //Gsc = 1.0; - gamma = 1.0;//may also have gammaA and gammaB; - kappaA = 1.0e-3; - kappaB = 1.0e-3; - lambdaA = 1.0e-3; - lambdaB = 1.0e-3; - - // ---------------------- Greyscale Model parameters -----------------------// - if (greyscaleFE_db->keyExists( "timestepMax" )){ - timestepMax = greyscaleFE_db->getScalar( "timestepMax" ); - } - if (greyscaleFE_db->keyExists( "tauA" )){ - tauA = greyscaleFE_db->getScalar( "tauA" ); - } - if (greyscaleFE_db->keyExists( "tauB" )){ - tauB = greyscaleFE_db->getScalar( "tauB" ); - } - tauA_eff = greyscaleFE_db->getWithDefault( "tauA_eff", tauA); - tauB_eff = greyscaleFE_db->getWithDefault( "tauB_eff", tauB); - if (greyscaleFE_db->keyExists( "rhoA" )){ - rhoA = greyscaleFE_db->getScalar( "rhoA" ); - } - if (greyscaleFE_db->keyExists( "rhoB" )){ - rhoB = greyscaleFE_db->getScalar( "rhoB" ); - } - if (greyscaleFE_db->keyExists( "gamma" )){ - gamma = greyscaleFE_db->getScalar( "gamma" ); - } - if (greyscaleFE_db->keyExists( "kappaA" )){ - kappaA = greyscaleFE_db->getScalar( "kappaA" ); - } - if (greyscaleFE_db->keyExists( "kappaB" )){ - kappaB = greyscaleFE_db->getScalar( "kappaB" ); - } - if (greyscaleFE_db->keyExists( "lambdaA" )){ - lambdaA = greyscaleFE_db->getScalar( "lambdaA" ); - } - if (greyscaleFE_db->keyExists( "lambdaB" )){ - lambdaB = greyscaleFE_db->getScalar( "lambdaB" ); - } - if (greyscaleFE_db->keyExists( "dp" )){ - dp = greyscaleFE_db->getScalar( "dp" ); - } - if (greyscaleFE_db->keyExists( "F" )){ - Fx = greyscaleFE_db->getVector( "F" )[0]; - Fy = greyscaleFE_db->getVector( "F" )[1]; - Fz = greyscaleFE_db->getVector( "F" )[2]; - } - if (greyscaleFE_db->keyExists( "Restart" )){ - Restart = greyscaleFE_db->getScalar( "Restart" ); - } - if (greyscaleFE_db->keyExists( "din" )){ - din = greyscaleFE_db->getScalar( "din" ); - } - if (greyscaleFE_db->keyExists( "dout" )){ - dout = greyscaleFE_db->getScalar( "dout" ); - } - if (greyscaleFE_db->keyExists( "flux" )){ - flux = greyscaleFE_db->getScalar( "flux" ); - } - if (greyscaleFE_db->keyExists( "tolerance" )){ - tolerance = greyscaleFE_db->getScalar( "tolerance" ); - } - //auto collision = greyscaleFE_db->getWithDefault( "collision", "IMRT" ); - //if (collision == "BGK"){ - // CollisionType=2; - //} - // ------------------------------------------------------------------------// - - //------------------------ Other Domain parameters ------------------------// - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } - // ------------------------------------------------------------------------// -} - -void ScaLBL_GreyscaleFEModel::SetDomain(){ - Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - // domain parameters - Nx = Dm->Nx; - Ny = Dm->Ny; - Nz = Dm->Nz; - Lx = Dm->Lx; - Ly = Dm->Ly; - Lz = Dm->Lz; - N = Nx*Ny*Nz; - - SignDist.resize(Nx,Ny,Nz); - Velocity_x.resize(Nx,Ny,Nz); - Velocity_y.resize(Nx,Ny,Nz); - Velocity_z.resize(Nx,Ny,Nz); - PorosityMap.resize(Nx,Ny,Nz); - Pressure.resize(Nx,Ny,Nz); - - id = new signed char [N]; - for (int i=0; iid[i] = 1; // initialize this way - MPI_Barrier(comm); - Dm->CommInit(); - MPI_Barrier(comm); - // Read domain parameters - rank = Dm->rank(); - nprocx = Dm->nprocx(); - nprocy = Dm->nprocy(); - nprocz = Dm->nprocz(); -} - -void ScaLBL_GreyscaleFEModel::ReadInput(){ - - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - - if (domain_db->keyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask->Decomp(Filename); - } - else{ - if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); - Mask->ReadIDs(); - } - for (int i=0; iid[i]; // save what was read - - // Generate the signed distance map - // Initialize the domain and communication - Array id_solid(Nx,Ny,Nz); - int count = 0; - // Solve for the position of the solid phase - for (int k=0;kid[n]; - if (label > 0) id_solid(i,j,k) = 1; - else id_solid(i,j,k) = 0; - } - } - } - // Initialize the signed distance function - for (int k=0;kid[nn] <= 0)||(Mask->id[nn]>=3)){ - double vec_x = double(ii-1); - double vec_y = double(jj-1); - double vec_z = double(kk-1); - double GWNS=SolidPotential[nn]; - phi_x += GWNS*weight*vec_x; - phi_y += GWNS*weight*vec_y; - phi_z += GWNS*weight*vec_z; - } - } - } - } - SolidForce[idx] = phi_x; - SolidForce[idx+Np] = phi_y; - SolidForce[idx+2*Np] = phi_z; - } - } - } - } - delete [] Dst; -} - - -void ScaLBL_GreyscaleFEModel::AssignComponentLabels(double *Porosity, double *Permeability, double *SolidPotential) -{ - size_t NLABELS=0; - signed char VALUE=0; - double POROSITY=0.f; - double PERMEABILITY=0.f; - double AFFINITY=0.f; - - auto LabelList = greyscaleFE_db->getVector( "ComponentLabels" ); - auto AffinityList = greyscaleFE_db->getVector( "ComponentAffinity" ); - auto PorosityList = greyscaleFE_db->getVector( "PorosityList" ); - auto PermeabilityList = greyscaleFE_db->getVector( "PermeabilityList" ); - - //1. Requirement for "ComponentLabels": - // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 - // *label = 1 and 2 are reserved for NW and W phase respectively. - //2. Requirement for "ComponentAffinity": - // *should be in the same length as "ComponentLabels" - // *could leave Affinity=0.0 for label=1 and 2 - //3. Requirement for "PorosityList": - // *for ComponentLables <=0, put porosity value = 0.0; - // *for ComponentLabels >=3, put the corresponding sub-resolution porosity - // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) - //4. Requirement for "PermeabilityList": - // *for ComponentLabels <=2, does not matter, can leave it as 1.0 - - NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size() || NLABELS != AffinityList.size() || NLABELS != PermeabilityList.size()){ - ERROR("Error: ComponentLabels, ComponentAffinity, PorosityList and PermeabilityList must all be the same length! \n"); - } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - - for (int idx=0; idx 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - POROSITY=PorosityList[idx]; - label_count[idx] += 1.0; - idx = NLABELS; - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (POROSITY<=0.0){ - ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); - } - else{ - Porosity[idx] = POROSITY; - } - } - } - } - } - - //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - PERMEABILITY=PermeabilityList[idx]; - idx = NLABELS; - //Mask->id[n] = 0; // set mask to zero since this is an immobile component - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (PERMEABILITY<=0.0){ - ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); - } - else{ - Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; - } - } - } - } - } - - //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase - for (int k=0;k=3){ - AFFINITY=AffinityList[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 - } - else{//i.e. label = 1 or 2 - AFFINITY=0.0; - } - idx = NLABELS; - } - } - //NOTE: node_ID = 1 and 2 are reserved - if ((VALUE == 1)||(VALUE == 2)) AFFINITY=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists - SolidPotential[n] = AFFINITY; - } - } - } - - - // Set Dm to match Mask - for (int i=0; iid[i] = Mask->id[i]; - - for (int idx=0; idxComm.sumReduce(label_count[idx]); - - //Initialize a weighted porosity after considering grey voxels - GreyPorosity=0.0; - for (unsigned int idx=0; idxvoxel_length); - printf("Component labels: %lu \n",NLABELS); - for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); - printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); - } - printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); - } -} - -void ScaLBL_GreyscaleFEModel::Density_and_Phase_Init(){ - - size_t NLABELS=0; - signed char VALUE=0; - - vector LabelList{1,2}; - vector SwList{0.0,1.0}; - - if (greyscaleFE_db->keyExists( "GreyNodeLabels" )){ - LabelList.clear(); - LabelList = greyscaleFE_db->getVector( "GreyNodeLabels" ); - } - if (greyscaleFE_db->keyExists( "GreyNodeSw" )){ - SwList.clear(); - SwList = greyscaleFE_db->getVector( "GreyNodeSw" ); - } - - NLABELS=LabelList.size(); - if (NLABELS != SwList.size()){ - ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); - } - -// double *Den_temp; -// Den_temp=new double [2*Np]; - double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. - double nB=0.5; - - double *Phi_temp; - Phi_temp=new double [Np]; - double phi = 0.0; - - for (int k=0; kid[n]; - if (VALUE>0){ - for (unsigned int idx=0; idx < NLABELS; idx++){ - if (VALUE == LabelList[idx]){ - double Sw = SwList[idx]; - if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); - nB=Sw; - nA=1.0-Sw; - phi = nA-nB; - idx = NLABELS; - } - } - if (VALUE==1){//label=1 reserved for NW phase - nA=1.0; - nB=0.0; - phi = nA-nB; - } - else if(VALUE==2){//label=2 reserved for W phase - nA=0.0; - nB=1.0; - phi = nA-nB; - } - int idx = Map(i,j,k); - //Den_temp[idx+0*Np] = nA; - //Den_temp[idx+1*Np] = nB; - Phi_temp[idx] = phi; - } - } - } - } - //copy to device - //ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); - ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - //delete [] Den_temp; - delete [] Phi_temp; -} - -void ScaLBL_GreyscaleFEModel::Create(){ - /* - * This function creates the variables needed to run a LBM - */ - //......................................................... - // don't perform computations at the eight corners - //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - - //......................................................... - // Initialize communication structures in averaging domain - for (int i=0; iid[i] = Mask->id[i]; - Mask->CommInit(); - Np=Mask->PoreCount(); - //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - // Create a communicator for the device (will use optimized layout) - // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); - - int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); - Map.resize(Nx,Ny,Nz); Map.fill(-2); - auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - MPI_Barrier(comm); - - //........................................................................... - // MAIN VARIABLES ALLOCATED HERE - //........................................................................... - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - //......................device distributions................................. - dist_mem_size = Np*sizeof(double); - neighborSize=18*(Np*sizeof(int)); - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Cq, 7*sizeof(double)*Np);//phase field distribution - ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &PressureGrad, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &PhiLap, sizeof(double)*Np);//laplacian of phase field - ScaLBL_AllocateDeviceMemory((void **) &SolidForce, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &PressTensor, 6*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &PressTensorGrad, 18*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &Den, 2*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &DenLapA, sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &DenLapB, sizeof(double)*Np); - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device neighbor list \n"); - fflush(stdout); - // copy the neighbor list - ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - // initialize phi based on PhaseLabel (include solid component labels) - double *Poros, *Perm; - Poros = new double[Np]; - Perm = new double[Np]; - double *SolidForce_host = new double[3*Np]; - double *SolidPotential_host = new double [Nx*Ny*Nz]; - AssignComponentLabels(Poros,Perm,SolidPotential_host); - AssignSolidForce(SolidPotential_host,SolidForce_host); - ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); - ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); - ScaLBL_CopyToDevice(SolidForce, SolidForce_host, 3*Np*sizeof(double)); - - ScaLBL_DeviceBarrier(); - //TODO make the following smart pointers - delete [] SolidForce_host; - delete [] SolidPotential_host; - delete [] Poros; - delete [] Perm; -} - - -void ScaLBL_GreyscaleFEModel::Initialize(){ - if (Restart == true){ - //TODO: Restart funtion is currently not working; need updates - if (rank==0){ - printf("Initializing density field and distributions from Restart! \n"); - } - // Read in the restart file to CPU buffers - std::shared_ptr cfq; - cfq = std::shared_ptr(new double[19*Np],DeleteArray); - std::shared_ptr cDen; - cDen = std::shared_ptr(new double[2*Np],DeleteArray); - FILE *File; - File=fopen(LocalRestartFile,"rb"); - fread(cfq.get(),sizeof(double),19*Np,File); - fread(cDen.get(),sizeof(double),2*Np,File); - fclose(File); - - // Copy the restart data to the GPU - ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); - ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - //TODO need proper initialization ! - - //TODO need to initialize velocity field ! - //this is required for calculating the pressure_dvc - //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity - } - else{ - if (rank==0) printf ("Initializing density field \n"); - Density_and_Phase_Init();//initialize density field - ScaLBL_D3Q19_GreyscaleFE_Laplacian(NeighborList, Phi, PhiLap, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q19_GreyscaleFE_Laplacian(NeighborList, Phi, PhiLap, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_GreyscaleFE_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, 0, ScaLBL_Comm->LastExterior(), Np);//initialize D3Q7 density components - ScaLBL_D3Q7_GreyscaleFE_Init(Phi, Cq, PhiLap, gamma,kappaA,kappaB,lambdaA,lambdaB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - - if (rank==0) printf ("Initializing distributions \n"); - //ScaLBL_D3Q19_GreyColorIMRT_Init(fq, Den, rhoA, rhoB, Np); - ScaLBL_D3Q19_Init(fq, Np); - - //Velocity also needs initialization (for old incompressible momentum transport) - //if (rank==0) printf ("Initializing velocity field \n"); - //double *vel_init; - //vel_init = new double [3*Np]; - //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; - //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); - //ScaLBL_DeviceBarrier(); - //delete [] vel_init; - } -} - -void ScaLBL_GreyscaleFEModel::Run(){ - int nprocs=nprocx*nprocy*nprocz; - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - int analysis_interval = 1000; // number of timesteps in between in situ analysis - int visualization_interval = 1000; - int restart_interval = 10000; // number of timesteps in between in saving distributions for restart - if (analysis_db->keyExists( "analysis_interval" )){ - analysis_interval = analysis_db->getScalar( "analysis_interval" ); - } - if (analysis_db->keyExists( "visualization_interval" )){ - visualization_interval = analysis_db->getScalar( "visualization_interval" ); - } - if (analysis_db->keyExists( "restart_interval" )){ - restart_interval = analysis_db->getScalar( "restart_interval" ); - } - if (greyscaleFE_db->keyExists( "timestep" )){ - timestep = greyscaleFE_db->getScalar( "timestep" ); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("No. of timesteps: %i \n", timestepMax); - fflush(stdout); - } - - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... - - Minkowski Morphology(Mask); - - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - auto current_db = db->cloneDatabase(); - double error = 1.0; - double flow_rate_previous = 0.0; - while (timestep < timestepMax && error > tolerance) { - //************************************************************************/ - // *************ODD TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(NeighborList, Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(NeighborList, Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - - // Update local pressure - //ScaLBL_D3Q19_GreyscaleFE_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); - ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); - // Compute pressure gradient - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(Pressure_dvc); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); - ScaLBL_DeviceBarrier(); - // Compute Pressure Tensor - //NOTE send and recv halo causes problems - it errorneously changes Phi - //ScaLBL_Comm->SendHalo(Phi); - ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); - //ScaLBL_Comm->RecvHalo(Phi); - //ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); - - /* Compute gradient of the pressure tensor */ - // call the recv Grad function once per tensor element - // 1st tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); - // 2nd tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); - // 3rd tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); - // 4th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); - // 5th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); - // 6th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); - - - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleFEChem(NeighborList, fq, Cq, Phi, SolidForce, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, - Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - -// // Set BCs -// if (BoundaryCondition == 3){ -// ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); -// ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); -// } - ScaLBL_D3Q19_AAodd_GreyscaleFEChem(NeighborList, fq, Cq, Phi, SolidForce, - 0, ScaLBL_Comm->LastExterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, - Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - - // *************EVEN TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->SendD3Q7AA(Cq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(Cq, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(Cq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(Cq, Phi, 0, ScaLBL_Comm->LastExterior(), Np); - - // Update local pressure - //ScaLBL_D3Q19_GreyscaleFE_Pressure(fq, Den, Porosity, Velocity, Pressure_dvc, rhoA, rhoB, Np); - ScaLBL_D3Q19_Pressure(fq, Pressure_dvc, Np); - // Compute pressure gradient - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(Pressure_dvc); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, Pressure_dvc, PressureGrad, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(Pressure_dvc,PressureGrad); - ScaLBL_DeviceBarrier(); - // Compute Pressure Tensor - //ScaLBL_Comm->SendHalo(Phi); - //NOTE send and recv halo causes problems - it errorneously changes Phi - ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(),Np); - //ScaLBL_Comm->RecvHalo(Phi); - //ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleFE_PressureTensor(NeighborList,Phi,Pressure_dvc,PressTensor,PhiLap,kappaA,kappaB,lambdaA,lambdaB,0,ScaLBL_Comm->LastExterior(),Np); - /* Compute gradient of the pressure tensor */ - // call the recv Grad function once per tensor element - // 1st tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[0*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[0*Np], &PressTensorGrad[0*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[0*Np],&PressTensorGrad[0*Np]); - // 2nd tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[1*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[1*Np], &PressTensorGrad[3*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[1*Np],&PressTensorGrad[3*Np]); - // 3rd tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[2*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[2*Np], &PressTensorGrad[6*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[2*Np],&PressTensorGrad[6*Np]); - // 4th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[3*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[3*Np], &PressTensorGrad[9*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[3*Np],&PressTensorGrad[9*Np]); - // 5th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[4*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[4*Np], &PressTensorGrad[12*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[4*Np],&PressTensorGrad[12*Np]); - // 6th tensor element - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->SendHalo(&PressTensor[5*Np]); - ScaLBL_D3Q19_GreyscaleFE_Gradient(NeighborList, &PressTensor[5*Np], &PressTensorGrad[15*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->RecvGrad(&PressTensor[5*Np],&PressTensorGrad[15*Np]); - - - ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleFEChem(fq, Cq, Phi, SolidForce, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, - Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); - ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - -// // Set BCs -// if (BoundaryCondition == 3){ -// ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); -// ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); -// } - ScaLBL_D3Q19_AAeven_GreyscaleFEChem(fq, Cq, Phi, SolidForce, - 0, ScaLBL_Comm->LastExterior(), Np, - tauA, tauB, tauA_eff, tauB_eff, rhoA, rhoB, gamma,kappaA,kappaB,lambdaA,lambdaB, Fx, Fy, Fz, - Porosity, Permeability, Velocity, Pressure_dvc,PressureGrad,PressTensorGrad,PhiLap); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - //************************************************************************/ - -// if (timestep%analysis_interval==0){ -// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); -// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); -// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); -// -// double count_loc=0; -// double count; -// double vax,vay,vaz; -// double vax_loc,vay_loc,vaz_loc; -// //double px_loc,py_loc,pz_loc; -// //double px,py,pz; -// //double mass_loc,mass_glb; -// -// //parameters for domain average -// int64_t i,j,k,n,imin,jmin,kmin,kmax; -// // If external boundary conditions are set, do not average over the inlet and outlet -// kmin=1; kmax=Nz-1; -// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 -// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; -// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; -// -// imin=jmin=1; -// // If inlet/outlet layers exist use these as default -// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; -// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; -// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer -// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; -// -//// px_loc = py_loc = pz_loc = 0.f; -//// mass_loc = 0.f; -//// for (int k=kmin; k 0){ -//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); -//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); -//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); -//// mass_loc += Den*PorosityMap(i,j,k); -//// } -//// } -//// } -//// } -//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// -//// vax = px/mass_glb; -//// vay = py/mass_glb; -//// vaz = pz/mass_glb; -// -// vax_loc = vay_loc = vaz_loc = 0.f; -// for (int k=kmin; k 0){ -// vax_loc += Velocity_x(i,j,k); -// vay_loc += Velocity_y(i,j,k); -// vaz_loc += Velocity_z(i,j,k); -// count_loc+=1.0; -// } -// } -// } -// } -// //MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// //MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// //MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// //MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -// -// vax = Mask->Comm.sumReduce( vax_loc ); -// vay = Mask->Comm.sumReduce( vay_loc ); -// vaz = Mask->Comm.sumReduce( vaz_loc ); -// count = Mask->Comm.sumReduce( count_loc ); -// -// vax /= count; -// vay /= count; -// vaz /= count; -// -// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); -// double dir_x = Fx/force_mag; -// double dir_y = Fy/force_mag; -// double dir_z = Fz/force_mag; -// if (force_mag == 0.0){ -// // default to z direction -// dir_x = 0.0; -// dir_y = 0.0; -// dir_z = 1.0; -// force_mag = 1.0; -// } -// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; -// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); -// -// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); -// flow_rate_previous = flow_rate; -// -// //if (rank==0) printf("Computing Minkowski functionals \n"); -// Morphology.ComputeScalar(SignDist,0.f); -// //Morphology.PrintAll(); -// double mu = (tau-0.5)/3.f; -// double Vs = Morphology.V(); -// double As = Morphology.A(); -// double Hs = Morphology.H(); -// double Xs = Morphology.X(); -// Vs = Dm->Comm.sumReduce( Vs); -// As = Dm->Comm.sumReduce( As); -// Hs = Dm->Comm.sumReduce( Hs); -// Xs = Dm->Comm.sumReduce( Xs); -// -// double h = Dm->voxel_length; -// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; -// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; -// -// if (rank==0){ -// printf(" AbsPerm = %.5g [micron^2]\n",absperm); -// bool WriteHeader=false; -// FILE * log_file = fopen("Permeability.csv","r"); -// if (log_file != NULL) -// fclose(log_file); -// else -// WriteHeader=true; -// log_file = fopen("Permeability.csv","a"); -// if (WriteHeader) -// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", -// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); -// -// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, -// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); -// fclose(log_file); -// } -// } - - if (timestep%visualization_interval==0){ - VelocityField(); - } - - if (timestep%restart_interval==0){ - //Use rank=0 write out Restart.db - if (rank==0) { - greyscaleFE_db->putScalar("timestep",timestep); - greyscaleFE_db->putScalar( "Restart", true ); - current_db->putDatabase("GreyscaleFE", greyscaleFE_db); - std::ofstream OutStream("Restart.db"); - current_db->print(OutStream, ""); - OutStream.close(); - - } - //Write out Restart data. - std::shared_ptr cfq; - cfq = std::shared_ptr(new double[19*Np],DeleteArray); - ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU - std::shared_ptr cDen; - cDen = std::shared_ptr(new double[2*Np],DeleteArray); - ScaLBL_CopyToHost(cDen.get(),Den,2*Np*sizeof(double));// Copy restart data to the CPU - - FILE *RESTARTFILE; - RESTARTFILE=fopen(LocalRestartFile,"wb"); - fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); - fwrite(cDen.get(),sizeof(double),2*Np,RESTARTFILE); - fclose(RESTARTFILE); - MPI_Barrier(comm); - } - } - - PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_greyscaleFE_simulator",1); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // ************************************************************************ -} - -void ScaLBL_GreyscaleFEModel::VelocityField(){ - -/* Minkowski Morphology(Mask); - int SIZE=Np*sizeof(double); - ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); - - memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); - Morphology.Initialize(); - Morphology.UpdateMeshValues(); - Morphology.ComputeLocal(); - Morphology.Reduce(); - - double count_loc=0; - double count; - double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - vax_loc = vay_loc = vaz_loc = 0.f; - for (int n=0; nLastExterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - - for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - - vax /= count; - vay /= count; - vaz /= count; - - double mu = (tau-0.5)/3.f; - if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); - if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, - Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); - */ - - std::vector visData; - fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); - - auto VxVar = std::make_shared(); - auto VyVar = std::make_shared(); - auto VzVar = std::make_shared(); - auto SignDistVar = std::make_shared(); - auto PressureVar = std::make_shared(); - - IO::initialize("","silo","false"); - // Create the MeshDataStruct - visData.resize(1); - visData[0].meshName = "domain"; - visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); - SignDistVar->name = "SignDist"; - SignDistVar->type = IO::VariableType::VolumeVariable; - SignDistVar->dim = 1; - SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(SignDistVar); - - VxVar->name = "Velocity_x"; - VxVar->type = IO::VariableType::VolumeVariable; - VxVar->dim = 1; - VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VxVar); - VyVar->name = "Velocity_y"; - VyVar->type = IO::VariableType::VolumeVariable; - VyVar->dim = 1; - VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VyVar); - VzVar->name = "Velocity_z"; - VzVar->type = IO::VariableType::VolumeVariable; - VzVar->dim = 1; - VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VzVar); - - PressureVar->name = "Pressure"; - PressureVar->type = IO::VariableType::VolumeVariable; - PressureVar->dim = 1; - PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(PressureVar); - - Array& SignData = visData[0].vars[0]->data; - Array& VelxData = visData[0].vars[1]->data; - Array& VelyData = visData[0].vars[2]->data; - Array& VelzData = visData[0].vars[3]->data; - Array& PressureData = visData[0].vars[4]->data; - - ASSERT(visData[0].vars[0]->name=="SignDist"); - ASSERT(visData[0].vars[1]->name=="Velocity_x"); - ASSERT(visData[0].vars[2]->name=="Velocity_y"); - ASSERT(visData[0].vars[3]->name=="Velocity_z"); - ASSERT(visData[0].vars[4]->name=="Pressure"); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); - - fillData.copy(SignDist,SignData); - fillData.copy(Velocity_x,VelxData); - fillData.copy(Velocity_y,VelyData); - fillData.copy(Velocity_z,VelzData); - fillData.copy(Pressure,PressureData); - - IO::writeData( timestep, visData, Dm->Comm ); - -} - -void ScaLBL_GreyscaleFEModel::WriteDebug(){ - // Copy back final phase indicator field and convert to regular layout - DoubleArray PhaseField(Nx,Ny,Nz); - - //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); - -// FILE *OUTFILE; -// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); -// OUTFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,OUTFILE); -// fclose(OUTFILE); -// - -// ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A.%05i.raw",rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B.%05i.raw",rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); - - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); - FILE *PFILE; - sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); - PFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PFILE); - fclose(PFILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); - FILE *VELX_FILE; - sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); - VELX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELX_FILE); - fclose(VELX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); - FILE *VELY_FILE; - sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); - VELY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELY_FILE); - fclose(VELY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); - FILE *VELZ_FILE; - sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); - VELZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELZ_FILE); - fclose(VELZ_FILE); - - - ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); - FILE *PhiFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - PhiFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PhiFILE); - fclose(PhiFILE); - -// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); -// FILE *POROS_FILE; -// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); -// POROS_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,POROS_FILE); -// fclose(POROS_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); -// FILE *PERM_FILE; -// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); -// PERM_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,PERM_FILE); -// fclose(PERM_FILE); -} diff --git a/models/GreyscaleFEModel.h b/models/GreyscaleFEModel.h deleted file mode 100644 index 34a6f999..00000000 --- a/models/GreyscaleFEModel.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -Implementation of multicomponent greyscale free-energy based lattice boltzmann model - */ -#include -#include -#include -#include -#include -#include -#include - -#include "common/Communication.h" -#include "common/MPI.h" -#include "common/Database.h" -#include "common/ScaLBL.h" -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" - -class ScaLBL_GreyscaleFEModel{ -public: - ScaLBL_GreyscaleFEModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleFEModel(); - - // functions in they should be run - void ReadParams(string filename); - void ReadParams(std::shared_ptr db0); - void SetDomain(); - void ReadInput(); - void Create(); - void Initialize(); - void Run(); - void WriteDebug(); - void VelocityField(); - - bool Restart,pBC; - int timestep,timestepMax; - int BoundaryCondition; - double tauA,tauB; - double tauA_eff,tauB_eff; - double rhoA,rhoB; - double tolerance; - double Fx,Fy,Fz,flux; - double din,dout; - double dp;//solid particle diameter, unit in voxel - double GreyPorosity; - //double Gsc; - double gamma; - double kappaA,kappaB; - double lambdaA,lambdaB; - - int Nx,Ny,Nz,N,Np; - int rank,nprocx,nprocy,nprocz,nprocs; - double Lx,Ly,Lz; - - std::shared_ptr Dm; // this domain is for analysis - std::shared_ptr Mask; // this domain is for lbm - std::shared_ptr ScaLBL_Comm; - - // input database - std::shared_ptr db; - std::shared_ptr domain_db; - std::shared_ptr greyscaleFE_db; - std::shared_ptr analysis_db; - std::shared_ptr vis_db; - - signed char *id; - int *NeighborList; - //double *fq,*Aq,*Bq; - double *fq,*Cq; - double *Den; - double *Permeability;//grey voxel permeability - double *Porosity; - double *Velocity; - double *SolidForce; - double *Pressure_dvc; - double *PressureGrad;// gradiant of pressure - double *PressTensor;//pressure tensor - double *PressTensorGrad;// gradient of pressure tensor - double *Phi; - double *PhiLap;//laplacian of phase field phi -// double *DenGradA; -// double *DenGradB; -// double *DenLapA; -// double *DenLapB; - IntArray Map; - DoubleArray SignDist; - DoubleArray Velocity_x; - DoubleArray Velocity_y; - DoubleArray Velocity_z; - DoubleArray PorosityMap; - DoubleArray Pressure; - -private: - MPI_Comm comm; - - int dist_mem_size; - int neighborSize; - // filenames - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - - void AssignComponentLabels(double *Porosity, double *Permeablity, double *SolidPotential); - void AssignSolidForce(double *SolidPotential, double *SolidForce); - void Density_and_Phase_Init(); - -}; - diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 5dd3e0a5..f2f94f9d 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -584,15 +584,10 @@ void ScaLBL_GreyscaleModel::Run(){ } } } - // Disable the this, if the one below does not work, then enable this - //vax = sumReduce( Mask->Comm, vax_loc); - //vay = sumReduce( Mask->Comm, vay_loc); - //vaz = sumReduce( Mask->Comm, vaz_loc); - //count = sumReduce( Mask->Comm, count_loc); - vax = Mask->Comm.sumReduce( vax_loc ); - vay = Mask->Comm.sumReduce( vay_loc ); - vaz = Mask->Comm.sumReduce( vaz_loc ); - count = Mask->Comm.sumReduce( count_loc ); + vax = sumReduce( Mask->Comm, vax_loc); + vay = sumReduce( Mask->Comm, vay_loc); + vaz = sumReduce( Mask->Comm, vaz_loc); + count = sumReduce( Mask->Comm, count_loc); vax /= count; vay /= count; diff --git a/models/GreyscaleSCModel.cpp b/models/GreyscaleSCModel.cpp deleted file mode 100644 index 6fa125fd..00000000 --- a/models/GreyscaleSCModel.cpp +++ /dev/null @@ -1,1433 +0,0 @@ -/* -Greyscale lattice boltzmann model - */ -#include "models/GreyscaleSCModel.h" -#include "analysis/distance.h" -#include "analysis/morphology.h" -#include -#include - -template -void DeleteArray( const TYPE *p ) -{ - delete [] p; -} - -ScaLBL_GreyscaleSCModel::ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),Gsc(0), -rhoA(0),rhoB(0),rhoA_minor(0),rhoB_minor(0),Fx(0),Fy(0),Fz(0),fluxA(0),fluxB(0),dinA(0),doutA(0),dinB(0),doutB(0),GreyPorosity(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) -{ - SignDist.resize(Nx,Ny,Nz); - SignDist.fill(0); - -} -ScaLBL_GreyscaleSCModel::~ScaLBL_GreyscaleSCModel(){ - -} - -void ScaLBL_GreyscaleSCModel::ReadParams(string filename){ - // read the input database - db = std::make_shared( filename ); - domain_db = db->getDatabase( "Domain" ); - greyscaleSC_db = db->getDatabase( "GreyscaleSC" ); - analysis_db = db->getDatabase( "Analysis" ); - vis_db = db->getDatabase( "Visualization" ); - - // set defaults - timestepMax = 100000; - tauA = 1.0; - tauB = 1.0; - tauA_eff = tauA; - tauB_eff = tauB; - rhoA = rhoB = 1.0; - rhoA_minor = rhoB_minor = 0.01;//dissolved density - Gsc = 2.0;//SC fluid-fluid interaction coefficient - tolerance = 0.01; - Fx = Fy = Fz = 0.0; - Restart=false; - dinA=rhoA;//inlet density for fluid A - dinB=rhoB_minor;//inlet density for fluid B - doutA=rhoA_minor;//outlet denisty for fluid A - doutB=rhoB;//outlet density for fluid B - fluxA=fluxB=0.0; - - // ---------------------- Greyscale Model parameters -----------------------// - if (greyscaleSC_db->keyExists( "timestepMax" )){ - timestepMax = greyscaleSC_db->getScalar( "timestepMax" ); - } - if (greyscaleSC_db->keyExists( "tauA" )){ - tauA = greyscaleSC_db->getScalar( "tauA" ); - } - if (greyscaleSC_db->keyExists( "tauB" )){ - tauB = greyscaleSC_db->getScalar( "tauB" ); - } - tauA_eff = greyscaleSC_db->getWithDefault( "tauA_eff", tauA ); - tauB_eff = greyscaleSC_db->getWithDefault( "tauB_eff", tauB ); - rhoA = greyscaleSC_db->getWithDefault( "rhoA", rhoA ); - rhoB = greyscaleSC_db->getWithDefault( "rhoB", rhoB ); - rhoA_minor = greyscaleSC_db->getWithDefault( "rhoA_minor", rhoA_minor ); - rhoB_minor = greyscaleSC_db->getWithDefault( "rhoB_minor", rhoB_minor ); - dinA = greyscaleSC_db->getWithDefault( "dinA", dinA ); - dinB = greyscaleSC_db->getWithDefault( "dinB", dinB ); - doutA = greyscaleSC_db->getWithDefault( "doutA", doutA ); - doutB = greyscaleSC_db->getWithDefault( "doutB", doutB ); - if (greyscaleSC_db->keyExists( "Gsc" )){ - Gsc = greyscaleSC_db->getScalar( "Gsc" ); - } - if (greyscaleSC_db->keyExists( "F" )){ - Fx = greyscaleSC_db->getVector( "F" )[0]; - Fy = greyscaleSC_db->getVector( "F" )[1]; - Fz = greyscaleSC_db->getVector( "F" )[2]; - } - if (greyscaleSC_db->keyExists( "Restart" )){ - Restart = greyscaleSC_db->getScalar( "Restart" ); - } - if (greyscaleSC_db->keyExists( "fluxA" )){ - fluxA = greyscaleSC_db->getScalar( "fluxA" ); - } - if (greyscaleSC_db->keyExists( "fluxB" )){ - fluxB = greyscaleSC_db->getScalar( "fluxB" ); - } - if (greyscaleSC_db->keyExists( "tolerance" )){ - tolerance = greyscaleSC_db->getScalar( "tolerance" ); - } - // ------------------------------------------------------------------------// - - //------------------------ Other Domain parameters ------------------------// - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } - // ------------------------------------------------------------------------// -} - -void ScaLBL_GreyscaleSCModel::SetDomain(){ - Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - // domain parameters - Nx = Dm->Nx; - Ny = Dm->Ny; - Nz = Dm->Nz; - Lx = Dm->Lx; - Ly = Dm->Ly; - Lz = Dm->Lz; - N = Nx*Ny*Nz; - - SignDist.resize(Nx,Ny,Nz); - Velocity_x.resize(Nx,Ny,Nz); - Velocity_y.resize(Nx,Ny,Nz); - Velocity_z.resize(Nx,Ny,Nz); - PorosityMap.resize(Nx,Ny,Nz); - Pressure.resize(Nx,Ny,Nz); - DenA_data.resize(Nx,Ny,Nz); - DenB_data.resize(Nx,Ny,Nz); - - id = new signed char [N]; - for (int i=0; iid[i] = 1; // initialize this way - MPI_Barrier(comm); - Dm->CommInit(); - MPI_Barrier(comm); - // Read domain parameters - rank = Dm->rank(); - nprocx = Dm->nprocx(); - nprocy = Dm->nprocy(); - nprocz = Dm->nprocz(); -} - -void ScaLBL_GreyscaleSCModel::ReadInput(){ - - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - - if (domain_db->keyExists( "Filename" )){ - auto Filename = domain_db->getScalar( "Filename" ); - Mask->Decomp(Filename); - } - else{ - if (rank==0) printf("Filename of input image is not found, reading ID.0* instead."); - Mask->ReadIDs(); - } - for (int i=0; iid[i]; // save what was read - - // Generate the signed distance map - // Initialize the domain and communication - Array id_solid(Nx,Ny,Nz); - int count = 0; - // Solve for the position of the solid phase - for (int k=0;kid[n]; - if (label > 0) id_solid(i,j,k) = 1; - else id_solid(i,j,k) = 0; - } - } - } - // Initialize the signed distance function - for (int k=0;kgetVector( "PorosityList" ); - auto PermeabilityList = greyscaleSC_db->getVector( "PermeabilityList" ); - //auto RelPermListA = greyscaleSC_db->getVector( "RelPermListA" ); - //auto RelPermListB = greyscaleSC_db->getVector( "RelPermListB" ); - auto LabelList = greyscaleSC_db->getVector( "ComponentLabels" ); - auto AffinityListA = greyscaleSC_db->getVector( "ComponentAffinityA" ); - auto AffinityListB = greyscaleSC_db->getVector( "ComponentAffinityB" ); - - //1. Requirement for "ComponentLabels": - // *labels can be a nagative integer, 0, 1, 2, or a positive integer >= 3 - // *label = 1 and 2 are reserved for NW and W phase respectively. - //2. Requirement for "ComponentAffinity": - // *should be in the same length as "ComponentLabels" - // *could leave ComponentAffinityA and B=0.0 for label=1 and 2 - //3. Requirement for "PorosityList": - // *for ComponentLables <=0, put porosity value = 0.0; - // *for ComponentLabels >=3, put the corresponding sub-resolution porosity - // *for ComponentLabels =1, 2, put porosity=1 (or if users accidentally put other values it should still be fine) - //4. Requirement for "PermeabilityList": - // *for ComponentLabels <=2, does not matter, can leave it as 1.0 - //5. Requirement for "RelPermListA" and "RelPermListB": - // *for ComponentLabels <=2, does not matter, can leave both RelPermA and RelPermB as 1.0 - - NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || - NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ - ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, and PermeabilityList must all be the same length! \n"); - } -// if (NLABELS != PorosityList.size() || NLABELS != PermeabilityList.size() || -// NLABELS != RelPermListA.size() || NLABELS != RelPermListB.size() || -// NLABELS != AffinityListA.size() || NLABELS != AffinityListB.size() ){ -// ERROR("Error: ComponentLabels, ComponentAffinityA/B, PorosityList, PermeabilityList, and RelPermListA/B must all be the same length! \n"); -// } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - - for (int idx=0; idx 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - POROSITY=PorosityList[idx]; - label_count[idx] += 1.0; - idx = NLABELS; - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (POROSITY<=0.0){ - ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); - } - else{ - Poros[idx] = POROSITY; - } - } - } - } - } - - //Populate the permeability map, NOTE only for node_ID > 0, i.e. open or grey nodes - //For node_ID <= 0: these are solid nodes of various wettability - for (int k=0;k0) && (VALUE == LabelList[idx])){ - PERMEABILITY=PermeabilityList[idx]; - idx = NLABELS; - //Mask->id[n] = 0; // set mask to zero since this is an immobile component - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (PERMEABILITY<=0.0){ - ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); - } - else{ - Perm[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; - } - } - } - } - } -// // New way of initializing the relperm values -// for (int k=0;k0) && (VALUE == LabelList[idx])){ -// RELPERMA=PermeabilityList[idx]*RelPermListA[idx]; -// RELPERMB=PermeabilityList[idx]*RelPermListB[idx]; -// idx = NLABELS; -// //Mask->id[n] = 0; // set mask to zero since this is an immobile component -// } -// } -// int idx = Map(i,j,k); -// if (!(idx < 0)){ -// if (RELPERMA<=0.0 || RELPERMB<=0.0){ -// ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); -// } -// else{ -// relPermA_host[idx] = RELPERMA/Dm->voxel_length/Dm->voxel_length; -// relPermB_host[idx] = RELPERMB/Dm->voxel_length/Dm->voxel_length; -// } -// } -// } -// } -// } - - //Populate the solid potential map, for ALL range of node_ID except node = 1,2, i.e. NW and W phase - for (int k=0;k=3){ - AFFINITY_A=AffinityListA[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 - AFFINITY_B=AffinityListB[idx]*(1.0-PorosityList[idx]);//BE CAREFUL! Requires for node_ID<=0, user puts porosity=0.0 - } - else{//i.e. label = 1 or 2 - AFFINITY_A=0.0; - AFFINITY_B=0.0; - } - idx = NLABELS; - } - } - //NOTE: node_ID = 1 and 2 are reserved - if ((VALUE == 1)||(VALUE == 2)){ - AFFINITY_A=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists - AFFINITY_B=0.0;//NOTE: still need this as users may forget to put label=1,2 in ComponentLabelLists - } - SolidPotentialA_host[n] = AFFINITY_A; - SolidPotentialB_host[n] = AFFINITY_B; - } - } - } - - // Calculate Shan-Chen fluid-solid forces - double *Dst; - Dst = new double [3*3*3]; - for (int kk=0; kk<3; kk++){ - for (int jj=0; jj<3; jj++){ - for (int ii=0; ii<3; ii++){ - int index = kk*9+jj*3+ii; - Dst[index] = sqrt(double(ii-1)*double(ii-1) + double(jj-1)*double(jj-1)+ double(kk-1)*double(kk-1)); - } - } - } - double w_face = 1.f/18.f; - double w_edge = 1.f/36.f; - double w_corner = 0.f; - //local - Dst[13] = 0.f; - //faces - Dst[4] = w_face; - Dst[10] = w_face; - Dst[12] = w_face; - Dst[14] = w_face; - Dst[16] = w_face; - Dst[22] = w_face; - // corners - Dst[0] = w_corner; - Dst[2] = w_corner; - Dst[6] = w_corner; - Dst[8] = w_corner; - Dst[18] = w_corner; - Dst[20] = w_corner; - Dst[24] = w_corner; - Dst[26] = w_corner; - // edges - Dst[1] = w_edge; - Dst[3] = w_edge; - Dst[5] = w_edge; - Dst[7] = w_edge; - Dst[9] = w_edge; - Dst[11] = w_edge; - Dst[15] = w_edge; - Dst[17] = w_edge; - Dst[19] = w_edge; - Dst[21] = w_edge; - Dst[23] = w_edge; - Dst[25] = w_edge; - - for (int k=1; kid[nn] <= 0)||(Mask->id[nn]>=3)){ - double vec_x = double(ii-1); - double vec_y = double(jj-1); - double vec_z = double(kk-1); - double GWNS_A=SolidPotentialA_host[nn]; - double GWNS_B=SolidPotentialB_host[nn]; - phi_x_A += -1.0*GWNS_A*weight*vec_x; - phi_y_A += -1.0*GWNS_A*weight*vec_y; - phi_z_A += -1.0*GWNS_A*weight*vec_z; - phi_x_B += -1.0*GWNS_B*weight*vec_x; - phi_y_B += -1.0*GWNS_B*weight*vec_y; - phi_z_B += -1.0*GWNS_B*weight*vec_z; - } - } - } - } - SolidForceA_host[idx+0*Np] = phi_x_A; - SolidForceA_host[idx+1*Np] = phi_y_A; - SolidForceA_host[idx+2*Np] = phi_z_A; - SolidForceB_host[idx+0*Np] = phi_x_B; - SolidForceB_host[idx+1*Np] = phi_y_B; - SolidForceB_host[idx+2*Np] = phi_z_B; - } - } - } - } - - // Set Dm to match Mask - for (int i=0; iid[i] = Mask->id[i]; - - for (int idx=0; idxComm.sumReduce(label_count[idx]); - - //Initialize a weighted porosity after considering grey voxels - GreyPorosity=0.0; - for (unsigned int idx=0; idxvoxel_length); - printf("Component labels: %lu \n",NLABELS); - for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); - printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); - } - printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); - } - - //Copy all data to device - ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); - ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); - //ScaLBL_CopyToDevice(relPermA, relPermA_host, Np*sizeof(double)); - //ScaLBL_CopyToDevice(relPermB, relPermB_host, Np*sizeof(double)); - ScaLBL_CopyToDevice(SolidForceA, SolidForceA_host, 3*Np*sizeof(double)); - ScaLBL_CopyToDevice(SolidForceB, SolidForceB_host, 3*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] SolidPotentialA_host; - delete [] SolidPotentialB_host; - delete [] SolidForceA_host; - delete [] SolidForceB_host; - delete [] Poros; - delete [] Perm; - //delete [] relPermA_host; - //delete [] relPermB_host; - delete [] Dst; -} - -//void ScaLBL_GreyscaleSCModel::Density_Init(){ -// -// size_t NLABELS=0; -// signed char VALUE=0; -// -// vector LabelList{1,2}; -// vector SwList{0.0,1.0}; -// -// if (greyscaleSC_db->keyExists( "GreyNodeLabels" )){ -// LabelList.clear(); -// LabelList = greyscaleSC_db->getVector( "GreyNodeLabels" ); -// } -// if (greyscaleSC_db->keyExists( "GreyNodeSw" )){ -// SwList.clear(); -// SwList = greyscaleSC_db->getVector( "GreyNodeSw" ); -// } -// -// NLABELS=LabelList.size(); -// if (NLABELS != SwList.size()){ -// ERROR("Error: GreyNodeLabels and GreyNodeSw must be the same length! \n"); -// } -// -// double *Den_temp; -// Den_temp=new double [2*Np]; -// double nA=0.5;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. -// double nB=0.5; -// -// //double *Phi_temp; -// //Phi_temp=new double [Np]; -// //double phi = 0.0; -// -// for (int k=0; kid[n]; -// if (VALUE>0){ -// for (unsigned int idx=0; idx < NLABELS; idx++){ -// if (VALUE == LabelList[idx]){ -// double Sw = SwList[idx]; -// if ((Sw<0.0) || (Sw>1.0)) ERROR("Error: Initial saturation for grey nodes must be between [0.0, 1.0]! \n"); -// nB=Sw; -// nA=1.0-Sw; -// //phi = nA-nB; -// idx = NLABELS; -// } -// } -// if (VALUE==1){//label=1 reserved for NW phase -// //TODO; maybe need rho_major and rho_minor initialization -// nA=rhoA; -// nB=rhoB_minor; -// //phi = nA-nB; -// } -// else if(VALUE==2){//label=2 reserved for W phase -// //TODO; maybe need rho_major and rho_minor initialization -// nA=rhoA_minor; -// nB=rhoB; -// //phi = nA-nB; -// } -// int idx = Map(i,j,k); -// Den_temp[idx+0*Np] = nA; -// Den_temp[idx+1*Np] = nB; -// //Phi_temp[idx] = phi; -// } -// } -// } -// } -// //copy to device -// ScaLBL_CopyToDevice(Den, Den_temp, 2*Np*sizeof(double)); -// //ScaLBL_CopyToDevice(Phi, Phi_temp, 1*Np*sizeof(double)); -// ScaLBL_DeviceBarrier(); -// delete [] Den_temp; -// //delete [] Phi_temp; -//} - -void ScaLBL_GreyscaleSCModel::Density_Init(){ - - size_t NLABELS=0; - signed char VALUE=0; - - vector LabelList{1,2}; - vector GreyDenAList{rhoA,rhoB_minor}; - vector GreyDenBList{rhoB,rhoA_minor}; - - if (greyscaleSC_db->keyExists( "GreyNodeLabels" )){ - LabelList.clear(); - LabelList = greyscaleSC_db->getVector( "GreyNodeLabels" ); - } - if (greyscaleSC_db->keyExists( "GreyNodeDenAInit" )){ - GreyDenAList.clear(); - GreyDenAList = greyscaleSC_db->getVector( "GreyNodeDenAInit" ); - } - if (greyscaleSC_db->keyExists( "GreyNodeDenBInit" )){ - GreyDenBList.clear(); - GreyDenBList = greyscaleSC_db->getVector( "GreyNodeDenBInit" ); - } - - NLABELS=LabelList.size(); - if (NLABELS != GreyDenAList.size() || NLABELS != GreyDenBList.size()){ - ERROR("Error: GreyNodeLabels, GreyNodeDenAInit, and GreyNodeDenBInit must all be the same length! \n"); - } - - double *DenA_temp,*DenB_temp; - DenA_temp=new double [Nx*Ny*Nz]; - DenB_temp=new double [Nx*Ny*Nz]; - double nA=0.0;//to prevent use may forget to specify all greynodes, then must initialize something to start with, givning just zeros is too risky. - double nB=0.0; - - //double *Phi_temp; - //Phi_temp=new double [Np]; - //double phi = 0.0; - - for (int k=0; kid[n]; - if (VALUE>0){ - for (unsigned int idx=0; idx < NLABELS; idx++){ - if (VALUE == LabelList[idx]){ - nA=GreyDenAList[idx]; - nB=GreyDenBList[idx]; - //phi = nA-nB; - idx = NLABELS; - } - } - if (VALUE==1){//label=1 reserved for NW phase - nA=rhoA; - nB=rhoB_minor; - //phi = nA-nB; - } - else if(VALUE==2){//label=2 reserved for W phase - nA=rhoA_minor; - nB=rhoB; - //phi = nA-nB; - } - DenA_temp[n] = nA; - DenB_temp[n] = nB; - } - else{ //for ID<=0, i.e. all sorts of solid minerals, density is zero - DenA_temp[n] = 0.0; - DenB_temp[n] = 0.0; - } - } - } - } - - //copy to device - ScaLBL_CopyToDevice(DenA, DenA_temp, Nx*Ny*Nz*sizeof(double)); - ScaLBL_CopyToDevice(DenB, DenB_temp, Nx*Ny*Nz*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] DenA_temp; - delete [] DenB_temp; - - if (BoundaryCondition >0 ){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,2); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-3); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-3); - } - } - -} - -void ScaLBL_GreyscaleSCModel::Create(){ - /* - * This function creates the variables needed to run a LBM - */ - //......................................................... - // don't perform computations at the eight corners - //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - - //......................................................... - // Initialize communication structures in averaging domain - for (int i=0; iid[i] = Mask->id[i]; - Mask->CommInit(); - Np=Mask->PoreCount(); - //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - // Create a communicator for the device (will use optimized layout) - // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); - ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); - - int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N); - Map.resize(Nx,Ny,Nz); Map.fill(-2); - auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - MPI_Barrier(comm); - - //........................................................................... - // MAIN VARIABLES ALLOCATED HERE - //........................................................................... - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - //......................device distributions................................. - dist_mem_size = Np*sizeof(double); - neighborSize=18*(Np*sizeof(int)); - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &fqA, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &fqB, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &DenA, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &DenB, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &relPermA, sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &relPermB, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &SolidForceA, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &SolidForceB, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradA, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &DenGradB, 3*sizeof(double)*Np); - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device neighbor list \n"); - fflush(stdout); - // Copy the Map to device - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; kLastExterior(); idx++){ - auto n = TmpMap[idx]; - if (n > Nx*Ny*Nz){ - printf("Bad value! idx=%i \n", n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ - auto n = TmpMap[idx]; - if ( n > Nx*Ny*Nz ){ - printf("Bad value! idx=%i \n",n); - TmpMap[idx] = Nx*Ny*Nz-1; - } - } - ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); - ScaLBL_DeviceBarrier(); - delete [] TmpMap; - // copy the neighbor list - ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); -} - -void ScaLBL_GreyscaleSCModel::Initialize(){ - if (Restart == true){ -// //TODO: Restart funtion is currently not working; need updates -// if (rank==0){ -// printf("Initializing density field and distributions from Restart! \n"); -// } -// // Read in the restart file to CPU buffers -// std::shared_ptr cfq; -// cfq = std::shared_ptr(new double[19*Np],DeleteArray); -// std::shared_ptr cDen; -// cDen = std::shared_ptr(new double[2*Np],DeleteArray); -// FILE *File; -// File=fopen(LocalRestartFile,"rb"); -// fread(cfq.get(),sizeof(double),19*Np,File); -// fread(cDen.get(),sizeof(double),2*Np,File); -// fclose(File); -// -// // Copy the restart data to the GPU -// ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double)); -// ScaLBL_CopyToDevice(Den,cDen.get(),2*Np*sizeof(double)); -// ScaLBL_DeviceBarrier(); -// MPI_Barrier(comm); -// -// //TODO need proper initialization ! -// -// //TODO need to initialize velocity field ! -// //this is required for calculating the pressure_dvc -// //can make a funciton to update velocity, such as ScaLBL_D3Q19_GreyColorIMRT_Velocity - } - else{ - if (rank==0) printf ("Initializing solid affinities \n"); - AssignGreyscaleAndSolidLabels(); - if (rank==0) printf ("Initializing density field \n"); - Density_Init();//initialize density field - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_GreyscaleSC_Init(dvcMap,fqA, fqB, DenA,DenB, Np); - -// //debug -// DoubleArray PhaseField(Nx,Ny,Nz); -// //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); -// ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); -// FILE *AFILE; -// sprintf(LocalRankFilename,"A_init.%05i.raw",rank); -// AFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,AFILE); -// fclose(AFILE); -// -// //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); -// ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); -// FILE *BFILE; -// sprintf(LocalRankFilename,"B_init.%05i.raw",rank); -// BFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,BFILE); -// fclose(BFILE); - - //Velocity also needs initialization (for old incompressible momentum transport) - //if (rank==0) printf ("Initializing velocity field \n"); - //double *vel_init; - //vel_init = new double [3*Np]; - //for (int i=0;i<3*Np;i++) vel_init[i]=0.0; - //ScaLBL_CopyToDevice(Velocity,vel_init,3*Np*sizeof(double)); - //ScaLBL_DeviceBarrier(); - //delete [] vel_init; - } -} - -void ScaLBL_GreyscaleSCModel::Run(){ - int nprocs=nprocx*nprocy*nprocz; - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - int analysis_interval = 1000; // number of timesteps in between in situ analysis - int visualization_interval = 1000; - int restart_interval = 10000; // number of timesteps in between in saving distributions for restart - if (analysis_db->keyExists( "analysis_interval" )){ - analysis_interval = analysis_db->getScalar( "analysis_interval" ); - } - if (analysis_db->keyExists( "visualization_interval" )){ - visualization_interval = analysis_db->getScalar( "visualization_interval" ); - } - if (analysis_db->keyExists( "restart_interval" )){ - restart_interval = analysis_db->getScalar( "restart_interval" ); - } - if (greyscaleSC_db->keyExists( "timestep" )){ - timestep = greyscaleSC_db->getScalar( "timestep" ); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("No. of timesteps: %i \n", timestepMax); - fflush(stdout); - } - - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... - - Minkowski Morphology(Mask); - - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - auto current_db = db->cloneDatabase(); - double error = 1.0; - double flow_rate_previous = 0.0; - while (timestep < timestepMax && error > tolerance) { - //************************************************************************/ - // *************ODD TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - if (BoundaryCondition == 4){ - dinA = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqA, fluxA, timestep); - dinB = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqB, fluxB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(NeighborList, dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - - //if (BoundaryCondition > 0){ - // ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); - // ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - //} - - // Compute density gradient - // fluid component A - ScaLBL_Comm_Regular->SendHalo(DenA); - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenA); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - // fluid component B - ScaLBL_Comm_Regular->SendHalo(DenB); - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenB); - if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - - // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Collsion - ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(NeighborList, dvcMap, fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - - // *************EVEN TIMESTEP*************// - timestep++; - // Compute the density field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm->BiSendD3Q19AA(fqA,fqB); //READ FROM NORMAL - ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->BiRecvD3Q19AA(fqA,fqB); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set BCs - if (BoundaryCondition == 3){ - ScaLBL_Comm->GreyscaleSC_Pressure_BC_z(NeighborList, fqA, fqB, dinA, dinB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - if (BoundaryCondition == 4){ - dinA = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqA, fluxA, timestep); - dinB = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fqB, fluxB, timestep); - ScaLBL_Comm->GreyscaleSC_Pressure_BC_Z(NeighborList, fqA, fqB, doutA, doutB, timestep); - } - ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(dvcMap, fqA, fqB, DenA, DenB, 0, ScaLBL_Comm->LastExterior(), Np); - - //if (BoundaryCondition > 0){ - // ScaLBL_Comm->GreyscaleSC_BC_z(dvcMap, DenA, DenB, dinA, dinB); - // ScaLBL_Comm->GreyscaleSC_BC_Z(dvcMap, DenA, DenB, doutA, doutB); - //} - - // Compute density gradient - // fluid component A - ScaLBL_Comm_Regular->SendHalo(DenA); - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenA); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenA,dinA,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenA,doutA,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenA, DenGradA, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - // fluid component B - ScaLBL_Comm_Regular->SendHalo(DenB); - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(DenB); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition ==3 || BoundaryCondition ==4){//not necessarily applied to velBC (BC=2) - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(DenB,dinB,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(DenB,doutB,Nx,Ny,Nz,Nz-1); - } - } - ScaLBL_D3Q19_GreyscaleSC_Gradient(NeighborList, dvcMap, DenB, DenGradB, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - - // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - // Collsion - ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(dvcMap,fqA, fqB, DenA, DenB, DenGradA, DenGradB, SolidForceA, SolidForceB, Porosity,Permeability,Velocity,Pressure_dvc, - tauA, tauB, tauA_eff, tauB_eff, Gsc, Fx, Fy, Fz, - 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - //************************************************************************/ - -// if (timestep%analysis_interval==0){ -// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); -// //ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap); -// //ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); -// -// double count_loc=0; -// double count; -// double vax,vay,vaz; -// double vax_loc,vay_loc,vaz_loc; -// //double px_loc,py_loc,pz_loc; -// //double px,py,pz; -// //double mass_loc,mass_glb; -// -// //parameters for domain average -// int64_t i,j,k,n,imin,jmin,kmin,kmax; -// // If external boundary conditions are set, do not average over the inlet and outlet -// kmin=1; kmax=Nz-1; -// //In case user forgets to specify the inlet/outlet buffer layers for BC>0 -// if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; -// if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; -// -// imin=jmin=1; -// // If inlet/outlet layers exist use these as default -// //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x; -// //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y; -// if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer -// if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; -// -//// px_loc = py_loc = pz_loc = 0.f; -//// mass_loc = 0.f; -//// for (int k=kmin; k 0){ -//// px_loc += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k); -//// py_loc += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k); -//// pz_loc += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k); -//// mass_loc += Den*PorosityMap(i,j,k); -//// } -//// } -//// } -//// } -//// MPI_Allreduce(&px_loc, &px, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&py_loc, &py, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&pz_loc, &pz, 1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); -//// -//// vax = px/mass_glb; -//// vay = py/mass_glb; -//// vaz = pz/mass_glb; -// -// vax_loc = vay_loc = vaz_loc = 0.f; -// for (int k=kmin; k 0){ -// vax_loc += Velocity_x(i,j,k); -// vay_loc += Velocity_y(i,j,k); -// vaz_loc += Velocity_z(i,j,k); -// count_loc+=1.0; -// } -// } -// } -// } -// vax = Mask->Comm.sumReduce( vax_loc ); -// vay = Mask->Comm.sumReduce( vay_loc ); -// vaz = Mask->Comm.sumReduce( vaz_loc ); -// count = Mask->Comm.sumReduce( count_loc ); -// -// vax /= count; -// vay /= count; -// vaz /= count; -// -// double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); -// double dir_x = Fx/force_mag; -// double dir_y = Fy/force_mag; -// double dir_z = Fz/force_mag; -// if (force_mag == 0.0){ -// // default to z direction -// dir_x = 0.0; -// dir_y = 0.0; -// dir_z = 1.0; -// force_mag = 1.0; -// } -// //double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb; -// double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); -// -// error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); -// flow_rate_previous = flow_rate; -// -// //if (rank==0) printf("Computing Minkowski functionals \n"); -// Morphology.ComputeScalar(SignDist,0.f); -// //Morphology.PrintAll(); -// double mu = (tau-0.5)/3.f; -// double Vs = Morphology.V(); -// double As = Morphology.A(); -// double Hs = Morphology.H(); -// double Xs = Morphology.X(); -// Vs = Dm->Comm.sumReduce( Vs); -// As = Dm->Comm.sumReduce( As); -// Hs = Dm->Comm.sumReduce( Hs); -// Xs = Dm->Comm.sumReduce( Xs); -// -// double h = Dm->voxel_length; -// //double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag; -// double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag; -// -// if (rank==0){ -// printf(" AbsPerm = %.5g [micron^2]\n",absperm); -// bool WriteHeader=false; -// FILE * log_file = fopen("Permeability.csv","r"); -// if (log_file != NULL) -// fclose(log_file); -// else -// WriteHeader=true; -// log_file = fopen("Permeability.csv","a"); -// if (WriteHeader) -// fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n", -// timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm); -// -// fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, -// h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm); -// fclose(log_file); -// } -// } - if (timestep==2&&BoundaryCondition==4){ - if (rank==0) printf(" Timestep dinA dinB doutA doutB\n"); - } - if (timestep%analysis_interval==0){ - if (BoundaryCondition==4){ - if (rank==0) printf(" %i %.3g %.3g %.3g %.3g\n",timestep,dinA,dinB,doutA,doutB); - } - } - - if (timestep%visualization_interval==0){ - WriteOutput(); - } - -// if (timestep%restart_interval==0){ -// //Use rank=0 write out Restart.db -// if (rank==0) { -// greyscaleSC_db->putScalar("timestep",timestep); -// greyscaleSC_db->putScalar( "Restart", true ); -// current_db->putDatabase("GreyscaleSC", greyscaleSC_db); -// std::ofstream OutStream("Restart.db"); -// current_db->print(OutStream, ""); -// OutStream.close(); -// -// } -// //Write out Restart data. -// std::shared_ptr cfq; -// cfq = std::shared_ptr(new double[19*Np],DeleteArray); -// ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU -// -// FILE *RESTARTFILE; -// RESTARTFILE=fopen(LocalRestartFile,"wb"); -// fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE); -// fclose(RESTARTFILE); -// MPI_Barrier(comm); -// } - } - - PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_greyscale_simulator",1); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // ************************************************************************ -} - -void ScaLBL_GreyscaleSCModel::WriteOutput(){ - -/* Minkowski Morphology(Mask); - int SIZE=Np*sizeof(double); - ScaLBL_D3Q19_Momentum(fq,Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE); - - memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double)); - Morphology.Initialize(); - Morphology.UpdateMeshValues(); - Morphology.ComputeLocal(); - Morphology.Reduce(); - - double count_loc=0; - double count; - double vax,vay,vaz; - double vax_loc,vay_loc,vaz_loc; - vax_loc = vay_loc = vaz_loc = 0.f; - for (int n=0; nLastExterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - - for (int n=ScaLBL_Comm->FirstInterior(); nLastInterior(); n++){ - vax_loc += VELOCITY[n]; - vay_loc += VELOCITY[Np+n]; - vaz_loc += VELOCITY[2*Np+n]; - count_loc+=1.0; - } - MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); - - vax /= count; - vay /= count; - vaz /= count; - - double mu = (tau-0.5)/3.f; - if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n"); - if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, - Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz); - */ - - std::vector visData; - fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); - - auto VxVar = std::make_shared(); - auto VyVar = std::make_shared(); - auto VzVar = std::make_shared(); - auto SignDistVar = std::make_shared(); - auto PressureVar = std::make_shared(); - auto DenAVar = std::make_shared(); - auto DenBVar = std::make_shared(); - - IO::initialize("","silo","false"); - // Create the MeshDataStruct - visData.resize(1); - visData[0].meshName = "domain"; - visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); - SignDistVar->name = "SignDist"; - SignDistVar->type = IO::VariableType::VolumeVariable; - SignDistVar->dim = 1; - SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(SignDistVar); - - VxVar->name = "Velocity_x"; - VxVar->type = IO::VariableType::VolumeVariable; - VxVar->dim = 1; - VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VxVar); - VyVar->name = "Velocity_y"; - VyVar->type = IO::VariableType::VolumeVariable; - VyVar->dim = 1; - VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VyVar); - VzVar->name = "Velocity_z"; - VzVar->type = IO::VariableType::VolumeVariable; - VzVar->dim = 1; - VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(VzVar); - - PressureVar->name = "Pressure"; - PressureVar->type = IO::VariableType::VolumeVariable; - PressureVar->dim = 1; - PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(PressureVar); - - DenAVar->name = "DenA"; - DenAVar->type = IO::VariableType::VolumeVariable; - DenAVar->dim = 1; - DenAVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(DenAVar); - DenBVar->name = "DenB"; - DenBVar->type = IO::VariableType::VolumeVariable; - DenBVar->dim = 1; - DenBVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(DenBVar); - - Array& SignData = visData[0].vars[0]->data; - Array& VelxData = visData[0].vars[1]->data; - Array& VelyData = visData[0].vars[2]->data; - Array& VelzData = visData[0].vars[3]->data; - Array& PressureData = visData[0].vars[4]->data; - Array& DenAData = visData[0].vars[5]->data; - Array& DenBData = visData[0].vars[6]->data; - - ASSERT(visData[0].vars[0]->name=="SignDist"); - ASSERT(visData[0].vars[1]->name=="Velocity_x"); - ASSERT(visData[0].vars[2]->name=="Velocity_y"); - ASSERT(visData[0].vars[3]->name=="Velocity_z"); - ASSERT(visData[0].vars[4]->name=="Pressure"); - ASSERT(visData[0].vars[5]->name=="DenA"); - ASSERT(visData[0].vars[6]->name=="DenB"); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure); - ScaLBL_CopyToHost(DenA_data.data(), DenA, sizeof(double)*N); - ScaLBL_CopyToHost(DenB_data.data(), DenB, sizeof(double)*N); - - fillData.copy(SignDist,SignData); - fillData.copy(Velocity_x,VelxData); - fillData.copy(Velocity_y,VelyData); - fillData.copy(Velocity_z,VelzData); - fillData.copy(Pressure,PressureData); - fillData.copy(DenA_data,DenAData); - fillData.copy(DenB_data,DenBData); - - IO::writeData( timestep, visData, Dm->Comm ); - -} - -void ScaLBL_GreyscaleSCModel::WriteDebug(){ - // Copy back final phase indicator field and convert to regular layout - DoubleArray PhaseField(Nx,Ny,Nz); - - //ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N); - -// FILE *OUTFILE; -// sprintf(LocalRankFilename,"Phase.%05i.raw",rank); -// OUTFILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,OUTFILE); -// fclose(OUTFILE); -// - //ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenA, sizeof(double)*N); - FILE *AFILE; - sprintf(LocalRankFilename,"A.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,AFILE); - fclose(AFILE); - - //ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField); - ScaLBL_CopyToHost(PhaseField.data(), DenB, sizeof(double)*N); - FILE *BFILE; - sprintf(LocalRankFilename,"B.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,BFILE); - fclose(BFILE); - - ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,PhaseField); - FILE *PFILE; - sprintf(LocalRankFilename,"Pressure.%05i.raw",rank); - PFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,PFILE); - fclose(PFILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); - FILE *VELX_FILE; - sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); - VELX_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELX_FILE); - fclose(VELX_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); - FILE *VELY_FILE; - sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); - VELY_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELY_FILE); - fclose(VELY_FILE); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); - FILE *VELZ_FILE; - sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); - VELZ_FILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,VELZ_FILE); - fclose(VELZ_FILE); - -// ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField); -// FILE *POROS_FILE; -// sprintf(LocalRankFilename,"Porosity.%05i.raw",rank); -// POROS_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,POROS_FILE); -// fclose(POROS_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField); -// FILE *PERM_FILE; -// sprintf(LocalRankFilename,"Permeability.%05i.raw",rank); -// PERM_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,PERM_FILE); -// fclose(PERM_FILE); -} diff --git a/models/GreyscaleSCModel.h b/models/GreyscaleSCModel.h deleted file mode 100644 index e5e79d68..00000000 --- a/models/GreyscaleSCModel.h +++ /dev/null @@ -1,103 +0,0 @@ -/* -Implementation of color lattice boltzmann model - */ -#include -#include -#include -#include -#include -#include -#include - -#include "common/Communication.h" -#include "common/MPI.h" -#include "common/Database.h" -#include "common/ScaLBL.h" -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" - -class ScaLBL_GreyscaleSCModel{ -public: - ScaLBL_GreyscaleSCModel(int RANK, int NP, MPI_Comm COMM); - ~ScaLBL_GreyscaleSCModel(); - - // functions in they should be run - void ReadParams(string filename); - void ReadParams(std::shared_ptr db0); - void SetDomain(); - void ReadInput(); - void Create(); - void Initialize(); - void Run(); - void WriteDebug(); - void WriteOutput(); - - bool Restart,pBC; - int timestep,timestepMax; - int BoundaryCondition; - int CollisionType; - double tauA,tauB; - double tauA_eff,tauB_eff; - double Gsc; - double rhoA,rhoB; - double rhoA_minor,rhoB_minor;//dissolved density - double tolerance; - double Fx,Fy,Fz; - double fluxA,fluxB; - double dinA,doutA; - double dinB,doutB; - double GreyPorosity; - - int Nx,Ny,Nz,N,Np; - int rank,nprocx,nprocy,nprocz,nprocs; - double Lx,Ly,Lz; - - std::shared_ptr Dm; // this domain is for analysis - std::shared_ptr Mask; // this domain is for lbm - std::shared_ptr ScaLBL_Comm; - std::shared_ptr ScaLBL_Comm_Regular; - - // input database - std::shared_ptr db; - std::shared_ptr domain_db; - std::shared_ptr greyscaleSC_db; - std::shared_ptr analysis_db; - std::shared_ptr vis_db; - - signed char *id; - int *NeighborList; - int *dvcMap; - double *fqA, *fqB; - double *Permeability;//grey voxel permeability - //double relPermA,relPermB;//grey voxel relperm - double *Porosity; - double *Velocity; - double *Pressure_dvc; - double *DenA, *DenB; - double *DenGradA,*DenGradB; - double *SolidForceA,*SolidForceB; - - IntArray Map; - DoubleArray SignDist; - DoubleArray Velocity_x; - DoubleArray Velocity_y; - DoubleArray Velocity_z; - DoubleArray PorosityMap; - DoubleArray Pressure; - DoubleArray DenA_data; - DoubleArray DenB_data; - -private: - MPI_Comm comm; - - int dist_mem_size; - int neighborSize; - // filenames - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - - void AssignGreyscaleAndSolidLabels(); - void Density_Init(); -}; - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2495593a..c536a7ec 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,8 +4,6 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_greyscaleFE_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_greyscaleSC_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) diff --git a/tests/lbpm_greyscaleFE_simulator.cpp b/tests/lbpm_greyscaleFE_simulator.cpp deleted file mode 100644 index 22157bee..00000000 --- a/tests/lbpm_greyscaleFE_simulator.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "common/ScaLBL.h" -#include "common/Communication.h" -#include "common/MPI.h" -#include "models/GreyscaleFEModel.h" -//#define WRITE_SURFACES - -using namespace std; - - -int main(int argc, char **argv) -{ - //***************************************** - // ***** MPI STUFF **************** - //***************************************** - // Initialize MPI - int rank,nprocs; - MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ - printf("****************************************\n"); - printf("Running Greyscale Two-Phase Calculation \n"); - printf("****************************************\n"); - } - // Initialize compute device - int device=ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - ScaLBL_GreyscaleFEModel GreyscaleFE(rank,nprocs,comm); - auto filename = argv[1]; - GreyscaleFE.ReadParams(filename); - GreyscaleFE.SetDomain(); // this reads in the domain - GreyscaleFE.ReadInput(); - GreyscaleFE.Create(); // creating the model will create data structure to match the pore structure and allocate variables - GreyscaleFE.Initialize(); // initializing the model will set initial conditions for variables - GreyscaleFE.Run(); - //GreyscaleFE.VelocityField(); - GreyscaleFE.WriteDebug(); - } - // **************************************************** - MPI_Barrier(comm); - MPI_Finalize(); - // **************************************************** -} diff --git a/tests/lbpm_greyscaleSC_simulator.cpp b/tests/lbpm_greyscaleSC_simulator.cpp deleted file mode 100644 index 340be938..00000000 --- a/tests/lbpm_greyscaleSC_simulator.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "common/ScaLBL.h" -#include "common/Communication.h" -#include "common/MPI.h" -#include "models/GreyscaleSCModel.h" -//#define WRITE_SURFACES - -using namespace std; - - -int main(int argc, char **argv) -{ - //***************************************** - // ***** MPI STUFF **************** - //***************************************** - // Initialize MPI - int rank,nprocs; - MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ - printf("****************************************\n"); - printf("Running Greyscale Two-Phase Calculation \n"); - printf("****************************************\n"); - } - // Initialize compute device - int device=ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - ScaLBL_GreyscaleSCModel GreyscaleSC(rank,nprocs,comm); - auto filename = argv[1]; - GreyscaleSC.ReadParams(filename); - GreyscaleSC.SetDomain(); // this reads in the domain - GreyscaleSC.ReadInput(); - GreyscaleSC.Create(); // creating the model will create data structure to match the pore structure and allocate variables - GreyscaleSC.Initialize(); // initializing the model will set initial conditions for variables - GreyscaleSC.Run(); - GreyscaleSC.WriteDebug(); - } - // **************************************************** - MPI_Barrier(comm); - MPI_Finalize(); - // **************************************************** -} From 8996b582da861f28ccb95bae3b5c273069ef66b1 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 19 Aug 2020 13:21:31 -0400 Subject: [PATCH 213/270] still debugging; add a few checkpoint print out --- cpu/Ion.cpp | 14 ++++++++++++++ models/IonModel.cpp | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index 2c7e72a9..82398e92 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -1,7 +1,9 @@ +#include extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){ int n,nread; double fq,Ci; + printf("ScaLBL_D3Q7_AAodd_IonConcentration: entering the kernel successfully\n"); for (n=start; n Date: Wed, 19 Aug 2020 18:56:06 -0400 Subject: [PATCH 214/270] fix trivial bug in tau initialization --- cpu/Ion.cpp | 13 ------------- models/IonModel.cpp | 10 +--------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index 82398e92..ea4f39f4 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -3,7 +3,6 @@ extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){ int n,nread; double fq,Ci; - printf("ScaLBL_D3Q7_AAodd_IonConcentration: entering the kernel successfully\n"); for (n=start; nSendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -391,9 +390,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } //LB-Ion collison - if (rank==0) printf("timestep=%i; execute collision step 1/2\n",timestep); for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); } @@ -401,9 +398,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // Set boundary conditions /* ... */ - if (rank==0) printf("timestep=%i; execute collision step 2/2\n",timestep); for (int ic=0; icLastExterior(), Np); } @@ -418,7 +413,6 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // *************EVEN TIMESTEP*************// timestep++; //Update ion concentration and charge density - if (rank==0) printf("timestep=%i; updating ion concentration and charge density\n",timestep); for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -429,7 +423,6 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } //LB-Ion collison - if (rank==0) printf("timestep=%i; execute collision step 1/2\n",timestep); for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -438,7 +431,6 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ // Set boundary conditions /* ... */ - if (rank==0) printf("timestep=%i; execute collision step 2/2\n",timestep); for (int ic=0; icLastExterior(), Np); From d24198e76a278cc8cd32d6435480555087c05b95 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 19 Aug 2020 22:22:35 -0400 Subject: [PATCH 215/270] done cleaning up the code --- models/GreyscaleColorModel.cpp | 6 +- models/GreyscaleModel.cpp | 2 +- tests/lbpm_greyscaleColor_simulator.cpp | 93 ++++++++++++++---------- tests/lbpm_greyscale_simulator.cpp | 97 ++++++++++++++----------- 4 files changed, 110 insertions(+), 88 deletions(-) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 6525c09c..d97c844d 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -596,15 +596,15 @@ void ScaLBL_GreyscaleColorModel::AssignGreyPoroPermLabels() if (rank==0){ printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); - printf("Number of component labels: %lu \n",NLABELS); + printf("Number of Grey-fluid labels: %lu \n",NLABELS); for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); - printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); } printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); } diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index f2f94f9d..85832a4b 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -274,7 +274,7 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm if (rank==0){ printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); - printf("Component labels: %lu \n",NLABELS); + printf("Number of component labels: %lu \n",NLABELS); for (unsigned int idx=0; idx #include -#include "common/ScaLBL.h" -#include "common/Communication.h" -#include "common/MPI.h" #include "models/GreyscaleColorModel.h" +#include "common/Utilities.h" //#define WRITE_SURFACES +//************************************************************************* +// Implementation of Greyscale Two-Fluid Color LBM using CUDA +//************************************************************************* + using namespace std; int main(int argc, char **argv) { - //***************************************** - // ***** MPI STUFF **************** - //***************************************** - // Initialize MPI - int rank,nprocs; - MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - if (rank == 0){ - printf("****************************************\n"); - printf("Running Greyscale Two-Phase Calculation \n"); - printf("****************************************\n"); - } - // Initialize compute device - int device=ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm); - auto filename = argv[1]; - GreyscaleColor.ReadParams(filename); - GreyscaleColor.SetDomain(); // this reads in the domain - GreyscaleColor.ReadInput(); - GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables - GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables - GreyscaleColor.Run(); - GreyscaleColor.WriteDebug(); - } - // **************************************************** - MPI_Barrier(comm); - MPI_Finalize(); - // **************************************************** + // Initialize MPI and error handlers + Utilities::startup( argc, argv ); + + { // Limit scope so variables that contain communicators will free before MPI_Finialize + + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + + if (rank == 0){ + printf("****************************************\n"); + printf("Running Greyscale Two-Phase Calculation \n"); + printf("****************************************\n"); + } + // Initialize compute device + ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); + + auto filename = argv[1]; + ScaLBL_GreyscaleColorModel GreyscaleColor(rank,nprocs,comm); + GreyscaleColor.ReadParams(filename); + GreyscaleColor.SetDomain(); + GreyscaleColor.ReadInput(); + GreyscaleColor.Create(); // creating the model will create data structure to match the pore structure and allocate variables + GreyscaleColor.Initialize(); // initializing the model will set initial conditions for variables + GreyscaleColor.Run(); + GreyscaleColor.WriteDebug(); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_greyscaleColor_simulator",1); + // **************************************************** + + MPI_Barrier(comm); + MPI_Comm_free(&comm); + + } // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::shutdown(); + } diff --git a/tests/lbpm_greyscale_simulator.cpp b/tests/lbpm_greyscale_simulator.cpp index 782ed5b5..df8cb3cb 100644 --- a/tests/lbpm_greyscale_simulator.cpp +++ b/tests/lbpm_greyscale_simulator.cpp @@ -6,58 +6,67 @@ #include #include -#include "common/ScaLBL.h" -#include "common/Communication.h" -#include "common/MPI_Helpers.h" #include "models/GreyscaleModel.h" +#include "common/Utilities.h" //#define WRITE_SURFACES -/* - * Simulator for two-phase flow in porous media - * James E. McClure 2013-2014 - */ +//**************************************************************** +// Implementation of Greyscale Single-Fluid LBM using CUDA +//**************************************************************** using namespace std; int main(int argc, char **argv) { - //***************************************** - // ***** MPI STUFF **************** - //***************************************** - // Initialize MPI - int rank,nprocs; - MPI_Init(&argc,&argv); - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { - // parallel domain size (# of sub-domains) - if (rank == 0){ - printf("********************************************************\n"); - printf("Running Greyscale Single Phase Permeability Calculation \n"); - printf("********************************************************\n"); - } - // Initialize compute device - int device=ScaLBL_SetDevice(rank); - NULL_USE(device); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - ScaLBL_GreyscaleModel Greyscale(rank,nprocs,comm); - auto filename = argv[1]; - Greyscale.ReadParams(filename); - Greyscale.SetDomain(); // this reads in the domain - Greyscale.ReadInput(); - Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables - Greyscale.Initialize(); // initializing the model will set initial conditions for variables - Greyscale.Run(); - Greyscale.VelocityField(); - //Greyscale.WriteDebug(); - } - // **************************************************** - MPI_Barrier(comm); - MPI_Finalize(); - // **************************************************** + // Initialize MPI and error handlers + Utilities::startup( argc, argv ); + + { // Limit scope so variables that contain communicators will free before MPI_Finialize + + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Greyscale Single Phase Permeability Calculation \n"); + printf("********************************************************\n"); + } + // Initialize compute device + ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); + + auto filename = argv[1]; + ScaLBL_GreyscaleModel Greyscale(rank,nprocs,comm); + Greyscale.ReadParams(filename); + Greyscale.SetDomain(); + Greyscale.ReadInput(); + Greyscale.Create(); // creating the model will create data structure to match the pore structure and allocate variables + Greyscale.Initialize(); // initializing the model will set initial conditions for variables + Greyscale.Run(); + Greyscale.VelocityField(); + //Greyscale.WriteDebug(); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_greyscale_simulator",1); + // **************************************************** + + MPI_Barrier(comm); + MPI_Comm_free(&comm); + + } // Limit scope so variables that contain communicators will free before MPI_Finialize + + Utilities::shutdown(); + } From 59ffd7bfd66cbfa609386a069bd45aa34ba943bf Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 20 Aug 2020 22:47:10 -0400 Subject: [PATCH 216/270] fix several miscellaneous bugs --- models/IonModel.cpp | 20 +++++++++++--------- models/PoissonSolver.cpp | 2 +- models/StokesModel.cpp | 17 +++++++++-------- tests/lbpm_electrokinetic_dfh_simulator.cpp | 3 +++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index d6265abe..55c848b9 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -48,6 +48,15 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); //--------------------------------------------------------------------------// + // Read domain parameters + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + // LB-Ion Model parameters //if (ion_db->keyExists( "timestepMax" )){ // timestepMax = ion_db->getScalar( "timestepMax" ); @@ -112,14 +121,6 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke BoundaryConditionSolid = ion_db->getScalar( "BC_Solid" ); } - // Read domain parameters - if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu - h = domain_db->getScalar( "voxel_length" ); - } - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } if (rank==0) printf("*****************************************************\n"); if (rank==0) printf("LB Ion Transport Solver: \n"); @@ -275,7 +276,7 @@ void ScaLBL_IonModel::AssignSolidBoundary(double *ion_solid) label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]); if (rank==0){ - printf("LB Ion Solver: Ion Solid labels: %lu \n",NLABELS); + printf("LB Ion Solver: number of ion solid labels: %lu \n",NLABELS); for (unsigned int idx=0; idxFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_IonConcentration_Phys(Ci, h, ic, 0, ScaLBL_Comm->LastExterior(), Np); } DoubleArray PhaseField(Nx,Ny,Nz); diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 3cf8a6d2..701c009c 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -224,7 +224,7 @@ void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]); if (rank==0){ - printf("LB-Poisson Solver: Poisson Solid labels: %lu \n",NLABELS); + printf("LB-Poisson Solver: number of Poisson solid labels: %lu \n",NLABELS); for (unsigned int idx=0; idxkeyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + // Single-fluid Navier-Stokes Model parameters //if (stokes_db->keyExists( "timestepMax" )){ // timestepMax = stokes_db->getScalar( "timestepMax" ); @@ -75,14 +84,6 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ if (stokes_db->keyExists( "flux" )){ flux = stokes_db->getScalar( "flux" ); } - - // Read domain parameters - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } - if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu - h = domain_db->getScalar( "voxel_length" ); - } // Re-calculate model parameters due to parameter read mu=(tau-0.5)/3.0; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 7cf835da..156fbc18 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -98,6 +98,9 @@ int main(int argc, char **argv) PoissonSolver.getElectricalPotential(); IonModel.getIonConcentration(); + if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); + if (rank==0) printf("*************************************************************\n"); + PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_electrokinetic_simulator",1); // **************************************************** From aa26fcafdaac09beb2e572efd9c1d98bc20f6131 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 28 Aug 2020 11:15:55 -0400 Subject: [PATCH 217/270] fix miscellaneous bugs and update the data structure of electric potential --- common/ScaLBL.cpp | 33 ++ common/ScaLBL.h | 40 +- cpu/D3Q7BC.cpp | 107 ++++++ cpu/Poisson.cpp | 350 +++++++++++++++--- cpu/Stokes.cpp | 162 ++++----- models/IonModel.cpp | 31 +- models/IonModel.h | 3 +- models/PoissonSolver.cpp | 383 ++++++++++++++++++-- models/PoissonSolver.h | 11 +- models/StokesModel.cpp | 90 +++-- models/StokesModel.h | 7 +- tests/lbpm_electrokinetic_dfh_simulator.cpp | 16 +- 12 files changed, 1027 insertions(+), 206 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 854c8b49..ed7c5e59 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2053,3 +2053,36 @@ void ScaLBL_Communicator::PrintD3Q19(){ delete [] TempBuffer; } +void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time){ + if (kproc == 0) { + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(dvcSendList_z, fq, Vin, sendCount_z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(neighborList, dvcSendList_z, fq, Vin, sendCount_z, N); + } + } +} + +void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time){ + if (kproc == nprocz-1){ + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(dvcSendList_Z, fq, Vout, sendCount_Z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(neighborList, dvcSendList_Z, fq, Vout, sendCount_Z, N); + } + } +} + +void ScaLBL_Communicator::Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin){ + if (kproc == 0) { + ScaLBL_Poisson_D3Q7_BC_z(dvcSendList_z, Map, Psi, Vin, sendCount_z); + } +} + +void ScaLBL_Communicator::Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout){ + if (kproc == nprocz-1){ + ScaLBL_Poisson_D3Q7_BC_Z(dvcSendList_Z, Map, Psi, Vout, sendCount_Z); + } +} diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 4d8a3dc3..414653a8 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -46,11 +46,8 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np); - extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np); -extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np); - extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np); // BGK MODEL @@ -95,21 +92,30 @@ extern "C" void ScaLBL_IonConcentration_Phys(double *Den, double h, int ion_comp // LBM Poisson solver -extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_Poisson(double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_Poisson_Init(double *dist, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np); + +extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np); + +extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, + int strideY, int strideZ,int start, int finish, int Np); // LBM Stokes Model (adapted from MRT model) extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, double Ex, double Ey, double Ez, int start, int finish, int Np); + double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, double Ex, double Ey, double Ez, int start, int finish, int Np); +extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, + double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np); // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, @@ -190,6 +196,18 @@ extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,i extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N); +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np); + +extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count); + +extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count); + class ScaLBL_Communicator{ public: //...................................................................................... @@ -249,6 +267,10 @@ public: void D3Q19_Reflection_BC_z(double *fq); void D3Q19_Reflection_BC_Z(double *fq); double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time); + void D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time); + void D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time); + void Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin); + void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout); // Debugging and unit testing functions void PrintD3Q19(); diff --git a/cpu/D3Q7BC.cpp b/cpu/D3Q7BC.cpp index e7bfd3a4..8c2588d8 100644 --- a/cpu/D3Q7BC.cpp +++ b/cpu/D3Q7BC.cpp @@ -30,3 +30,110 @@ extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int } } +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){ + for (int idx=0; idx0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1 + //........................................................................ + nn = ijk+1; // neighbor index (get convention) + id = ID[nn]; + m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2 + //........................................................................ + nn = ijk-strideY; // neighbor index (get convention) + id = ID[nn]; + m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3 + //........................................................................ + nn = ijk+strideY; // neighbor index (get convention) + id = ID[nn]; + m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4 + //........................................................................ + nn = ijk-strideZ; // neighbor index (get convention) + id = ID[nn]; + m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5 + //........................................................................ + nn = ijk+strideZ; // neighbor index (get convention) + id = ID[nn]; + m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6 + //........................................................................ + nn = ijk-strideY-1; // neighbor index (get convention) + id = ID[nn]; + m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7 + //........................................................................ + nn = ijk+strideY+1; // neighbor index (get convention) + id = ID[nn]; + m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8 + //........................................................................ + nn = ijk+strideY-1; // neighbor index (get convention) + id = ID[nn]; + m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9 + //........................................................................ + nn = ijk-strideY+1; // neighbor index (get convention) + id = ID[nn]; + m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10 + //........................................................................ + nn = ijk-strideZ-1; // neighbor index (get convention) + id = ID[nn]; + m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11 + //........................................................................ + nn = ijk+strideZ+1; // neighbor index (get convention) + id = ID[nn]; + m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12 + //........................................................................ + nn = ijk+strideZ-1; // neighbor index (get convention) + id = ID[nn]; + m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13 + //........................................................................ + nn = ijk-strideZ+1; // neighbor index (get convention) + id = ID[nn]; + m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14 + //........................................................................ + nn = ijk-strideZ-strideY; // neighbor index (get convention) + id = ID[nn]; + m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15 + //........................................................................ + nn = ijk+strideZ+strideY; // neighbor index (get convention) + id = ID[nn]; + m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16 + //........................................................................ + nn = ijk+strideZ-strideY; // neighbor index (get convention) + id = ID[nn]; + m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17 + //........................................................................ + nn = ijk-strideZ+strideY; // neighbor index (get convention) + id = ID[nn]; + m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18 + //............Compute the Color Gradient................................... + nx = -1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + ny = -1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + nz = -1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + + ElectricField[n] = nx; + ElectricField[Np+n] = ny; + ElectricField[2*Np+n] = nz; + } +} + diff --git a/cpu/Stokes.cpp b/cpu/Stokes.cpp index a31a8bed..a3842345 100644 --- a/cpu/Stokes.cpp +++ b/cpu/Stokes.cpp @@ -1,7 +1,6 @@ #include -extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, double Ex_const, double Ey_const, double Ez_const, int start, int finish, int Np) +extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np) { double fq; // conserved momemnts @@ -32,13 +31,13 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do //Load data rhoE = ChargeDensity[n]; - Ex = ElectricField[n+0*Np]+Ex_const; - Ey = ElectricField[n+1*Np]+Ey_const; - Ez = ElectricField[n+2*Np]+Ez_const; + Ex = ElectricField[n+0*Np]; + Ey = ElectricField[n+1*Np]; + Ez = ElectricField[n+2*Np]; //compute total body force, including input body force (Gx,Gy,Gz) - Fx = Gx + rhoE*Ex; - Fy = Gy + rhoE*Ey; - Fz = Gz + rhoE*Ez; + Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB + Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; // q=0 fq = dist[n]; @@ -311,9 +310,9 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do m18 -= fq; // write the velocity - ux = jx / rho; - uy = jy / rho; - uz = jz / rho; + ux = jx / rho0; + uy = jy / rho0; + uz = jz / rho0; Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; @@ -326,18 +325,18 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2); m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); - m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); - m13 = m13 + rlx_setA*((jx*jy/rho) - m13); - m14 = m14 + rlx_setA*((jy*jz/rho) - m14); - m15 = m15 + rlx_setA*((jx*jz/rho) - m15); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12); + m13 = m13 + rlx_setA*((jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*((jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*((jx*jz/rho0) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); @@ -454,8 +453,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, do } } -extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, - double Gx, double Gy, double Gz, double Ex_const, double Ey_const, double Ez_const, int start, int finish, int Np) +extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np) { double fq; // conserved momemnts @@ -487,13 +485,13 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do //Load data rhoE = ChargeDensity[n]; - Ex = ElectricField[n+0*Np]+Ex_const; - Ey = ElectricField[n+1*Np]+Ey_const; - Ez = ElectricField[n+2*Np]+Ez_const; + Ex = ElectricField[n+0*Np]; + Ey = ElectricField[n+1*Np]; + Ez = ElectricField[n+2*Np]; //compute total body force, including input body force (Gx,Gy,Gz) - Fx = Gx + rhoE*Ex; - Fy = Gy + rhoE*Ey; - Fz = Gz + rhoE*Ez; + Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; // q=0 fq = dist[n]; @@ -803,27 +801,27 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do m18 -= fq; // write the velocity - ux = jx / rho; - uy = jy / rho; - uz = jz / rho; + ux = jx / rho0; + uy = jy / rho0; + uz = jz / rho0; Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; //..............incorporate external force................................................ //..............carry out relaxation process............................................... - m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1); - m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2); + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2); m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); - m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); - m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11); - m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12); - m13 = m13 + rlx_setA*((jx*jy/rho) - m13); - m14 = m14 + rlx_setA*((jy*jz/rho) - m14); - m15 = m15 + rlx_setA*((jx*jz/rho) - m15); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12); + m13 = m13 + rlx_setA*((jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*((jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*((jx*jz/rho0) - m15); m16 = m16 + rlx_setB*( - m16); m17 = m17 + rlx_setB*( - m17); m18 = m18 + rlx_setB*( - m18); @@ -955,47 +953,47 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do } } -extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np) -{ - //h: resolution [um/lu] - //time_conv: time conversion factor [sec/lt] - int n; - // distributions - double f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double vx,vy,vz; - - for (n=0; n rlx(tau.begin(),tau.end()); for (double item : rlx){ item = 1.0/item; } + //.......create and start timer............ //double starttime,stoptime,cputime; //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -462,7 +467,9 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } -void ScaLBL_IonModel::getIonConcentration(){ +//TODO this ruin the ion concentration on device +//need to do something similar to electric field +void ScaLBL_IonModel::getIonConcentration(int timestep){ for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_IonConcentration_Phys(Ci, h, ic, 0, ScaLBL_Comm->LastExterior(), Np); @@ -474,7 +481,7 @@ void ScaLBL_IonModel::getIonConcentration(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); FILE *OUTFILE; - sprintf(LocalRankFilename,"Ion%02i.%05i.raw",ic+1,rank); + sprintf(LocalRankFilename,"Ion%02i_Time_%i.%05i.raw",ic+1,timestep,rank); OUTFILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,OUTFILE); fclose(OUTFILE); @@ -482,3 +489,23 @@ void ScaLBL_IonModel::getIonConcentration(){ } +//void ScaLBL_IonModel::getIonConcentration(){ +// for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_IonConcentration_Phys(Ci, h, ic, 0, ScaLBL_Comm->LastExterior(), Np); +// } +// +// DoubleArray PhaseField(Nx,Ny,Nz); +// for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); +// ScaLBL_DeviceBarrier(); MPI_Barrier(comm); +// +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Ion%02i.%05i.raw",ic+1,rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +// } +// +//} + diff --git a/models/IonModel.h b/models/IonModel.h index 6232d105..59e5b6e6 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -29,7 +29,7 @@ public: void Create(); void Initialize(); void Run(double *Velocity, double *ElectricField); - void getIonConcentration(); + void getIonConcentration(int timestep); //bool Restart,pBC; int timestep,timestepMax; @@ -40,6 +40,7 @@ public: double kb,electron_charge,T,Vt; double k2_inv; double tolerance; + double Ex,Ey,Ez; int number_ion_species; vector IonDiffusivity;//User input unit [m^2/sec] diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 701c009c..c3e5c019 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -7,7 +7,7 @@ ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),gamma(0),tolerance(0),h(0), -epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), +epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -22,17 +22,20 @@ void ScaLBL_Poisson::ReadParams(string filename){ domain_db = db->getDatabase( "Domain" ); electric_db = db->getDatabase( "Poisson" ); - k2_inv = 4.5;//speed of sound for D3Q7 lattice - gamma = 0.3;//time step of LB-Poisson equation + //k2_inv = 4.5;//speed of sound for D3Q7 lattice + k2_inv = 4.0;//speed of sound for D3Q7 lattice + gamma = 1.0;//time step of LB-Poisson equation tau = 0.5+k2_inv*gamma; timestepMax = 100000; tolerance = 1.0e-6;//stopping criterion for obtaining steady-state electricla potential h = 1.0;//resolution; unit: um/lu - epsilon0 = 8.85e-12;//electrical permittivity of vaccum; unit:[C/(V*m)] + epsilon0 = 8.85e-12;//electric permittivity of vaccum; unit:[C/(V*m)] epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] epsilonR = 78.4;//default dielectric constant of water - epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity + epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity analysis_interval = 1000; + Vin = 1.0; //Boundary-z (inlet) electric potential + Vout = 1.0; //Boundary-Z (outlet) electric potential // LB-Poisson Model parameters if (electric_db->keyExists( "timestepMax" )){ @@ -56,20 +59,23 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "BC_Solid" )){ BoundaryConditionSolid = electric_db->getScalar( "BC_Solid" ); } + // Read boundary condition for electric potentiona + // BC = 0: normal periodic BC + // BC = 1: fixed inlet and outlet potential + BoundaryCondition = 0; + if (electric_db->keyExists( "BC" )){ + BoundaryCondition = electric_db->getScalar( "BC" ); + } // Read domain parameters if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } //Re-calcualte model parameters if user updates input epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] - epsilon_LB = epsilon0_LB*epsilonR;//electrical permittivity + epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity tau = 0.5+k2_inv*gamma; if (rank==0) printf("***********************************************************************************\n"); @@ -202,13 +208,13 @@ void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) AFFINITY=0.f; // Assign the affinity from the paired list for (unsigned int idx=0; idx < NLABELS; idx++){ - //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); if (VALUE == LabelList[idx]){ AFFINITY=AffinityList[idx]; //NOTE need to convert the user input phys unit to LB unit if (BoundaryConditionSolid==2){ //for BCS=1, i.e. Dirichlet-type, no need for unit conversion - AFFINITY = AFFINITY*(h*h*1.0e-12); + //TODO maybe there is a factor of gamm missing here ? + AFFINITY = AFFINITY*(h*h*1.0e-12)/epsilon_LB; } label_count[idx] += 1.0; idx = NLABELS; @@ -244,7 +250,6 @@ void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) } } - void ScaLBL_Poisson::Create(){ /* * This function creates the variables needed to run a LBM @@ -260,6 +265,7 @@ void ScaLBL_Poisson::Create(){ // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm_Regular = std::shared_ptr(new ScaLBL_Communicator(Mask)); int Npad=(Np/16 + 2)*16; if (rank==0) printf ("LB-Poisson Solver: Set up memory efficient layout \n"); @@ -277,37 +283,125 @@ void ScaLBL_Poisson::Create(){ int neighborSize=18*(Np*sizeof(int)); //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &dvcID, sizeof(signed char)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &fq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &ElectricField, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &PoissonSolid, sizeof(double)*Nx*Ny*Nz); + //ScaLBL_AllocateDeviceMemory((void **) &PoissonSolid, sizeof(double)*Nx*Ny*Nz); //........................................................................... // Update GPU data structures if (rank==0) printf ("LB-Poisson Solver: Setting up device map and neighbor list \n"); + fflush(stdout); + int *TmpMap; + TmpMap=new int[Np]; + for (int k=1; kLastExterior(); idx++){ + auto n = TmpMap[idx]; + if (n > Nx*Ny*Nz){ + printf("Bad value! idx=%i \n", n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + auto n = TmpMap[idx]; + if ( n > Nx*Ny*Nz ){ + printf("Bad value! idx=%i \n",n); + TmpMap[idx] = Nx*Ny*Nz-1; + } + } + ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); + ScaLBL_DeviceBarrier(); + delete [] TmpMap; // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + delete [] neighborList; + // copy node ID + ScaLBL_CopyToDevice(dvcID, Mask->id, sizeof(signed char)*Nx*Ny*Nz); + ScaLBL_DeviceBarrier(); - //Initialize solid boundary for electrical potential + //Initialize solid boundary for electric potential ScaLBL_Comm->SetupBounceBackList(Map, Mask->id, Np); MPI_Barrier(comm); - double *PoissonSolid_host; - PoissonSolid_host = new double[Nx*Ny*Nz]; - AssignSolidBoundary(PoissonSolid_host); - ScaLBL_CopyToDevice(PoissonSolid, PoissonSolid_host, Nx*Ny*Nz*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] PoissonSolid_host; + //double *PoissonSolid_host; + //PoissonSolid_host = new double[Nx*Ny*Nz]; + //AssignSolidBoundary(PoissonSolid_host); + //ScaLBL_CopyToDevice(PoissonSolid, PoissonSolid_host, Nx*Ny*Nz*sizeof(double)); + //ScaLBL_DeviceBarrier(); + //delete [] PoissonSolid_host; } +// Method 1 +// Psi - size N +// ID_dvc - size N +// Method 2 +// Psi - size Np +// PoissonSolid size N + +void ScaLBL_Poisson::Potential_Init(double *psi_init){ + + if (BoundaryCondition==1){ + if (electric_db->keyExists( "Vin" )){ + Vin = electric_db->getScalar( "Vin" ); + } + if (electric_db->keyExists( "Vout" )){ + Vout = electric_db->getScalar( "Vout" ); + } + } + //By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis + double slope = (Vout-Vin)/(Nz-2); + double psi_linearized; + for (int k=0;kid[n]>0){ + psi_init[n] = psi_linearized; + } + } + } + } +} + void ScaLBL_Poisson::Initialize(){ /* * This function initializes model */ if (rank==0) printf ("LB-Poisson Solver: initializing D3Q7 distributions\n"); - ScaLBL_D3Q7_Poisson_Init(fq, Np); + //NOTE the initialization involves two steps: + //1. assign solid boundary value (surface potential or surface change density) + //2. Initialize electric potential for pore nodes + double *psi_host; + psi_host = new double [Nx*Ny*Nz]; + AssignSolidBoundary(psi_host);//step1 + Potential_Init(psi_host);//step2 + ScaLBL_CopyToDevice(Psi, psi_host, Nx*Ny*Nz*sizeof(double)); + ScaLBL_DeviceBarrier(); + ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); + delete [] psi_host; } void ScaLBL_Poisson::Run(double *ChargeDensity){ @@ -325,30 +419,83 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ // *************ODD TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); // Set boundary conditions - /* ... */ - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->SolidDirichletD3Q7(fq, PoissonSolid); + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); + + //compute electric field + ScaLBL_Comm_Regular->SendHalo(Psi); + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, + Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Psi); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition == 1){ + ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); + ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); + } + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + + //perform collision + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); // Set boundary conditions - /* ... */ - ScaLBL_D3Q7_AAeven_Poisson(fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_Comm->SolidDirichletD3Q7(fq, PoissonSolid); + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); + + //compute electric field + ScaLBL_Comm_Regular->SendHalo(Psi); + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, + Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Psi); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition == 1){ + ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); + ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); + } + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + + //perform collision + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ // Check convergence of steady-state solution if (timestep%analysis_interval==0){ - ScaLBL_Comm->RegularLayout(Map,Psi,Psi_host); + //ScaLBL_Comm->RegularLayout(Map,Psi,Psi_host); + ScaLBL_CopyToHost(Psi_host.data(),Psi,sizeof(double)*Nx*Ny*Nz); double count_loc=0; double count; double psi_avg; @@ -393,14 +540,180 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ } -void ScaLBL_Poisson::getElectricalPotential(){ +void ScaLBL_Poisson::getElectricPotential(int timestep){ DoubleArray PhaseField(Nx,Ny,Nz); - ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); + //ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); + ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz); //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); FILE *OUTFILE; - sprintf(LocalRankFilename,"Electrical_Potential.%05i.raw",rank); + sprintf(LocalRankFilename,"Electric_Potential_Time_%i.%05i.raw",timestep,rank); OUTFILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,OUTFILE); fclose(OUTFILE); } + +void ScaLBL_Poisson::getElectricField(int timestep){ + + //ScaLBL_D3Q7_Poisson_getElectricField(fq,ElectricField,tau,Np); + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + FILE *EX; + sprintf(LocalRankFilename,"ElectricField_X_Time_%i.%05i.raw",timestep,rank); + EX = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,EX); + fclose(EX); + + ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + FILE *EY; + sprintf(LocalRankFilename,"ElectricField_Y_Time_%i.%05i.raw",timestep,rank); + EY = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,EY); + fclose(EY); + + ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + FILE *EZ; + sprintf(LocalRankFilename,"ElectricField_Z_Time_%i.%05i.raw",timestep,rank); + EZ = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,EZ); + fclose(EZ); +} + + +void ScaLBL_Poisson::ElectricField_LB_to_Phys(DoubleArray &Efield_reg){ + for (int k=0;kRegularLayout(Map,Psi,PhaseField); +// //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); +// FILE *OUTFILE; +// sprintf(LocalRankFilename,"Electric_Potential.%05i.raw",rank); +// OUTFILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,OUTFILE); +// fclose(OUTFILE); +//} + +//old version where Psi is of size Np +//void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) +//{ +// size_t NLABELS=0; +// signed char VALUE=0; +// double AFFINITY=0.f; +// +// auto LabelList = electric_db->getVector( "SolidLabels" ); +// auto AffinityList = electric_db->getVector( "SolidValues" ); +// +// NLABELS=LabelList.size(); +// if (NLABELS != AffinityList.size()){ +// ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n"); +// } +// +// double label_count[NLABELS]; +// double label_count_global[NLABELS]; +// // Assign the labels +// +// for (size_t idx=0; idxid[n]; +// AFFINITY=0.f; +// // Assign the affinity from the paired list +// for (unsigned int idx=0; idx < NLABELS; idx++){ +// //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]); +// if (VALUE == LabelList[idx]){ +// AFFINITY=AffinityList[idx]; +// //NOTE need to convert the user input phys unit to LB unit +// if (BoundaryConditionSolid==2){ +// //for BCS=1, i.e. Dirichlet-type, no need for unit conversion +// //TODO maybe there is a factor of gamm missing here ? +// AFFINITY = AFFINITY*(h*h*1.0e-12)/epsilon_LB; +// } +// label_count[idx] += 1.0; +// idx = NLABELS; +// //Mask->id[n] = 0; // set mask to zero since this is an immobile component +// } +// } +// poisson_solid[n] = AFFINITY; +// } +// } +// } +// +// for (size_t idx=0; idxComm, label_count[idx]); +// +// if (rank==0){ +// printf("LB-Poisson Solver: number of Poisson solid labels: %lu \n",NLABELS); +// for (unsigned int idx=0; idxkeyExists( "Vin" )){ +// Vin = electric_db->getScalar( "Vin" ); +// } +// if (electric_db->keyExists( "Vout" )){ +// Vout = electric_db->getScalar( "Vout" ); +// } +// } +// //By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis +// double slope = (Vout-Vin)/(Nz-2); +// double psi_linearized; +// for (int k=0;k Dm; // this domain is for analysis std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; + std::shared_ptr ScaLBL_Comm_Regular; // input database std::shared_ptr db; std::shared_ptr domain_db; @@ -57,10 +60,12 @@ public: DoubleArray Distance; DoubleArray Psi_host; int *NeighborList; + int *dvcMap; + signed char *dvcID; double *fq; double *Psi; double *ElectricField; - double *PoissonSolid; + //double *PoissonSolid; private: MPI_Comm comm; @@ -73,4 +78,6 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); void AssignSolidBoundary(double *poisson_solid); + void Potential_Init(double *psi_init); + void ElectricField_LB_to_Phys(DoubleArray &Efield_reg); }; diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 41520253..caaf2877 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -7,7 +7,7 @@ ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), -Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),h(0),nu_phys(0),time_conv(0),tolerance(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),h(0),nu_phys(0),rho_phys(0),rho0(0),den_scale(0),time_conv(0),tolerance(0), Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -27,17 +27,17 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ //-------------------------------------------------------------------// //---------------------- Default model parameters --------------------------// + rho_phys = 1000.0; //by default use water density; unit [kg/m^3] nu_phys = 1.004e-6;//by default use water kinematic viscosity at 20C; unit [m^2/sec] h = 1.0;//image resolution;[um] tau = 1.0; mu = (tau-0.5)/3.0;//LB kinematic viscosity;unit [lu^2/lt] time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + rho0 = 1.0;//LB density + den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density tolerance = 1.0e-8; Fx = Fy = 0.0; Fz = 1.0e-5; - //Body electric field [V/lu] - Ex = Ey = 0.0; - Ez = 1.0e-3; //--------------------------------------------------------------------------// // Read domain parameters @@ -59,19 +59,20 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ if (stokes_db->keyExists( "tau" )){ tau = stokes_db->getScalar( "tau" ); } + if (stokes_db->keyExists( "rho0" )){ + rho0 = stokes_db->getScalar( "rho0" ); + } if (stokes_db->keyExists( "nu_phys" )){ nu_phys = stokes_db->getScalar( "nu_phys" ); } + if (stokes_db->keyExists( "rho_phys" )){ + rho_phys = stokes_db->getScalar( "rho_phys" ); + } if (stokes_db->keyExists( "F" )){ Fx = stokes_db->getVector( "F" )[0]; Fy = stokes_db->getVector( "F" )[1]; Fz = stokes_db->getVector( "F" )[2]; } - if (stokes_db->keyExists( "ElectricField" )){//NOTE user-input has physical unit [V/m] - Ex = stokes_db->getVector( "ElectricField" )[0]; - Ey = stokes_db->getVector( "ElectricField" )[1]; - Ez = stokes_db->getVector( "ElectricField" )[2]; - } if (stokes_db->keyExists( "Restart" )){ Restart = stokes_db->getScalar( "Restart" ); } @@ -88,10 +89,7 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ // Re-calculate model parameters due to parameter read mu=(tau-0.5)/3.0; time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] - // convert user-input electric field ([V/m]) from physical unit to LB unit - Ex = Ex*(h*1.0e-6);//LB electric field: V/lu - Ey = Ey*(h*1.0e-6); - Ez = Ez*(h*1.0e-6); + den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density if (rank==0) printf("*****************************************************\n"); if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n"); @@ -246,7 +244,7 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ while (timestep < timestepMax) { //************************************************************************/ ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions @@ -262,12 +260,14 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, + 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, + ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions if (BoundaryCondition == 3){ @@ -282,41 +282,87 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz, Ex, Ey, Ez, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, + 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ } } -void ScaLBL_StokesModel::getVelocity(){ +void ScaLBL_StokesModel::getVelocity(int timestep){ //get velocity in physical unit [m/sec] - ScaLBL_D3Q19_Momentum_Phys(fq, Velocity, h, time_conv, Np); + ScaLBL_D3Q19_Momentum(fq, Velocity, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); DoubleArray PhaseField(Nx,Ny,Nz); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + Velocity_LB_to_Phys(PhaseField); FILE *VELX_FILE; - sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); + sprintf(LocalRankFilename,"Velocity_X_Time_%i.%05i.raw",timestep,rank); VELX_FILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,VELX_FILE); fclose(VELX_FILE); ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + Velocity_LB_to_Phys(PhaseField); FILE *VELY_FILE; - sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); + sprintf(LocalRankFilename,"Velocity_Y_Time_%i.%05i.raw",timestep,rank); VELY_FILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,VELY_FILE); fclose(VELY_FILE); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + Velocity_LB_to_Phys(PhaseField); FILE *VELZ_FILE; - sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); + sprintf(LocalRankFilename,"Velocity_Z_Time_%i.%05i.raw",timestep,rank); VELZ_FILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,VELZ_FILE); fclose(VELZ_FILE); } +void ScaLBL_StokesModel::Velocity_LB_to_Phys(DoubleArray &Vel_reg){ + for (int k=0;kRegularLayout(Map,&Velocity[0],PhaseField); +// FILE *VELX_FILE; +// sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); +// VELX_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,VELX_FILE); +// fclose(VELX_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); +// FILE *VELY_FILE; +// sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); +// VELY_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,VELY_FILE); +// fclose(VELY_FILE); +// +// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); +// FILE *VELZ_FILE; +// sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); +// VELZ_FILE = fopen(LocalRankFilename,"wb"); +// fwrite(PhaseField.data(),8,N,VELZ_FILE); +// fclose(VELZ_FILE); +// +//} + void ScaLBL_StokesModel::Run(){ double rlx_setA=1.0/tau; double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); diff --git a/models/StokesModel.h b/models/StokesModel.h index f0a4de6a..d40df415 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -30,19 +30,21 @@ public: void Run(); void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); - void getVelocity(); + void getVelocity(int timestep); bool Restart,pBC; int timestep,timestepMax; int BoundaryCondition; double tau,mu; + double rho0; double Fx,Fy,Fz,flux; - double Ex,Ey,Ez; double din,dout; double tolerance; double nu_phys; + double rho_phys; double time_conv; double h;//image resolution + double den_scale;//scale factor for density int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -78,4 +80,5 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); + void Velocity_LB_to_Phys(DoubleArray &Vel_reg); }; diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 156fbc18..1df5c5e1 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -78,14 +78,17 @@ int main(int argc, char **argv) while (timestep < Study.timestepMax){ timestep++; - if (rank==0) printf("timestep=%i; running Poisson solver\n",timestep); + //if (rank==0) printf("timestep=%i; running Poisson solver\n",timestep); PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental + //PoissonSolver.getElectricPotential(timestep); - if (rank==0) printf("timestep=%i; running StokesModel\n",timestep); + //if (rank==0) printf("timestep=%i; running StokesModel\n",timestep); StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity + //StokesModel.getVelocity(timestep); - if (rank==0) printf("timestep=%i; running Ion model\n",timestep); + //if (rank==0) printf("timestep=%i; running Ion model\n",timestep); IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential + //IonModel.getIonConcentration(timestep); timestep++;//AA operations @@ -94,9 +97,10 @@ int main(int argc, char **argv) //-------------------------------------------- } - StokesModel.getVelocity(); - PoissonSolver.getElectricalPotential(); - IonModel.getIonConcentration(); + StokesModel.getVelocity(timestep); + PoissonSolver.getElectricPotential(timestep); + PoissonSolver.getElectricField(timestep); + IonModel.getIonConcentration(timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); From 75e8647051f71acf3a1275fdd406abaeeb8d5a5f Mon Sep 17 00:00:00 2001 From: James McClure Date: Fri, 28 Aug 2020 13:31:43 -0400 Subject: [PATCH 218/270] re-factor electric solvers for unit testing --- models/PoissonSolver.cpp | 129 ++++++++++++++++++++------------------- models/PoissonSolver.h | 6 ++ 2 files changed, 73 insertions(+), 62 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index c3e5c019..8b6ca850 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -418,76 +418,20 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ //************************************************************************/ // *************ODD TIMESTEP*************// timestep++; - ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); - } - //-------------------------// - ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); - + SolveElectricPotentialAAodd(); //compute electric field - ScaLBL_Comm_Regular->SendHalo(Psi); - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, - Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(Psi); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition == 1){ - ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); - ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); - } - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - + SolveElectricField(); //perform collision - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryConditionSolid==1){ - ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); - } - else if (BoundaryConditionSolid==2){ - ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); - } + SolvePoissonAAodd(); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; - ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); - } - //-------------------------// - ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); - + SolveElectricPotentialAAeven(); //compute electric field - ScaLBL_Comm_Regular->SendHalo(Psi); - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, - Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(Psi); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition == 1){ - ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); - ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); - } - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - + SolveElectricField(); //perform collision - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryConditionSolid==1){ - ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); - } - else if (BoundaryConditionSolid==2){ - ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); - } + SolvePoissonAAeven() ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -584,6 +528,67 @@ void ScaLBL_Poisson::getElectricField(int timestep){ fclose(EZ); } +void ScaLBL_Poisson::SolveElectricPotentialAAodd(){ + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); +} +void ScaLBL_Poisson::SolveElectricField(){ + ScaLBL_Comm_Regular->SendHalo(Psi); + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, + Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm_Regular->RecvHalo(Psi); + ScaLBL_DeviceBarrier(); + if (BoundaryCondition == 1){ + ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); + ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); + } + ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + +} +void ScaLBL_Poisson::SolvePoissonAAodd(){ + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } +} + +void ScaLBL_Poisson::SolveElectricPotentialAAeven(){ + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); +} +void ScaLBL_Poisson::SolvePoissonAAeven(){ + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } +} + void ScaLBL_Poisson::ElectricField_LB_to_Phys(DoubleArray &Efield_reg){ for (int k=0;k Date: Fri, 28 Aug 2020 13:37:36 -0400 Subject: [PATCH 219/270] fix poisson solver for unit test --- models/PoissonSolver.cpp | 8 ++++---- models/PoissonSolver.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 8b6ca850..605ad3a2 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -422,7 +422,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ //compute electric field SolveElectricField(); //perform collision - SolvePoissonAAodd(); + SolvePoissonAAodd(ChargeDensity); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// @@ -431,7 +431,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ //compute electric field SolveElectricField(); //perform collision - SolvePoissonAAeven() + SolvePoissonAAeven(ChargeDensity); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -554,7 +554,7 @@ void ScaLBL_Poisson::SolveElectricField(){ ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); } -void ScaLBL_Poisson::SolvePoissonAAodd(){ +void ScaLBL_Poisson::SolvePoissonAAodd(double *ChargeDensity){ ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); if (BoundaryConditionSolid==1){ @@ -578,7 +578,7 @@ void ScaLBL_Poisson::SolveElectricPotentialAAeven(){ //-------------------------// ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); } -void ScaLBL_Poisson::SolvePoissonAAeven(){ +void ScaLBL_Poisson::SolvePoissonAAeven(double *ChargeDensity){ ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); if (BoundaryConditionSolid==1){ diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index bdb90e53..42c47afb 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -31,8 +31,8 @@ public: void SolveElectricPotentialAAodd(); void SolveElectricPotentialAAeven(); void SolveElectricField(); - void SolvePoissonAAodd(); - void SolvePoissonAAeven(); + void SolvePoissonAAodd(double *ChargeDensity); + void SolvePoissonAAeven(double *ChargeDensity); void getElectricPotential(int timestep); void getElectricField(int timestep); From 0a7b1c331b510a80e0194d507749567f8cd52d6c Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 28 Aug 2020 21:53:41 -0400 Subject: [PATCH 220/270] further clean up the code --- common/ScaLBL.h | 6 +- cpu/Poisson.cpp | 148 ++++++++++++++++++++------------------- models/PoissonSolver.cpp | 12 ++-- 3 files changed, 85 insertions(+), 81 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 414653a8..14010169 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -92,10 +92,10 @@ extern "C" void ScaLBL_IonConcentration_Phys(double *Den, double h, int ion_comp // LBM Poisson solver -extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double tau, double epsilon_LB,double gamma, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma, +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double tau, double epsilon_LB,double gamma, int start, int finish, int Np); extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np); @@ -104,8 +104,6 @@ extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *d extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np); - extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, int strideY, int strideZ,int start, int finish, int Np); diff --git a/cpu/Poisson.cpp b/cpu/Poisson.cpp index 8deaac8e..583f9c1d 100644 --- a/cpu/Poisson.cpp +++ b/cpu/Poisson.cpp @@ -88,10 +88,10 @@ extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *d } } -extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma,int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double tau, double epsilon_LB,double gamma,int start, int finish, int Np){ int n; double psi;//electric potential - double Ex,Ey,Ez;//electric field + //double Ex,Ey,Ez;//electric field double rho_e;//local charge density double f0,f1,f2,f3,f4,f5,f6; int nr1,nr2,nr3,nr4,nr5,nr6; @@ -142,40 +142,40 @@ extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *d //ElectricField[n+2*Np] = Ez; // q = 0 - //dist[n] = f0*(1.0-rlx) + 0.3333333333333333*(rlx*psi+rho_e); - dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e); + dist[n] = f0*(1.0-rlx) + 0.3333333333333333*(rlx*psi+rho_e); + //dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e); // q = 1 - //dist[nr2] = f1*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr2] = f1*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 2 - //dist[nr1] = f2*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr1] = f2*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 3 - //dist[nr4] = f3*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr4] = f3*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 4 - //dist[nr3] = f4*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr3] = f4*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 5 - //dist[nr6] = f5*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr6] = f5*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 6 - //dist[nr5] = f6*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[nr5] = f6*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); //........................................................................ } } -extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,double gamma,int start, int finish, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double tau, double epsilon_LB,double gamma,int start, int finish, int Np){ int n; double psi;//electric potential - double Ex,Ey,Ez;//electric field + //double Ex,Ey,Ez;//electric field double rho_e;//local charge density double f0,f1,f2,f3,f4,f5,f6; double rlx=1.0/tau; @@ -209,32 +209,32 @@ extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_c //ElectricField[n+2*Np] = Ez; // q = 0 - //dist[n] = f0*(1.0-rlx) + 0.3333333333333333*(rlx*psi+rho_e); - dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e); + dist[n] = f0*(1.0-rlx) + 0.3333333333333333*(rlx*psi+rho_e); + //dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e); // q = 1 - //dist[1*Np+n] = f1*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[1*Np+n] = f1*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 2 - //dist[2*Np+n] = f2*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[2*Np+n] = f2*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 3 - //dist[3*Np+n] = f3*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[3*Np+n] = f3*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 4 - //dist[4*Np+n] = f4*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[4*Np+n] = f4*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 5 - //dist[5*Np+n] = f5*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[5*Np+n] = f5*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); // q = 6 - //dist[6*Np+n] = f6*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); - dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + dist[6*Np+n] = f6*(1.0-rlx) + 0.1111111111111111*(rlx*psi+rho_e); + //dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); //........................................................................ } } @@ -260,45 +260,20 @@ extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, in //dist[5*Np+n] = 0.125*Psi[n]; //dist[6*Np+n] = 0.125*Psi[n]; - dist[0*Np+n] = 0.25*Psi[ijk]; - dist[1*Np+n] = 0.125*Psi[ijk]; - dist[2*Np+n] = 0.125*Psi[ijk]; - dist[3*Np+n] = 0.125*Psi[ijk]; - dist[4*Np+n] = 0.125*Psi[ijk]; - dist[5*Np+n] = 0.125*Psi[ijk]; - dist[6*Np+n] = 0.125*Psi[ijk]; - } -} - -extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np){ - int n; - // distributions - double f1,f2,f3,f4,f5,f6; - double Ex,Ey,Ez; - double rlx=1.0/tau; - - for (n=0; ngetDatabase( "Domain" ); electric_db = db->getDatabase( "Poisson" ); - //k2_inv = 4.5;//speed of sound for D3Q7 lattice - k2_inv = 4.0;//speed of sound for D3Q7 lattice + k2_inv = 4.5;//speed of sound for D3Q7 lattice + //k2_inv = 4.0;//speed of sound for D3Q7 lattice gamma = 1.0;//time step of LB-Poisson equation tau = 0.5+k2_inv*gamma; timestepMax = 100000; @@ -443,8 +443,8 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); //perform collision - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); if (BoundaryConditionSolid==1){ ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); } @@ -480,8 +480,8 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); //perform collision - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); if (BoundaryConditionSolid==1){ ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); } From 0b480294b5d60e636ed479b75305532537bbc582 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 29 Aug 2020 12:22:20 -0400 Subject: [PATCH 221/270] fix spill of integer index in AggregateLabels --- common/Domain.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index eadc4592..7a1a6230 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -695,7 +695,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int full_ny = npy*(ny-2); int full_nz = npz*(nz-2); int local_size = (nx-2)*(ny-2)*(nz-2); - long int full_size = long(full_nx)*long(full_ny)*long(full_nz); + unsigned long int full_size = long(full_nx)*long(full_ny)*long(full_nz); signed char *LocalID; LocalID = new signed char [local_size]; @@ -725,7 +725,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int y = j-1; int z = k-1; int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1; - int n_full = z*full_nx*full_ny + y*full_nx + x; + unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x; FullID[n_full] = LocalID[n_local]; } } @@ -745,7 +745,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int y = j-1 + ipy*(ny-2); int z = k-1 + ipz*(nz-2); int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1; - int n_full = z*full_nx*full_ny + y*full_nx + x; + unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x; FullID[n_full] = LocalID[n_local]; } } From ae9e7a04080ac21b400de382cfa789c9c2d1fafc Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 29 Aug 2020 13:11:56 -0400 Subject: [PATCH 222/270] fix integer index bug in AggregateLabels in morphdrain --- common/Domain.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index eadc4592..7a1a6230 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -695,7 +695,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int full_ny = npy*(ny-2); int full_nz = npz*(nz-2); int local_size = (nx-2)*(ny-2)*(nz-2); - long int full_size = long(full_nx)*long(full_ny)*long(full_nz); + unsigned long int full_size = long(full_nx)*long(full_ny)*long(full_nz); signed char *LocalID; LocalID = new signed char [local_size]; @@ -725,7 +725,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int y = j-1; int z = k-1; int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1; - int n_full = z*full_nx*full_ny + y*full_nx + x; + unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x; FullID[n_full] = LocalID[n_local]; } } @@ -745,7 +745,7 @@ void Domain::AggregateLabels( const std::string& filename ){ int y = j-1 + ipy*(ny-2); int z = k-1 + ipz*(nz-2); int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1; - int n_full = z*full_nx*full_ny + y*full_nx + x; + unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x; FullID[n_full] = LocalID[n_local]; } } From 20c8cc9c3b307eb756f7ea8c0cfa9953dded19a5 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 2 Sep 2020 11:37:23 -0400 Subject: [PATCH 223/270] update PoissonSolver and fix numerous bugs --- common/Domain.cpp | 64 +++++------ common/ScaLBL.cpp | 4 +- common/ScaLBL.h | 4 +- cpu/D3Q7BC.cpp | 2 +- cpu/Poisson.cpp | 129 ++++++++-------------- models/PoissonSolver.cpp | 209 +++++++++++++++++++----------------- models/PoissonSolver.h | 18 ++-- tests/CMakeLists.txt | 1 + tests/TestPoissonSolver.cpp | 102 ++++++++++++++++++ 9 files changed, 305 insertions(+), 228 deletions(-) create mode 100644 tests/TestPoissonSolver.cpp diff --git a/common/Domain.cpp b/common/Domain.cpp index eadc4592..d75ec4a5 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -626,38 +626,38 @@ void Domain::Decomp( const std::string& Filename ) if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && BoundaryCondition !=5 && kproc() == 0){ - if (inlet_layers_z < 4){ - inlet_layers_z=4; - if(RANK==0){ - printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase); - } - } - for (int k=0; k 0 && BoundaryCondition !=5 && kproc() == nprocz-1){ - if (outlet_layers_z < 4){ - outlet_layers_z=4; - if(RANK==nprocs-1){ - printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase); - } - } - for (int k=Nz-outlet_layers_z; k 0 && BoundaryCondition !=5 && kproc() == 0){ +// if (inlet_layers_z < 4){ +// inlet_layers_z=4; +// if(RANK==0){ +// printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase); +// } +// } +// for (int k=0; k 0 && BoundaryCondition !=5 && kproc() == nprocz-1){ +// if (outlet_layers_z < 4){ +// outlet_layers_z=4; +// if(RANK==nprocs-1){ +// printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase); +// } +// } +// for (int k=Nz-outlet_layers_z; k0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18 //............Compute the Color Gradient................................... - nx = -1.f/18.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - ny = -1.f/18.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - nz = -1.f/18.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + //nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); + //ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); + //nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); + nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3 + ny = 1.f/6.f*(m3-m4); + nz = 1.f/6.f*(m5-m6); ElectricField[n] = nx; ElectricField[Np+n] = ny; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index d400df05..89a1c42a 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -6,8 +6,9 @@ #include "common/ReadMicroCT.h" ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM): -rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),gamma(0),tolerance(0),h(0), +rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0), epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), +chargeDen_dummy(0), nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -22,10 +23,8 @@ void ScaLBL_Poisson::ReadParams(string filename){ domain_db = db->getDatabase( "Domain" ); electric_db = db->getDatabase( "Poisson" ); - k2_inv = 4.5;//speed of sound for D3Q7 lattice - //k2_inv = 4.0;//speed of sound for D3Q7 lattice - gamma = 1.0;//time step of LB-Poisson equation - tau = 0.5+k2_inv*gamma; + k2_inv = 4.0;//speed of sound for D3Q7 lattice + tau = 0.5+k2_inv; timestepMax = 100000; tolerance = 1.0e-6;//stopping criterion for obtaining steady-state electricla potential h = 1.0;//resolution; unit: um/lu @@ -36,6 +35,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ analysis_interval = 1000; Vin = 1.0; //Boundary-z (inlet) electric potential Vout = 1.0; //Boundary-Z (outlet) electric potential + chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3] // LB-Poisson Model parameters if (electric_db->keyExists( "timestepMax" )){ @@ -47,12 +47,12 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "tolerance" )){ tolerance = electric_db->getScalar( "tolerance" ); } - if (electric_db->keyExists( "gamma" )){ - gamma = electric_db->getScalar( "gamma" ); - } if (electric_db->keyExists( "epsilonR" )){ epsilonR = electric_db->getScalar( "epsilonR" ); } + if (electric_db->keyExists( "DummyChargeDen" )){ + chargeDen_dummy = electric_db->getScalar( "DummyChargeDen" ); + } // Read solid boundary condition specific to Poisson equation BoundaryConditionSolid = 1; @@ -76,7 +76,6 @@ void ScaLBL_Poisson::ReadParams(string filename){ //Re-calcualte model parameters if user updates input epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)] epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity - tau = 0.5+k2_inv*gamma; if (rank==0) printf("***********************************************************************************\n"); if (rank==0) printf("LB-Poisson Solver: steady-state MaxTimeStep = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance); @@ -213,7 +212,6 @@ void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid) //NOTE need to convert the user input phys unit to LB unit if (BoundaryConditionSolid==2){ //for BCS=1, i.e. Dirichlet-type, no need for unit conversion - //TODO maybe there is a factor of gamm missing here ? AFFINITY = AFFINITY*(h*h*1.0e-12)/epsilon_LB; } label_count[idx] += 1.0; @@ -284,11 +282,10 @@ void ScaLBL_Poisson::Create(){ //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &dvcID, sizeof(signed char)*Nx*Ny*Nz); + //ScaLBL_AllocateDeviceMemory((void **) &dvcID, sizeof(signed char)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &fq, 7*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &ElectricField, 3*sizeof(double)*Np); - //ScaLBL_AllocateDeviceMemory((void **) &PoissonSolid, sizeof(double)*Nx*Ny*Nz); //........................................................................... // Update GPU data structures @@ -329,26 +326,13 @@ void ScaLBL_Poisson::Create(){ MPI_Barrier(comm); delete [] neighborList; // copy node ID - ScaLBL_CopyToDevice(dvcID, Mask->id, sizeof(signed char)*Nx*Ny*Nz); - ScaLBL_DeviceBarrier(); + //ScaLBL_CopyToDevice(dvcID, Mask->id, sizeof(signed char)*Nx*Ny*Nz); + //ScaLBL_DeviceBarrier(); //Initialize solid boundary for electric potential ScaLBL_Comm->SetupBounceBackList(Map, Mask->id, Np); MPI_Barrier(comm); - - //double *PoissonSolid_host; - //PoissonSolid_host = new double[Nx*Ny*Nz]; - //AssignSolidBoundary(PoissonSolid_host); - //ScaLBL_CopyToDevice(PoissonSolid, PoissonSolid_host, Nx*Ny*Nz*sizeof(double)); - //ScaLBL_DeviceBarrier(); - //delete [] PoissonSolid_host; } -// Method 1 -// Psi - size N -// ID_dvc - size N -// Method 2 -// Psi - size Np -// PoissonSolid size N void ScaLBL_Poisson::Potential_Init(double *psi_init){ @@ -402,6 +386,16 @@ void ScaLBL_Poisson::Initialize(){ ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); delete [] psi_host; + + //extra treatment for halo layer + if (BoundaryCondition==1){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); + } + } } void ScaLBL_Poisson::Run(double *ChargeDensity){ @@ -418,20 +412,17 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ //************************************************************************/ // *************ODD TIMESTEP*************// timestep++; - SolveElectricPotentialAAodd(); - //compute electric field - SolveElectricField(); - //perform collision - SolvePoissonAAodd(ChargeDensity); + + SolveElectricPotentialAAodd();//update electric potential + //SolveElectricField(); //deprecated - compute electric field + SolvePoissonAAodd(ChargeDensity);//perform collision ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; - SolveElectricPotentialAAeven(); - //compute electric field - SolveElectricField(); - //perform collision - SolvePoissonAAeven(ChargeDensity); + SolveElectricPotentialAAeven();//update electric potential + //SolveElectricField();//deprecated - compute electric field + SolvePoissonAAeven(ChargeDensity);//perform collision ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ @@ -484,6 +475,75 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ } +void ScaLBL_Poisson::SolveElectricPotentialAAodd(){ + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); +} + +void ScaLBL_Poisson::SolveElectricPotentialAAeven(){ + ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); + ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); + } + //-------------------------// + ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); +} + +void ScaLBL_Poisson::SolvePoissonAAodd(double *ChargeDensity){ + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } +} + +void ScaLBL_Poisson::SolvePoissonAAeven(double *ChargeDensity){ + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, 0, ScaLBL_Comm->LastExterior(), Np); + if (BoundaryConditionSolid==1){ + ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); + } + else if (BoundaryConditionSolid==2){ + ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); + } +} + +void ScaLBL_Poisson::DummyChargeDensity(){ + double *ChargeDensity_host; + ChargeDensity_host = new double[Np]; + + for (int k=0; kSendD3Q7AA(fq, 0); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); - } - //-------------------------// - ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); -} -void ScaLBL_Poisson::SolveElectricField(){ - ScaLBL_Comm_Regular->SendHalo(Psi); - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, - Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm_Regular->RecvHalo(Psi); - ScaLBL_DeviceBarrier(); - if (BoundaryCondition == 1){ - ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); - ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); - } - ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - -} -void ScaLBL_Poisson::SolvePoissonAAodd(double *ChargeDensity){ - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryConditionSolid==1){ - ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); - } - else if (BoundaryConditionSolid==2){ - ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); - } -} - -void ScaLBL_Poisson::SolveElectricPotentialAAeven(){ - ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL - ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep); - ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep); - } - //-------------------------// - ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np); -} -void ScaLBL_Poisson::SolvePoissonAAeven(double *ChargeDensity){ - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, tau, epsilon_LB, gamma, 0, ScaLBL_Comm->LastExterior(), Np); - if (BoundaryConditionSolid==1){ - ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi); - } - else if (BoundaryConditionSolid==2){ - ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi); - } -} - - void ScaLBL_Poisson::ElectricField_LB_to_Phys(DoubleArray &Efield_reg){ for (int k=0;kSendHalo(Psi); +// ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, +// Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); +// ScaLBL_Comm_Regular->RecvHalo(Psi); +// ScaLBL_DeviceBarrier(); +// if (BoundaryCondition == 1){ +// ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin); +// ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout); +// } +// ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); +// +//} //void ScaLBL_Poisson::getElectricPotential(){ // diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 42c47afb..921963ed 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -28,13 +28,9 @@ public: void Create(); void Initialize(); void Run(double *ChargeDensity); - void SolveElectricPotentialAAodd(); - void SolveElectricPotentialAAeven(); - void SolveElectricField(); - void SolvePoissonAAodd(double *ChargeDensity); - void SolvePoissonAAeven(double *ChargeDensity); void getElectricPotential(int timestep); void getElectricField(int timestep); + void DummyChargeDensity();//for debugging //bool Restart,pBC; int timestep,timestepMax; @@ -43,9 +39,10 @@ public: int BoundaryConditionSolid; double tau; double tolerance; - double k2_inv,gamma; + double k2_inv; double epsilon0,epsilon0_LB,epsilonR,epsilon_LB; double Vin, Vout; + double chargeDen_dummy;//for debugging int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -66,11 +63,11 @@ public: DoubleArray Psi_host; int *NeighborList; int *dvcMap; - signed char *dvcID; + //signed char *dvcID; double *fq; double *Psi; double *ElectricField; - //double *PoissonSolid; + double *ChargeDensityDummy;// for debugging private: MPI_Comm comm; @@ -85,5 +82,10 @@ private: void AssignSolidBoundary(double *poisson_solid); void Potential_Init(double *psi_init); void ElectricField_LB_to_Phys(DoubleArray &Efield_reg); + void SolveElectricPotentialAAodd(); + void SolveElectricPotentialAAeven(); + //void SolveElectricField(); + void SolvePoissonAAodd(double *ChargeDensity); + void SolvePoissonAAeven(double *ChargeDensity); }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1a8bfac0..43167b3f 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,6 +47,7 @@ ADD_LBPM_TEST( TestTorusEvolve ) ADD_LBPM_TEST( TestTopo3D ) ADD_LBPM_TEST( TestFluxBC ) ADD_LBPM_TEST( TestMap ) +ADD_LBPM_TEST( TestPoissonSolver ) #ADD_LBPM_TEST( TestMRT ) #ADD_LBPM_TEST( TestColorGrad ) #ADD_LBPM_TEST( TestColorGradDFH ) diff --git a/tests/TestPoissonSolver.cpp b/tests/TestPoissonSolver.cpp new file mode 100644 index 00000000..309a03c7 --- /dev/null +++ b/tests/TestPoissonSolver.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "models/PoissonSolver.h" + +using namespace std; + +//******************************************************** +// Test lattice-Boltzmann solver of Poisson equation +//******************************************************** + +int main(int argc, char **argv) +{ + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_support Date: Fri, 11 Sep 2020 22:56:00 -0400 Subject: [PATCH 224/270] CPU only;finish preliminary work on testing Poisson and Ion models and their coupling --- common/ScaLBL.cpp | 22 + common/ScaLBL.h | 16 +- cpu/D3Q7BC.cpp | 86 ++ cpu/Ion.cpp | 56 +- gpu/D3Q7BC.cu | 1203 +++++++++++++++++++ models/IonModel.cpp | 557 +++++++-- models/IonModel.h | 22 +- models/MultiPhysController.cpp | 119 +- models/MultiPhysController.h | 12 +- models/PoissonSolver.cpp | 2 +- models/StokesModel.cpp | 234 +++- models/StokesModel.h | 3 + tests/CMakeLists.txt | 3 + tests/TestIonModel.cpp | 89 ++ tests/TestNernstPlanck.cpp | 101 ++ tests/TestPNP_Stokes.cpp | 123 ++ tests/TestPoissonSolver.cpp | 29 - tests/lbpm_electrokinetic_dfh_simulator.cpp | 2 +- 18 files changed, 2453 insertions(+), 226 deletions(-) create mode 100644 gpu/D3Q7BC.cu create mode 100644 tests/TestIonModel.cpp create mode 100644 tests/TestNernstPlanck.cpp create mode 100644 tests/TestPNP_Stokes.cpp diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index b621a517..8db59597 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2086,3 +2086,25 @@ void ScaLBL_Communicator::Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout){ ScaLBL_Poisson_D3Q7_BC_Z(dvcSendList_Z, Map, Psi, Vout, sendCount_Z); } } + +void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time){ + if (kproc == 0) { + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(dvcSendList_z, fq, Cin, sendCount_z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(neighborList, dvcSendList_z, fq, Cin, sendCount_z, N); + } + } +} + +void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time){ + if (kproc == nprocz-1){ + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(dvcSendList_Z, fq, Cout, sendCount_Z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(neighborList, dvcSendList_Z, fq, Cout, sendCount_Z, N); + } + } +} diff --git a/common/ScaLBL.h b/common/ScaLBL.h index ca5610d3..a4a5fad3 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -79,17 +79,15 @@ extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, i extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, - double Di, double zi, double rlx, double Vt, int start, int finish, int Np); + double Di, int zi, double rlx, double Vt, int start, int finish, int Np); extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, - double Di, double zi, double rlx, double Vt, int start, int finish, int Np); + double Di, int zi, double rlx, double Vt, int start, int finish, int Np); extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np); extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np); -extern "C" void ScaLBL_IonConcentration_Phys(double *Den, double h, int ion_component, int start, int finish, int Np); - // LBM Poisson solver extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB, @@ -206,6 +204,14 @@ extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, doubl extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count); +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np); + class ScaLBL_Communicator{ public: //...................................................................................... @@ -269,6 +275,8 @@ public: void D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time); void Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin); void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout); + void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time); + void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time); // Debugging and unit testing functions void PrintD3Q19(); diff --git a/cpu/D3Q7BC.cpp b/cpu/D3Q7BC.cpp index ee47fed4..161e6a5c 100644 --- a/cpu/D3Q7BC.cpp +++ b/cpu/D3Q7BC.cpp @@ -137,3 +137,89 @@ extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, doubl Psi[nm] = Vout; } } + +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){ + for (int idx=0; idx +#include +#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + + +__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count) +{ + + int idx; + int iq,ib; + double value_b,value_q; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + iq = BounceBackDist_list[idx]; + ib = BounceBackSolid_list[idx]; + value_b = BoundaryValue[ib];//get boundary value from a solid site + value_q = dist[iq]; + dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice + } +} + +__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count) +{ + + int idx; + int iq,ib; + double value_b,value_q; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + iq = BounceBackDist_list[idx]; + ib = BounceBackSolid_list[idx]; + value_b = BoundaryValue[ib];//get boundary value from a solid site + value_q = dist[iq]; + dist[iq] = value_q + value_b; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np) +{ + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f6 = dist[5*Np+n]; + //................................................... + f5 = Vin - (f0+f1+f2+f3+f4+f6); + dist[6*Np+n] = f5; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np) +{ + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f5 = dist[6*Np+n]; + //................................................... + f6 = Vout - (f0+f1+f2+f3+f4+f5); + dist[5*Np+n] = f6; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np) +{ + int idx, n; + int nread,nr5; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+5*Np]; + f6 = dist[nread]; + + // Unknown distributions + nr5 = d_neighborList[n+4*Np]; + f5 = Vin - (f0+f1+f2+f3+f4+f6); + dist[nr5] = f5; + } +} + +__global__ void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np) +{ + int idx, n; + int nread,nr6; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+4*Np]; + f5 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + // unknown distributions + nr6 = d_neighborList[n+5*Np]; + f6 = Vout - (f0+f1+f2+f3+f4+f5); + dist[nr6] = f6; + } +} + +__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count) +{ + int idx,n,nm; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + nm = Map[n]; + Psi[nm] = Vin; + } +} + + +__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count) +{ + int idx,n,nm; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + nm = Map[n]; + Psi[nm] = Vout; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np) +{ + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f6 = dist[5*Np+n]; + //................................................... + f5 = Cin - (f0+f1+f2+f3+f4+f6); + dist[6*Np+n] = f5; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np) +{ + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f5 = dist[6*Np+n]; + //................................................... + f6 = Cout - (f0+f1+f2+f3+f4+f5); + dist[5*Np+n] = f6; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np) +{ + int idx, n; + int nread,nr5; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+5*Np]; + f6 = dist[nread]; + + // Unknown distributions + nr5 = d_neighborList[n+4*Np]; + f5 = Cin - (f0+f1+f2+f3+f4+f6); + dist[nr5] = f5; + } +} + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np) +{ + int idx, n; + int nread,nr6; + double f0,f1,f2,f3,f4,f5,f6; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+4*Np]; + f5 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + // unknown distributions + nr6 = d_neighborList[n+5*Np]; + f6 = Cout - (f0+f1+f2+f3+f4+f5); + dist[nr6] = f6; + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np) +{ + int idx, n; + int nread; + int nr5,nr11,nr14,nr15,nr18; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double ux,uy,uz,Cyz,Cxz; + ux = uy = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + if (idx < count){ + + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+6*Np]; + f7 = dist[nread]; + + nread = d_neighborList[n+8*Np]; + f9 = dist[nread]; + + nread = d_neighborList[n+12*Np]; + f13 = dist[nread]; + + nread = d_neighborList[n+16*Np]; + f17 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+5*Np]; + f6 = dist[nread]; + + nread = d_neighborList[n+7*Np]; + f8 = dist[nread]; + + nread = d_neighborList[n+9*Np]; + f10 = dist[nread]; + + nread = d_neighborList[n+11*Np]; + f12 = dist[nread]; + + nread = d_neighborList[n+15*Np]; + f16 = dist[nread]; + + // Unknown distributions + nr5 = d_neighborList[n+4*Np]; + nr11 = d_neighborList[n+10*Np]; + nr15 = d_neighborList[n+14*Np]; + nr14 = d_neighborList[n+13*Np]; + nr18 = d_neighborList[n+17*Np]; + + //................................................... + //........Determine the inlet flow velocity......... + //ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14); + //uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18); + uz = din - (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); + + Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux; + Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy; + + f5 = f6 + 0.33333333333333338*uz; + f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz; + f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz; + f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz; + f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz; + //........Store in "opposite" memory location.......... + dist[nr5] = f5; + dist[nr11] = f11; + dist[nr14] = f14; + dist[nr15] = f15; + dist[nr18] = f18; + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, double *dist, double dout, int count, int Np) +{ + int idx,n,nread; + int nr6,nr12,nr13,nr16,nr17; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double ux,uy,uz,Cyz,Cxz; + ux = uy = 0.0; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + // Loop over the boundary - threadblocks delineated by start...finish + if ( idx < count ){ + + n = list[idx]; + //........................................................................ + // Read distributions + //........................................................................ + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+4*Np]; + f5 = dist[nread]; + + nread = d_neighborList[n+6*Np]; + f7 = dist[nread]; + + nread = d_neighborList[n+8*Np]; + f9 = dist[nread]; + + nread = d_neighborList[n+10*Np]; + f11 = dist[nread]; + + nread = d_neighborList[n+14*Np]; + f15 = dist[nread]; + + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+7*Np]; + f8 = dist[nread]; + + nread = d_neighborList[n+9*Np]; + f10 = dist[nread]; + + nread = d_neighborList[n+13*Np]; + f14 = dist[nread]; + + nread = d_neighborList[n+17*Np]; + f18 = dist[nread]; + + // unknown distributions + nr6 = d_neighborList[n+5*Np]; + nr12 = d_neighborList[n+11*Np]; + nr16 = d_neighborList[n+15*Np]; + nr17 = d_neighborList[n+16*Np]; + nr13 = d_neighborList[n+12*Np]; + + + //........Determine the outlet flow velocity......... + //ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; + //uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; + uz = -dout + (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18)); + + Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux; + Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy; + + f6 = f5 - 0.33333333333333338*uz; + f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz; + f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz; + f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz; + f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz; + + //........Store in "opposite" memory location.......... + dist[nr6] = f6; + dist[nr12] = f12; + dist[nr13] = f13; + dist[nr16] = f16; + dist[nr17] = f17; + //................................................... + } +} + + +__global__ void dvc_ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double Area, + double *dvcsum, int count, int Np) +{ + int idx, n; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double factor = 1.f/(Area); + double sum = 0.f; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + if (idx < count){ + + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f6 = dist[5*Np+n]; + f7 = dist[8*Np+n]; + f8 = dist[7*Np+n]; + f9 = dist[10*Np+n]; + f10 = dist[9*Np+n]; + f12 = dist[11*Np+n]; + f13 = dist[14*Np+n]; + f16 = dist[15*Np+n]; + f17 = dist[18*Np+n]; + sum = factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); + } + + //sum = blockReduceSum(sum); + //if (threadIdx.x==0) + // atomicAdd(dvcsum, sum); + + extern __shared__ double temp[]; + thread_group g = this_thread_block(); + double block_sum = reduce_sum(g, temp, sum); + + if (g.thread_rank() == 0) atomicAdd(dvcsum, block_sum); +} + + +__global__ void dvc_ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, double *dist, double flux, + double Area, double *dvcsum, int count, int Np) +{ + int idx, n; + int nread; + + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double factor = 1.f/(Area); + double sum = 0.f; + + idx = blockIdx.x*blockDim.x + threadIdx.x; + + if (idx < count){ + + n = list[idx]; + + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+6*Np]; + f7 = dist[nread]; + + nread = d_neighborList[n+8*Np]; + f9 = dist[nread]; + + nread = d_neighborList[n+12*Np]; + f13 = dist[nread]; + + nread = d_neighborList[n+16*Np]; + f17 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+5*Np]; + f6 = dist[nread]; + + nread = d_neighborList[n+7*Np]; + f8 = dist[nread]; + + nread = d_neighborList[n+9*Np]; + f10 = dist[nread]; + + nread = d_neighborList[n+11*Np]; + f12 = dist[nread]; + + nread = d_neighborList[n+15*Np]; + f16 = dist[nread]; + + sum = factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); + + } + + //sum = blockReduceSum(sum); + //if (threadIdx.x==0) + // atomicAdd(dvcsum, sum); + + extern __shared__ double temp[]; + thread_group g = this_thread_block(); + double block_sum = reduce_sum(g, temp, sum); + + if (g.thread_rank() == 0) atomicAdd(dvcsum, block_sum); +} + + +__global__ void dvc_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz, + int Nx, int Ny, int Nz) +{ + int n,N; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double din; + + N = Nx*Ny*Nz; + n = Nx*Ny + blockIdx.x*blockDim.x + threadIdx.x; + + if (n < 2*Nx*Ny){ + //........................................................................ + // Read distributions from "opposite" memory convention + //........................................................................ + //........................................................................ + f1 = distodd[n]; + f3 = distodd[N+n]; + f5 = distodd[2*N+n]; + f7 = distodd[3*N+n]; + f9 = distodd[4*N+n]; + f11 = distodd[5*N+n]; + f13 = distodd[6*N+n]; + f15 = distodd[7*N+n]; + f17 = distodd[8*N+n]; + //........................................................................ + f0 = disteven[n]; + f2 = disteven[N+n]; + f4 = disteven[2*N+n]; + f6 = disteven[3*N+n]; + f8 = disteven[4*N+n]; + f10 = disteven[5*N+n]; + f12 = disteven[6*N+n]; + f14 = disteven[7*N+n]; + f16 = disteven[8*N+n]; + f18 = disteven[9*N+n]; + //................................................... + + // Determine the outlet flow velocity + // uz = 1.0 - (f0+f4+f3+f2+f1+f8+f7+f9+f10 + + // 2*(f5+f15+f18+f11+f14))/din; + din = (f0+f4+f3+f2+f1+f8+f7+f9+f10+2*(f5+f15+f18+f11+f14))/(1.0-uz); + // Set the unknown distributions: + f6 = f5 + 0.3333333333333333*din*uz; + f16 = f15 + 0.1666666666666667*din*uz; + f17 = f16 + f4 - f3-f15+f18+f8-f7 +f9-f10; + f12= (din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f2+f1-f14+f11-f8+f7+f9-f10)*0.5; + f13= din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f12; + + //........Store in "opposite" memory location.......... + disteven[3*N+n] = f6; + disteven[6*N+n] = f12; + distodd[6*N+n] = f13; + disteven[8*N+n] = f16; + distodd[8*N+n] = f17; + //................................................... + } +} + +__global__ void dvc_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, + int Nx, int Ny, int Nz, int outlet){ + int n,N; + // distributions + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double dout; + + N = Nx*Ny*Nz; + n = outlet + blockIdx.x*blockDim.x + threadIdx.x; + + // Loop over the boundary - threadblocks delineated by start...finish + if ( n 0 ){ + f_even[n] = 0 + 0.01*0; + f_odd[n] = 0+ 0.01*1; //double(100*n)+1.f; + f_even[N+n] = 1+ 0.01*2; //double(100*n)+2.f; + f_odd[N+n] = 1+ 0.01*3; //double(100*n)+3.f; + f_even[2*N+n] = 2+ 0.01*4; //double(100*n)+4.f; + f_odd[2*N+n] = 2+ 0.01*5; //double(100*n)+5.f; + f_even[3*N+n] = 3+ 0.01*6; //double(100*n)+6.f; + f_odd[3*N+n] = 3+ 0.01*7; //double(100*n)+7.f; + f_even[4*N+n] = 4+ 0.01*8; //double(100*n)+8.f; + f_odd[4*N+n] = 4+ 0.01*9; //double(100*n)+9.f; + f_even[5*N+n] = 5+ 0.01*10; //double(100*n)+10.f; + f_odd[5*N+n] = 5+ 0.01*11; //double(100*n)+11.f; + f_even[6*N+n] = 6+ 0.01*12; //double(100*n)+12.f; + f_odd[6*N+n] = 6+ 0.01*13; //double(100*n)+13.f; + f_even[7*N+n] = 7+ 0.01*14; //double(100*n)+14.f; + f_odd[7*N+n] = 7+ 0.01*15; //double(100*n)+15.f; + f_even[8*N+n] = 8+ 0.01*16; //double(100*n)+16.f; + f_odd[8*N+n] = 8+ 0.01*17; //double(100*n)+17.f; + f_even[9*N+n] = 9+ 0.01*18; //double(100*n)+18.f; + } + else{ + for(int q=0; q<9; q++){ + f_even[q*N+n] = -1.0; + f_odd[q*N+n] = -1.0; + } + f_even[9*N+n] = -1.0; + } + } + } +} + + +//************************************************************************* + +//extern "C" void ScaLBL_D3Q19_MapRecv(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, +// int *d3q19_recvlist, int Nx, int Ny, int Nz){ +// int GRID = count / 512 + 1; +// dvc_ScaLBL_D3Q19_Unpack <<>>(q, Cqx, Cqy, Cqz, list, start, count, d3q19_recvlist, Nx, Ny, Nz); +//} + +extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Pack <<>>(q, list, start, count, sendbuf, dist, N); +} + +extern "C" void ScaLBL_D3Q19_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Unpack <<>>(q, list, start, count, recvbuf, dist, N); +} +//************************************************************************* + +extern "C" void ScaLBL_D3Q19_AA_Init(double *f_even, double *f_odd, int Np){ + dvc_ScaLBL_D3Q19_AA_Init<<>>(f_even, f_odd, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){ + dvc_ScaLBL_D3Q19_Init<<>>(dist, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ + dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ + dvc_ScaLBL_D3Q19_Swap<<>>(ID, disteven, distodd, Nx, Ny, Nz); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Swap: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np) +{ + + const int Q = 9; + // cudaStream_t streams[Q]; + // Launch the swap operation as different kernels + for (int q=0; q>>(neighborList, disteven, distodd, Np, q); + } + // cpu should wait for all kernels to finish (to avoid launch of dependent kernels) + //cudaDeviceSynchronize(); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Swap: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *d_dist, int Np) { + cudaFuncSetCacheConfig(dvc_ScaLBL_AAeven_Compact, cudaFuncCachePreferL1); + dvc_ScaLBL_AAeven_Compact<<>>(ID, d_dist, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *d_neighborList, double *d_dist, int Np) { + cudaFuncSetCacheConfig(dvc_ScaLBL_AAodd_Compact, cudaFuncCachePreferL1); + dvc_ScaLBL_AAodd_Compact<<>>(ID,d_neighborList, d_dist,Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np){ + + dvc_ScaLBL_D3Q19_Momentum<<>>(dist, vel, Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Velocity: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Pressure(double *fq, double *Pressure, int Np){ + dvc_ScaLBL_D3Q19_Pressure<<< NBLOCKS,NTHREADS >>>(fq, Pressure, Np); +} + +extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,int Nx, int Ny, int Nz){ + int GRID = Nx*Ny / 512 + 1; + dvc_D3Q19_Velocity_BC_z<<>>(disteven,distodd, uz, Nx, Ny, Nz); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_z: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, int Nx, int Ny, int Nz, int outlet){ + int GRID = Nx*Ny / 512 + 1; + dvc_D3Q19_Velocity_BC_Z<<>>(disteven, distodd, uz, Nx, Ny, Nz, outlet); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_Z: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, double flux,int Nx, int Ny, int Nz){ + + int GRID = Nx*Ny / 512 + 1; + + // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 + if (Nx*Ny > 512*512){ + printf("WARNING (ScaLBL_D3Q19_Flux_BC_z): CUDA reduction operation may fail if Nx*Ny > 512*512"); + } + + // Allocate memory to store the sums + double din; + double sum[1]; + double *dvcsum; + int sharedBytes = NTHREADS*sizeof(double); + cudaMalloc((void **)&dvcsum,sizeof(double)*Nx*Ny); + cudaMemset(dvcsum,0,sizeof(double)*Nx*Ny); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); + } + + // compute the local flux and store the result + dvc_D3Q19_Flux_BC_z<<>>(disteven, distodd, flux, dvcsum, Nx, Ny, Nz); + + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 1): %s \n",cudaGetErrorString(err)); + } + + // Now read the total flux + cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); + din=sum[0]; + + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 2): %s \n",cudaGetErrorString(err)); + } + + // free the memory needed for reduction + cudaFree(dvcsum); + + return din; +} + + +extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_z<<>>(list, dist, din, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z<<>>(list, dist, dout, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z<<>>(neighborList, list, dist, din, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int N){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_Z<<>>(neighborList, list, dist, dout, count, N); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} +//******************************************************************************* +//******************************************************************************* +//******************************************************************************* + + +//******************************************************************************* +extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area, + int count, int N){ + + int GRID = count / 512 + 1; + + // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 + if (count > 512*512){ + printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if count > 512*512"); + } + + // Allocate memory to store the sums + double din; + double sum[1]; + double *dvcsum; + cudaMalloc((void **)&dvcsum,sizeof(double)*count); + cudaMemset(dvcsum,0,sizeof(double)*count); + int sharedBytes = 512*sizeof(double); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); + } + + // compute the local flux and store the result + dvc_ScaLBL_D3Q19_AAeven_Flux_BC_z<<>>(list, dist, flux, area, dvcsum, count, N); + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } + + // Now read the total flux + cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); + din=sum[0]; + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err)); + } + + // free the memory needed for reduction + cudaFree(dvcsum); + + return din; +} + +extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, double *dist, double flux, + double area, int count, int N){ + + int GRID = count / 512 + 1; + + // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 + if (count > 512*512){ + printf("WARNING (ScaLBL_D3Q19_AAodd_Flux_BC_z): CUDA reduction operation may fail if count > 512*512"); + } + + // Allocate memory to store the sums + double din; + double sum[1]; + double *dvcsum; + cudaMalloc((void **)&dvcsum,sizeof(double)*count); + cudaMemset(dvcsum,0,sizeof(double)*count); + int sharedBytes = 512*sizeof(double); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); + } + + // compute the local flux and store the result + dvc_ScaLBL_D3Q19_AAodd_Flux_BC_z<<>>(neighborList, list, dist, flux, area, dvcsum, count, N); + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } + // Now read the total flux + cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); + din=sum[0]; + err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err)); + } + + // free the memory needed for reduction + cudaFree(dvcsum); + + return din; +} + +extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, double flux, int Nx, int Ny, int Nz, int outlet){ + + int GRID = Nx*Ny / 512 + 1; + + // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 + if (Nx*Ny > 512*512){ + printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if Nx*Ny > 512*512"); + } + + // Allocate memory to store the sums + double dout; + double sum[1]; + double *dvcsum; + cudaMalloc((void **)&dvcsum,sizeof(double)*Nx*Ny); + cudaMemset(dvcsum,0,sizeof(double)*Nx*Ny); + + // compute the local flux and store the result + dvc_D3Q19_Flux_BC_Z<<>>(disteven, distodd, flux, dvcsum, Nx, Ny, Nz, outlet); + + // Now read the total flux + cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); + + // free the memory needed for reduction + + dout = sum[0]; + + cudaFree(dvcsum); + + return dout; + +} + +extern "C" double deviceReduce(double *in, double* out, int N) { + int threads = 512; + int blocks = min((N + threads - 1) / threads, 1024); + + double sum = 0.f; + deviceReduceKernel<<>>(in, out, N); + deviceReduceKernel<<<1, 1024>>>(out, out, blocks); + return sum; +} + +extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Reflection_BC_z<<>>(list, dist, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q19_Reflection_BC_Z<<>>(list, dist, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, + double Fy, double Fz){ + + dvc_ScaLBL_AAeven_MRT<<>>(dist,start,finish,Np,rlx_setA,rlx_setB,Fx,Fy,Fz); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_MRT: %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborlist, double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, + double Fy, double Fz){ + + dvc_ScaLBL_AAodd_MRT<<>>(neighborlist,dist,start,finish,Np,rlx_setA,rlx_setB,Fx,Fy,Fz); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_MRT: %s \n",cudaGetErrorString(err)); + } +} + diff --git a/models/IonModel.cpp b/models/IonModel.cpp index c8c1fba8..0a8f779a 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -1,6 +1,7 @@ /* * Dilute Ion Transport LBM Model */ +#include #include "models/IonModel.h" #include "analysis/distance.h" #include "common/ReadMicroCT.h" @@ -8,6 +9,7 @@ ScaLBL_IonModel::ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM): rank(RANK),nprocs(NP),timestep(0),timestepMax(0),time_conv(0),kb(0),electron_charge(0),T(0),Vt(0),k2_inv(0),h(0), tolerance(0),number_ion_species(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0), +fluidVelx_dummy(0),fluidVely_dummy(0),fluidVelz_dummy(0), BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -16,18 +18,12 @@ ScaLBL_IonModel::~ScaLBL_IonModel(){ } -void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stokes,double time_conv_Stokes){ +void ScaLBL_IonModel::ReadParams(string filename,vector &num_iter){ // read the input database db = std::make_shared( filename ); domain_db = db->getDatabase( "Domain" ); ion_db = db->getDatabase( "Ions" ); - - //------ Load number of iteration from multiphysics controller ------// - timestepMax = num_iter; - //compute time conversion factor for ion model - time_conv = num_iter_Stokes*time_conv_Stokes/num_iter; - //-------------------------------------------------------------------// // Universal constant kb = 1.38e-23;//Boltzmann constant;unit [J/K] @@ -36,26 +32,30 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke //---------------------- Default model parameters --------------------------// T = 300.0;//temperature; unit [K] Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] - k2_inv = 4.5;//speed of sound for D3Q7 lattice + k2_inv = 4.0;//speed of sound for D3Q7 lattice h = 1.0;//resolution; unit: um/lu tolerance = 1.0e-8; number_ion_species = 1; + tau.push_back(1.0); IonDiffusivity.push_back(1.0e-9);//user-input diffusivity has physical unit [m^2/sec] IonValence.push_back(1);//algebraic valence charge IonConcentration.push_back(1.0e-3);//user-input ion concentration has physical unit [mol/m^3] - //deltaT.push_back(1.0); - //tau.push_back(0.5+k2_inv*deltaT[0]*IonDiffusivity[0]); - tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); + Cin.push_back(1.0e-3);//user-input inlet boundary ion concentration;unit [mol/m^3] + Cout.push_back(1.0e-3);//user-input outlet boundary ion concentration;unit [mol/m^3] + //tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); + time_conv.push_back((tau[0]-0.5)/k2_inv*(h*h*1.0e-12)/IonDiffusivity[0]); + fluidVelx_dummy = 0.0;//for debugging, unit [m/sec] + fluidVely_dummy = 0.0;//for debugging, unit [m/sec] + fluidVelz_dummy = 0.0;//for debugging, unit [m/sec] + Ex_dummy = 0.0;//for debugging, unit [V/m] + Ey_dummy = 0.0;//for debugging, unit [V/m] + Ez_dummy = 0.0;//for debugging, unit [V/m] //--------------------------------------------------------------------------// // Read domain parameters if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } // LB-Ion Model parameters //if (ion_db->keyExists( "timestepMax" )){ @@ -69,29 +69,63 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke //re-calculate thermal voltage Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] } + if (ion_db->keyExists( "FluidVelDummy" )){ + fluidVelx_dummy = ion_db->getVector( "FluidVelDummy" )[0]; + fluidVely_dummy = ion_db->getVector( "FluidVelDummy" )[1]; + fluidVelz_dummy = ion_db->getVector( "FluidVelDummy" )[2]; + } + if (ion_db->keyExists( "ElectricFieldDummy" )){ + Ex_dummy = ion_db->getVector( "ElectricFieldDummy" )[0]; + Ey_dummy = ion_db->getVector( "ElectricFieldDummy" )[1]; + Ez_dummy = ion_db->getVector( "ElectricFieldDummy" )[2]; + } if (ion_db->keyExists( "number_ion_species" )){ number_ion_species = ion_db->getScalar( "number_ion_species" ); } + //------ Load number of iteration from multiphysics controller ------// + if (num_iter.size()!=number_ion_species){ + ERROR("Error: number_ion_species and num_iter_Ion_List (from Multiphysics) must be of the same length! \n"); + } + else{ + timestepMax.assign(num_iter.begin(),num_iter.end()); + } + //-------------------------------------------------------------------// + if (ion_db->keyExists("tauList")){ + tau.clear(); + tau = ion_db->getVector( "tauList" ); + vectorDi = ion_db->getVector( "IonDiffusivityList" );//temp storing ion diffusivity in physical unit + if (tau.size()!=number_ion_species || Di.size()!=number_ion_species){ + ERROR("Error: number_ion_species, tauList and IonDiffusivityList must be of the same length! \n"); + } + else{ + time_conv.clear(); + for (int i=0; ikeyExists("IonDiffusivityList")){ IonDiffusivity.clear(); IonDiffusivity = ion_db->getVector( "IonDiffusivityList" ); - // time relaxation parameters tau also needs update - tau.clear(); if (IonDiffusivity.size()!=number_ion_species){ ERROR("Error: number_ion_species and IonDiffusivityList must be the same length! \n"); } else{ for (int i=0; ikeyExists("IonValenceList")){ IonValence.clear(); @@ -114,33 +148,255 @@ void ScaLBL_IonModel::ReadParams(string filename,int num_iter,int num_iter_Stoke } } } + else { + for (int i=0; ikeyExists( "BC_Solid" )){ BoundaryConditionSolid = ion_db->getScalar( "BC_Solid" ); } - - - if (rank==0) printf("*****************************************************\n"); - if (rank==0) printf("LB Ion Transport Solver: \n"); - if (rank==0) printf(" Time conversion factor: %.5g [sec/lt]\n", time_conv); - if (rank==0) printf(" Internal iteration: %i [lt]\n", timestepMax); - for (int i=0; ikeyExists( "BC" )){ + BoundaryCondition = ion_db->getScalar( "BC" ); + } + if (BoundaryCondition==1){ + //read boundary ion concentration list; INPUT unit [mol/m^3] + //it must be converted to LB unit [mol/lu^3] + + //inlet + if (ion_db->keyExists("CinList")){ + Cin.clear(); + Cin = ion_db->getVector( "CinList" ); + if (Cin.size()!=number_ion_species){ + ERROR("Error: number_ion_species and CinList must be the same length! \n"); + } + else{ + for (int i=0; ikeyExists("CoutList")){ + Cout.clear(); + Cout = ion_db->getVector( "CoutList" ); + if (Cout.size()!=number_ion_species){ + ERROR("Error: number_ion_species and CoutList must be the same length! \n"); + } + else{ + for (int i=0; i( filename ); + domain_db = db->getDatabase( "Domain" ); + ion_db = db->getDatabase( "Ions" ); + + // Universal constant + kb = 1.38e-23;//Boltzmann constant;unit [J/K] + electron_charge = 1.6e-19;//electron charge;unit [C] + + //---------------------- Default model parameters --------------------------// + T = 300.0;//temperature; unit [K] + Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] + k2_inv = 4.0;//speed of sound for D3Q7 lattice + h = 1.0;//resolution; unit: um/lu + tolerance = 1.0e-8; + number_ion_species = 1; + tau.push_back(1.0); + IonDiffusivity.push_back(1.0e-9);//user-input diffusivity has physical unit [m^2/sec] + IonValence.push_back(1);//algebraic valence charge + IonConcentration.push_back(1.0e-3);//user-input ion concentration has physical unit [mol/m^3] + Cin.push_back(1.0e-3);//user-input inlet boundary ion concentration;unit [mol/m^3] + Cout.push_back(1.0e-3);//user-input outlet boundary ion concentration;unit [mol/m^3] + //tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); + time_conv.push_back((tau[0]-0.5)/k2_inv*(h*h*1.0e-12)/IonDiffusivity[0]); + fluidVelx_dummy = 0.0;//for debugging, unit [m/sec] + fluidVely_dummy = 0.0;//for debugging, unit [m/sec] + fluidVelz_dummy = 0.0;//for debugging, unit [m/sec] + Ex_dummy = 0.0;//for debugging, unit [V/m] + Ey_dummy = 0.0;//for debugging, unit [V/m] + Ez_dummy = 0.0;//for debugging, unit [V/m] + //--------------------------------------------------------------------------// + + // Read domain parameters + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + + // LB-Ion Model parameters + //if (ion_db->keyExists( "timestepMax" )){ + // timestepMax = ion_db->getScalar( "timestepMax" ); + //} + if (ion_db->keyExists( "tolerance" )){ + tolerance = ion_db->getScalar( "tolerance" ); + } + if (ion_db->keyExists( "temperature" )){ + T = ion_db->getScalar( "temperature" ); + //re-calculate thermal voltage + Vt = kb*T/electron_charge;//thermal voltage; unit [Vy] + } + if (ion_db->keyExists( "FluidVelDummy" )){ + fluidVelx_dummy = ion_db->getVector( "FluidVelDummy" )[0]; + fluidVely_dummy = ion_db->getVector( "FluidVelDummy" )[1]; + fluidVelz_dummy = ion_db->getVector( "FluidVelDummy" )[2]; + } + if (ion_db->keyExists( "ElectricFieldDummy" )){ + Ex_dummy = ion_db->getVector( "ElectricFieldDummy" )[0]; + Ey_dummy = ion_db->getVector( "ElectricFieldDummy" )[1]; + Ez_dummy = ion_db->getVector( "ElectricFieldDummy" )[2]; + } + if (ion_db->keyExists( "number_ion_species" )){ + number_ion_species = ion_db->getScalar( "number_ion_species" ); + } + if (ion_db->keyExists("tauList")){ + tau.clear(); + tau = ion_db->getVector( "tauList" ); + vectorDi = ion_db->getVector( "IonDiffusivityList" );//temp storing ion diffusivity in physical unit + if (tau.size()!=number_ion_species || Di.size()!=number_ion_species){ + ERROR("Error: number_ion_species, tauList and IonDiffusivityList must be of the same length! \n"); + } + else{ + time_conv.clear(); + for (int i=0; ikeyExists("IonDiffusivityList")){ + IonDiffusivity.clear(); + IonDiffusivity = ion_db->getVector( "IonDiffusivityList" ); + if (IonDiffusivity.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonDiffusivityList must be the same length! \n"); + } + else{ + for (int i=0; ikeyExists("IonValenceList")){ + IonValence.clear(); + IonValence = ion_db->getVector( "IonValenceList" ); + if (IonValence.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonValenceList must be the same length! \n"); + } + } + //read initial ion concentration list; INPUT unit [mol/m^3] + //it must be converted to LB unit [mol/lu^3] + if (ion_db->keyExists("IonConcentrationList")){ + IonConcentration.clear(); + IonConcentration = ion_db->getVector( "IonConcentrationList" ); + if (IonConcentration.size()!=number_ion_species){ + ERROR("Error: number_ion_species and IonConcentrationList must be the same length! \n"); + } + else{ + for (int i=0; ikeyExists( "BC_Solid" )){ + BoundaryConditionSolid = ion_db->getScalar( "BC_Solid" ); + } + // Read boundary condition for ion transport + // BC = 0: normal periodic BC + // BC = 1: fixed inlet and outlet ion concentration + BoundaryCondition = 0; + if (ion_db->keyExists( "BC" )){ + BoundaryCondition = ion_db->getScalar( "BC" ); + } + if (BoundaryCondition==1){ + //read boundary ion concentration list; INPUT unit [mol/m^3] + //it must be converted to LB unit [mol/lu^3] + + //inlet + if (ion_db->keyExists("CinList")){ + Cin.clear(); + Cin = ion_db->getVector( "CinList" ); + if (Cin.size()!=number_ion_species){ + ERROR("Error: number_ion_species and CinList must be the same length! \n"); + } + else{ + for (int i=0; ikeyExists("CoutList")){ + Cout.clear(); + Cout = ion_db->getVector( "CoutList" ); + if (Cout.size()!=number_ion_species){ + ERROR("Error: number_ion_species and CoutList must be the same length! \n"); + } + else{ + for (int i=0; iid[n] = 0; // set mask to zero since this is an immobile component @@ -360,6 +616,27 @@ void ScaLBL_IonModel::Initialize(){ ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); } + + if (rank==0) printf("*****************************************************\n"); + if (rank==0) printf("LB Ion Transport Solver: \n"); + for (int i=0; iSendD3Q7AA(fq, ic); //READ FROM NORMAL + for (int ic=0; icSendD3Q7AA(fq, ic); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + } + //-------------------------// ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); - } + - //LB-Ion collison - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } - - // Set boundary conditions - /* ... */ - - for (int ic=0; icLastExterior(), Np); - } - if (BoundaryConditionSolid==1){ - for (int ic=0; icSolidNeumannD3Q7(&fq[ic*Np*7], IonSolid); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_Comm->SolidDirichletD3Q7(&fq[ic*Np*7], IonSolid); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); } - } - // *************EVEN TIMESTEP*************// - timestep++; - //Update ion concentration and charge density - for (int ic=0; icSendD3Q7AA(fq, ic); //READ FORM NORMAL + // *************EVEN TIMESTEP*************// + timestep++; + //Update ion concentration and charge density + ScaLBL_Comm->SendD3Q7AA(fq, ic); //READ FORM NORMAL ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE + ScaLBL_DeviceBarrier(); + // Set boundary conditions + if (BoundaryCondition == 1){ + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + } + //-------------------------// ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); - } + - //LB-Ion collison - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } - - // Set boundary conditions - /* ... */ - - for (int ic=0; icLastExterior(), Np); - } - if (BoundaryConditionSolid==1){ - for (int ic=0; icSolidNeumannD3Q7(&fq[ic*Np*7], IonSolid); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_Comm->SolidDirichletD3Q7(&fq[ic*Np*7], IonSolid); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); } } - //************************************************************************/ - } + } + + //Compute charge density for Poisson equation + for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); + } //************************************************************************/ //stoptime = MPI_Wtime(); //if (rank==0) printf("-------------------------------------------------------------------\n"); @@ -464,21 +737,15 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ //MLUPS *= nprocs; //if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); //if (rank==0) printf("********************************************************\n"); - } -//TODO this ruin the ion concentration on device -//need to do something similar to electric field void ScaLBL_IonModel::getIonConcentration(int timestep){ - for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ScaLBL_IonConcentration_Phys(Ci, h, ic, 0, ScaLBL_Comm->LastExterior(), Np); - } DoubleArray PhaseField(Nx,Ny,Nz); for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + IonConcentration_LB_to_Phys(PhaseField); FILE *OUTFILE; sprintf(LocalRankFilename,"Ion%02i_Time_%i.%05i.raw",ic+1,timestep,rank); @@ -489,6 +756,100 @@ void ScaLBL_IonModel::getIonConcentration(int timestep){ } +void ScaLBL_IonModel::IonConcentration_LB_to_Phys(DoubleArray &Den_reg){ + for (int k=0;k &ci_avg_previous){ + double *Ci_host; + Ci_host = new double[Np]; + vector error(number_ion_species,0.0); + + for (int ic=0; icLastExterior(); idx++){ + ci_loc +=Ci_host[idx]; + count_loc+=1.0; + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + ci_loc +=Ci_host[idx]; + count_loc+=1.0; + } + + MPI_Allreduce(&ci_loc,&ci_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + ci_avg /= count; + double ci_avg_mag=ci_avg; + if (ci_avg==0.0) ci_avg_mag=1.0; + error[ic] = fabs(ci_avg-ci_avg_previous[ic])/fabs(ci_avg_mag); + ci_avg_previous[ic] = ci_avg; + } + double error_max; + error_max = *max_element(error.begin(),error.end()); + if (rank==0){ + printf("IonModel: error max: %.5g\n",error_max); + } + return error_max; +} + //void ScaLBL_IonModel::getIonConcentration(){ // for (int ic=0; icFirstInterior(), ScaLBL_Comm->LastInterior(), Np); diff --git a/models/IonModel.h b/models/IonModel.h index 59e5b6e6..0bc881af 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -22,7 +22,8 @@ public: ~ScaLBL_IonModel(); // functions in they should be run - void ReadParams(string filename,int num_iter,int num_iter_Stokes,double time_conv_Stokes); + void ReadParams(string filename,vector &num_iter); + void ReadParams(string filename); void ReadParams(std::shared_ptr db0); void SetDomain(); void ReadInput(); @@ -30,24 +31,30 @@ public: void Initialize(); void Run(double *Velocity, double *ElectricField); void getIonConcentration(int timestep); - + void DummyFluidVelocity(); + void DummyElectricField(); + double CalIonDenConvergence(vector &ci_avg_previous); + //bool Restart,pBC; - int timestep,timestepMax; + int timestep; + vector timestepMax; int BoundaryCondition; int BoundaryConditionSolid; double h;//domain resolution, unit [um/lu] - double time_conv; double kb,electron_charge,T,Vt; double k2_inv; double tolerance; - double Ex,Ey,Ez; + double fluidVelx_dummy,fluidVely_dummy,fluidVelz_dummy; + double Ex_dummy,Ey_dummy,Ez_dummy; int number_ion_species; vector IonDiffusivity;//User input unit [m^2/sec] vector IonValence; vector IonConcentration;//unit [mol/m^3] - //vector deltaT; + vector Cin;//unit [mol/m^3] + vector Cout;//unit [mol/m^3] vector tau; + vector time_conv; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -68,6 +75,8 @@ public: double *Ci; double *ChargeDensity; double *IonSolid; + double *FluidVelocityDummy; + double *ElectricFieldDummy; private: MPI_Comm comm; @@ -80,4 +89,5 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); void AssignSolidBoundary(double *ion_solid); + void IonConcentration_LB_to_Phys(DoubleArray &Den_reg); }; diff --git a/models/MultiPhysController.cpp b/models/MultiPhysController.cpp index 1453067a..a54223d3 100644 --- a/models/MultiPhysController.cpp +++ b/models/MultiPhysController.cpp @@ -1,7 +1,8 @@ #include "models/MultiPhysController.h" ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM): -rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),SchmidtNum(0),comm(COMM) +rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0), +analysis_interval(0),tolerance(0),comm(COMM) { } @@ -19,34 +20,50 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){ // Default parameters timestepMax = 10000; Restart = false; - SchmidtNum = 1.0; num_iter_Stokes=1; - num_iter_Ion=1; + num_iter_Ion.push_back(1); + analysis_interval = 500; + tolerance = 1.0e-6; // load input parameters if (study_db->keyExists( "timestepMax" )){ timestepMax = study_db->getScalar( "timestepMax" ); } - if (study_db->keyExists( "Schmidt_Number" )){ - SchmidtNum = study_db->getScalar( "Schmidt_Number" ); + if (study_db->keyExists( "analysis_interval" )){ + analysis_interval = study_db->getScalar( "analysis_interval" ); } + if (study_db->keyExists( "tolerance" )){ + tolerance = study_db->getScalar( "tolerance" ); + } + //if (study_db->keyExists( "time_conv" )){ + // time_conv = study_db->getScalar( "time_conv" ); + //} + //if (study_db->keyExists( "Schmidt_Number" )){ + // SchmidtNum = study_db->getScalar( "Schmidt_Number" ); + //} + // recalculate relevant parameters - if (SchmidtNum>1){ - num_iter_Stokes = int(round(SchmidtNum/2)*2); - num_iter_Ion = 1; - } - else if (SchmidtNum>0 && SchmidtNum<1){ - num_iter_Ion = int(round((1.0/SchmidtNum)/2)*2); - num_iter_Stokes = 1; - } - else{ - ERROR("Error: SchmidtNum (Schmidt number) must be a positive number! \n"); - } + //if (SchmidtNum>1){ + // num_iter_Stokes = int(round(SchmidtNum/2)*2); + // num_iter_Ion = 1; + //} + //else if (SchmidtNum>0 && SchmidtNum<1){ + // num_iter_Ion = int(round((1.0/SchmidtNum)/2)*2); + // num_iter_Stokes = 1; + //} + //else if (SchmidtNum==1){ + // num_iter_Stokes = 1; + // num_iter_Ion = 1; + //} + //else{ + // ERROR("Error: SchmidtNum (Schmidt number) must be a positive number! \n"); + //} // load input parameters // in case user wants to have an absolute control over the iternal iteration - if (study_db->keyExists( "num_iter_Ion" )){ - num_iter_Ion = study_db->getScalar( "num_iter_Ion" ); + if (study_db->keyExists( "num_iter_Ion_List" )){ + num_iter_Ion.clear(); + num_iter_Ion = study_db->getVector( "num_iter_Ion_List" ); } if (study_db->keyExists( "num_iter_Stokes" )){ num_iter_Stokes = study_db->getScalar( "num_iter_Stokes" ); @@ -54,4 +71,70 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){ } +int ScaLBL_Multiphys_Controller::getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector &IonTimeConv){ + //Return number of internal iterations for the Stokes solver + int num_iter_stokes; + vector TimeConv; + printf("*****Debug; IonTimeConv size = %i\n",IonTimeConv.size()); + for (unsigned int i =0; i::iterator it_max = max_element(TimeConv.begin(),TimeConv.end()); + int idx_max = distance(TimeConv.begin(),it_max); + if (idx_max==0){ + num_iter_stokes = 2; + } + else{ + double temp = 2*TimeConv[idx_max]/StokesTimeConv;//the factor 2 is the number of iterations for the element has max time_conv + num_iter_stokes = int(round(temp/2)*2); + } + return num_iter_stokes; +} + +vector ScaLBL_Multiphys_Controller::getIonNumIter_PNP_coupling(double StokesTimeConv,const vector &IonTimeConv){ + //Return number of internal iterations for the Ion transport solver + vector num_iter_ion; + vector TimeConv; + TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end()); + TimeConv.insert(TimeConv.begin(),StokesTimeConv); + vector::iterator it_max = max_element(TimeConv.begin(),TimeConv.end()); + unsigned int idx_max = distance(TimeConv.begin(),it_max); + if (idx_max==0){ + for (unsigned int idx=1;idx #include #include +#include +#include #include "common/ScaLBL.h" #include "common/Communication.h" @@ -22,13 +24,17 @@ public: void ReadParams(string filename); void ReadParams(std::shared_ptr db0); + int getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector &IonTimeConv); + vector getIonNumIter_PNP_coupling(double StokesTimeConv,const vector &IonTimeConv); + //void getIonNumIter_PNP_coupling(double StokesTimeConv,vector &IonTimeConv,vector &IonTimeMax); bool Restart; - //int timestep; int timestepMax; int num_iter_Stokes; - int num_iter_Ion; - double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity + vector num_iter_Ion; + int analysis_interval; + double tolerance; + //double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity int rank,nprocs; diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 89a1c42a..b74536c5 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -59,7 +59,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "BC_Solid" )){ BoundaryConditionSolid = electric_db->getScalar( "BC_Solid" ); } - // Read boundary condition for electric potentiona + // Read boundary condition for electric potential // BC = 0: normal periodic BC // BC = 1: fixed inlet and outlet potential BoundaryCondition = 0; diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index caaf2877..3b0b2d3a 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -91,12 +91,84 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density - if (rank==0) printf("*****************************************************\n"); - if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n"); - if (rank==0) printf(" Time conversion factor: %.5g [sec/lt]\n", time_conv); - if (rank==0) printf(" Internal iteration: %i [lt]\n", timestepMax); - if (rank==0) printf("*****************************************************\n"); } + +void ScaLBL_StokesModel::ReadParams(string filename){ + //NOTE the max time step is left unspecified + + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + stokes_db = db->getDatabase( "Stokes" ); + + + //---------------------- Default model parameters --------------------------// + rho_phys = 1000.0; //by default use water density; unit [kg/m^3] + nu_phys = 1.004e-6;//by default use water kinematic viscosity at 20C; unit [m^2/sec] + h = 1.0;//image resolution;[um] + tau = 1.0; + mu = (tau-0.5)/3.0;//LB kinematic viscosity;unit [lu^2/lt] + time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + rho0 = 1.0;//LB density + den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density + tolerance = 1.0e-8; + Fx = Fy = 0.0; + Fz = 1.0e-5; + //--------------------------------------------------------------------------// + + // Read domain parameters + BoundaryCondition = 0; + if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } + if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu + h = domain_db->getScalar( "voxel_length" ); + } + + // Single-fluid Navier-Stokes Model parameters + //if (stokes_db->keyExists( "timestepMax" )){ + // timestepMax = stokes_db->getScalar( "timestepMax" ); + //} + if (stokes_db->keyExists( "tolerance" )){ + tolerance = stokes_db->getScalar( "tolerance" ); + } + if (stokes_db->keyExists( "tau" )){ + tau = stokes_db->getScalar( "tau" ); + } + if (stokes_db->keyExists( "rho0" )){ + rho0 = stokes_db->getScalar( "rho0" ); + } + if (stokes_db->keyExists( "nu_phys" )){ + nu_phys = stokes_db->getScalar( "nu_phys" ); + } + if (stokes_db->keyExists( "rho_phys" )){ + rho_phys = stokes_db->getScalar( "rho_phys" ); + } + if (stokes_db->keyExists( "F" )){ + Fx = stokes_db->getVector( "F" )[0]; + Fy = stokes_db->getVector( "F" )[1]; + Fz = stokes_db->getVector( "F" )[2]; + } + if (stokes_db->keyExists( "Restart" )){ + Restart = stokes_db->getScalar( "Restart" ); + } + if (stokes_db->keyExists( "din" )){ + din = stokes_db->getScalar( "din" ); + } + if (stokes_db->keyExists( "dout" )){ + dout = stokes_db->getScalar( "dout" ); + } + if (stokes_db->keyExists( "flux" )){ + flux = stokes_db->getScalar( "flux" ); + } + + // Re-calculate model parameters due to parameter read + mu=(tau-0.5)/3.0; + time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt] + den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density + +} + void ScaLBL_StokesModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases @@ -235,6 +307,12 @@ void ScaLBL_StokesModel::Initialize(){ if (rank==0) printf("LB Single-Fluid Solver: Initializing distributions \n"); if (rank==0) printf("****************************************************************\n"); ScaLBL_D3Q19_Init(fq, Np); + + if (rank==0) printf("*****************************************************\n"); + if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n"); + if (rank==0) printf(" Time conversion factor: %.5g [sec/lt]\n", time_conv); + if (rank==0) printf(" Internal iteration: %i [lt]\n", timestepMax); + if (rank==0) printf("*****************************************************\n"); } void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ @@ -243,6 +321,7 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ timestep = 0; while (timestep < timestepMax) { //************************************************************************/ + timestep++; ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); @@ -334,34 +413,123 @@ void ScaLBL_StokesModel::Velocity_LB_to_Phys(DoubleArray &Vel_reg){ } } -//void ScaLBL_StokesModel::getVelocity(){ -// //get velocity in physical unit [m/sec] -// ScaLBL_D3Q19_Momentum_Phys(fq, Velocity, h, time_conv, Np); -// ScaLBL_DeviceBarrier(); MPI_Barrier(comm); -// -// DoubleArray PhaseField(Nx,Ny,Nz); -// ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); -// FILE *VELX_FILE; -// sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank); -// VELX_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,VELX_FILE); -// fclose(VELX_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); -// FILE *VELY_FILE; -// sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank); -// VELY_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,VELY_FILE); -// fclose(VELY_FILE); -// -// ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); -// FILE *VELZ_FILE; -// sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank); -// VELZ_FILE = fopen(LocalRankFilename,"wb"); -// fwrite(PhaseField.data(),8,N,VELZ_FILE); -// fclose(VELZ_FILE); -// -//} +vector ScaLBL_StokesModel::computeElectricForceAvg(double *ChargeDensity, double *ElectricField){ + + double *Ex_host; + double *Ey_host; + double *Ez_host; + Ex_host = new double[Np]; + Ey_host = new double[Np]; + Ez_host = new double[Np]; + + double *rhoE_host; + rhoE_host = new double[Np]; + + ScaLBL_CopyToHost(Ex_host,&ElectricField[0*Np],Np*sizeof(double)); + ScaLBL_CopyToHost(Ey_host,&ElectricField[1*Np],Np*sizeof(double)); + ScaLBL_CopyToHost(Ez_host,&ElectricField[2*Np],Np*sizeof(double)); + ScaLBL_CopyToHost(rhoE_host,ChargeDensity,Np*sizeof(double)); + + double count_loc=0; + double count; + double Fx_avg,Fy_avg,Fz_avg;//average electric field induced force + double Fx_loc,Fy_loc,Fz_loc; + Fx_loc = Fy_loc = Fz_loc = 0.0; + + for (int idx=0; idxLastExterior(); idx++){ + Fx_loc += rhoE_host[idx]*Ex_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fy_loc += rhoE_host[idx]*Ey_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fz_loc += rhoE_host[idx]*Ez_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + count_loc+=1.0; + } + for (int idx=ScaLBL_Comm->FirstInterior(); idxLastInterior(); idx++){ + Fx_loc += rhoE_host[idx]*Ex_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fy_loc += rhoE_host[idx]*Ey_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + Fz_loc += rhoE_host[idx]*Ez_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale; + count_loc+=1.0; + } + + MPI_Allreduce(&Fx_loc,&Fx_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&Fy_loc,&Fy_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&Fz_loc,&Fz_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + Fx_avg /= count; + Fy_avg /= count; + Fz_avg /= count; + + vectorF_avg{Fx_avg,Fy_avg,Fz_avg}; + + delete [] Ex_host; + delete [] Ey_host; + delete [] Ez_host; + delete [] rhoE_host; + + return F_avg; +} + +double ScaLBL_StokesModel::CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField){ + + //----------------------------------------------------- + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + + double count_loc=0; + double count; + double vax,vay,vaz; + double vax_loc,vay_loc,vaz_loc; + vax_loc = vay_loc = vaz_loc = 0.f; + for (int k=1; k 0){ + vax_loc += Velocity_x(i,j,k); + vay_loc += Velocity_y(i,j,k); + vaz_loc += Velocity_z(i,j,k); + count_loc+=1.0; + } + } + } + } + MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm); + + vax /= count; + vay /= count; + vaz /= count; + + vector Eforce; + Eforce = computeElectricForceAvg(ChargeDensity,ElectricField); + double TFx = Fx+Eforce[0];//TF: total body force + double TFy = Fy+Eforce[1]; + double TFz = Fz+Eforce[2]; + double force_mag = sqrt(TFx*TFx+TFy*TFy+TFz*TFz); + double dir_x = TFx/force_mag; + double dir_y = TFy/force_mag; + double dir_z = TFz/force_mag; + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z); + double error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate); + flow_rate_previous = flow_rate; + //---------------------------------------------------- + + //for debugging + if (rank==0){ + printf("StokesModel: error: %.5g\n",error); + } + return error; +} void ScaLBL_StokesModel::Run(){ double rlx_setA=1.0/tau; diff --git a/models/StokesModel.h b/models/StokesModel.h index d40df415..346d75c3 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -22,6 +22,7 @@ public: // functions in they should be run void ReadParams(string filename,int num_iter); + void ReadParams(string filename); void ReadParams(std::shared_ptr db0); void SetDomain(); void ReadInput(); @@ -31,6 +32,7 @@ public: void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); void getVelocity(int timestep); + double CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField); bool Restart,pBC; int timestep,timestepMax; @@ -81,4 +83,5 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); void Velocity_LB_to_Phys(DoubleArray &Vel_reg); + vector computeElectricForceAvg(double *ChargeDensity, double *ElectricField); }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 43167b3f..64232406 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -48,6 +48,9 @@ ADD_LBPM_TEST( TestTopo3D ) ADD_LBPM_TEST( TestFluxBC ) ADD_LBPM_TEST( TestMap ) ADD_LBPM_TEST( TestPoissonSolver ) +ADD_LBPM_TEST( TestIonModel ) +ADD_LBPM_TEST( TestNernstPlanck ) +ADD_LBPM_TEST( TestPNP_Stokes ) #ADD_LBPM_TEST( TestMRT ) #ADD_LBPM_TEST( TestColorGrad ) #ADD_LBPM_TEST( TestColorGradDFH ) diff --git a/tests/TestIonModel.cpp b/tests/TestIonModel.cpp new file mode 100644 index 00000000..32f8302b --- /dev/null +++ b/tests/TestIonModel.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "models/IonModel.h" +#include "models/MultiPhysController.h" + +using namespace std; + +//*************************************************************************** +// Test lattice-Boltzmann Ion Model coupled with Poisson equation +//*************************************************************************** + +int main(int argc, char **argv) +{ + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_supportci_avg_previous{0.0,0.0};//assuming 1:1 solution + while (timestep < Study.timestepMax && error > Study.tolerance){ + + timestep++; + IonModel.Run(IonModel.FluidVelocityDummy,IonModel.ElectricFieldDummy); //solve for ion transport and electric potential + timestep++;//AA operations + + if (timestep%Study.analysis_interval==0){ + error = IonModel.CalIonDenConvergence(ci_avg_previous); + } + } + IonModel.getIonConcentration(timestep); + + if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); + if (rank==0) printf("*************************************************************\n"); + + PROFILE_STOP("Main"); + PROFILE_SAVE("TestIonModel",1); + // **************************************************** + MPI_Barrier(comm); + } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); + MPI_Finalize(); +} + + diff --git a/tests/TestNernstPlanck.cpp b/tests/TestNernstPlanck.cpp new file mode 100644 index 00000000..0344e1b1 --- /dev/null +++ b/tests/TestNernstPlanck.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "models/IonModel.h" +#include "models/PoissonSolver.h" +#include "models/MultiPhysController.h" + +using namespace std; + +//*************************************************************************** +// Test lattice-Boltzmann Ion Model coupled with Poisson equation +//*************************************************************************** + +int main(int argc, char **argv) +{ + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_supportci_avg_previous{0.0,0.0};//assuming 1:1 solution + while (timestep < Study.timestepMax && error > Study.tolerance){ + + timestep++; + PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental + IonModel.Run(IonModel.FluidVelocityDummy,PoissonSolver.ElectricField); //solve for ion transport and electric potential + + timestep++;//AA operations + + if (timestep%Study.analysis_interval==0){ + error = IonModel.CalIonDenConvergence(ci_avg_previous); + } + } + + PoissonSolver.getElectricPotential(timestep); + PoissonSolver.getElectricField(timestep); + IonModel.getIonConcentration(timestep); + + if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); + if (rank==0) printf("*************************************************************\n"); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_electrokinetic_simulator",1); + // **************************************************** + MPI_Barrier(comm); + } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); + MPI_Finalize(); +} + + diff --git a/tests/TestPNP_Stokes.cpp b/tests/TestPNP_Stokes.cpp new file mode 100644 index 00000000..ee5d43c5 --- /dev/null +++ b/tests/TestPNP_Stokes.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "models/IonModel.h" +#include "models/StokesModel.h" +#include "models/PoissonSolver.h" +#include "models/MultiPhysController.h" + +using namespace std; + +//*************************************************************************** +// Test lattice-Boltzmann Ion Model coupled with Poisson equation +//*************************************************************************** + +int main(int argc, char **argv) +{ + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + if ( rank==0 && provided_thread_supportci_avg_previous{0.0,0.0};//assuming 1:1 solution + double vel_avg_previous = 0.0; + while (timestep < Study.timestepMax && error > Study.tolerance){ + + timestep++; + PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental + StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity + IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential + + timestep++;//AA operations + + if (timestep%Study.analysis_interval==0){ + error_ion = IonModel.CalIonDenConvergence(ci_avg_previous); + error_stokes = StokesModel.CalVelocityConvergence(vel_avg_previous,IonModel.ChargeDensity,PoissonSolver.ElectricField); + error = max(error_ion,error_stokes); + } + } + + PoissonSolver.getElectricPotential(timestep); + PoissonSolver.getElectricField(timestep); + IonModel.getIonConcentration(timestep); + StokesModel.getVelocity(timestep); + + if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); + if (rank==0) printf("*************************************************************\n"); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_electrokinetic_simulator",1); + // **************************************************** + MPI_Barrier(comm); + } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); + MPI_Finalize(); +} + + diff --git a/tests/TestPoissonSolver.cpp b/tests/TestPoissonSolver.cpp index 309a03c7..7d44e573 100644 --- a/tests/TestPoissonSolver.cpp +++ b/tests/TestPoissonSolver.cpp @@ -50,7 +50,6 @@ int main(int argc, char **argv) PoissonSolver.ReadInput(); PoissonSolver.Create(); PoissonSolver.Initialize(); - PoissonSolver.getElectricPotential(0); //Initialize dummy charge density for test PoissonSolver.DummyChargeDensity(); @@ -59,34 +58,6 @@ int main(int argc, char **argv) PoissonSolver.getElectricPotential(1); PoissonSolver.getElectricField(1); - //int timestep=0; - //while (timestep < Study.timestepMax){ - // - // timestep++; - // //if (rank==0) printf("timestep=%i; running Poisson solver\n",timestep); - // PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental - // //PoissonSolver.getElectricPotential(timestep); - - // //if (rank==0) printf("timestep=%i; running StokesModel\n",timestep); - // StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity - // //StokesModel.getVelocity(timestep); - - // //if (rank==0) printf("timestep=%i; running Ion model\n",timestep); - // IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential - // //IonModel.getIonConcentration(timestep); - // - // - // timestep++;//AA operations - // //-------------------------------------------- - // //potentially leave analysis module for future - // //-------------------------------------------- - //} - - //StokesModel.getVelocity(timestep); - //PoissonSolver.getElectricPotential(timestep); - //PoissonSolver.getElectricField(timestep); - //IonModel.getIonConcentration(timestep); - if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_dfh_simulator.cpp index 1df5c5e1..75fe87e5 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_dfh_simulator.cpp @@ -61,7 +61,7 @@ int main(int argc, char **argv) StokesModel.Initialize(); // initializing the model will set initial conditions for variables // Initialize LB-Ion model - IonModel.ReadParams(filename,Study.num_iter_Ion,Study.num_iter_Stokes,StokesModel.time_conv); + IonModel.ReadParams(filename,Study.num_iter_Ion); IonModel.SetDomain(); IonModel.ReadInput(); IonModel.Create(); From 039978cc81cea94762a4ee877039b2aa2d6dcaa6 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 20 Sep 2020 11:00:36 -0400 Subject: [PATCH 225/270] GPU version is available now --- common/ScaLBL.h | 5 +- cpu/Ion.cpp | 8 +- cpu/Poisson.cpp | 206 ++-- gpu/D3Q7BC.cu | 918 +--------------- gpu/Ion.cu | 344 ++++++ gpu/Poisson.cu | 330 ++++++ gpu/Stokes.cu | 995 ++++++++++++++++++ models/MultiPhysController.cpp | 13 +- models/MultiPhysController.h | 1 + tests/CMakeLists.txt | 2 +- tests/TestPNP_Stokes.cpp | 2 +- ..._electrokinetic_SingleFluid_simulator.cpp} | 47 +- 12 files changed, 1858 insertions(+), 1013 deletions(-) create mode 100644 gpu/Ion.cu create mode 100644 gpu/Poisson.cu create mode 100644 gpu/Stokes.cu rename tests/{lbpm_electrokinetic_dfh_simulator.cpp => lbpm_electrokinetic_SingleFluid_simulator.cpp} (77%) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a4a5fad3..9a711330 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -102,8 +102,9 @@ extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *d extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np); -extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, - int strideY, int strideZ,int start, int finish, int Np); +//maybe deprecated +//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, +// int strideY, int strideZ,int start, int finish, int Np); // LBM Stokes Model (adapted from MRT model) diff --git a/cpu/Ion.cpp b/cpu/Ion.cpp index 236bca70..98d35142 100644 --- a/cpu/Ion.cpp +++ b/cpu/Ion.cpp @@ -229,10 +229,10 @@ extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant for (n=start; n0) + CD_tmp; + Ci = Den[n+ion_component*Np]; + CD = ChargeDensity[n]; + CD_tmp = F*IonValence*Ci; + ChargeDensity[n] = CD*(ion_component>0) + CD_tmp; } } diff --git a/cpu/Poisson.cpp b/cpu/Poisson.cpp index d76bbd42..c84350cd 100644 --- a/cpu/Poisson.cpp +++ b/cpu/Poisson.cpp @@ -235,109 +235,109 @@ extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, in } } -extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, - int strideY, int strideZ,int start, int finish, int Np){ - - int n,nn; - int ijk; - int id; - // distributions - double m1,m2,m3,m4,m5,m6,m7,m8,m9; - double m10,m11,m12,m13,m14,m15,m16,m17,m18; - double nx,ny,nz; - - for (n=start; n0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1 - //........................................................................ - nn = ijk+1; // neighbor index (get convention) - id = ID[nn]; - m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2 - //........................................................................ - nn = ijk-strideY; // neighbor index (get convention) - id = ID[nn]; - m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3 - //........................................................................ - nn = ijk+strideY; // neighbor index (get convention) - id = ID[nn]; - m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4 - //........................................................................ - nn = ijk-strideZ; // neighbor index (get convention) - id = ID[nn]; - m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5 - //........................................................................ - nn = ijk+strideZ; // neighbor index (get convention) - id = ID[nn]; - m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6 - //........................................................................ - nn = ijk-strideY-1; // neighbor index (get convention) - id = ID[nn]; - m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7 - //........................................................................ - nn = ijk+strideY+1; // neighbor index (get convention) - id = ID[nn]; - m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8 - //........................................................................ - nn = ijk+strideY-1; // neighbor index (get convention) - id = ID[nn]; - m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9 - //........................................................................ - nn = ijk-strideY+1; // neighbor index (get convention) - id = ID[nn]; - m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10 - //........................................................................ - nn = ijk-strideZ-1; // neighbor index (get convention) - id = ID[nn]; - m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11 - //........................................................................ - nn = ijk+strideZ+1; // neighbor index (get convention) - id = ID[nn]; - m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12 - //........................................................................ - nn = ijk+strideZ-1; // neighbor index (get convention) - id = ID[nn]; - m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13 - //........................................................................ - nn = ijk-strideZ+1; // neighbor index (get convention) - id = ID[nn]; - m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14 - //........................................................................ - nn = ijk-strideZ-strideY; // neighbor index (get convention) - id = ID[nn]; - m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15 - //........................................................................ - nn = ijk+strideZ+strideY; // neighbor index (get convention) - id = ID[nn]; - m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16 - //........................................................................ - nn = ijk+strideZ-strideY; // neighbor index (get convention) - id = ID[nn]; - m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17 - //........................................................................ - nn = ijk-strideZ+strideY; // neighbor index (get convention) - id = ID[nn]; - m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18 - //............Compute the Color Gradient................................... - //nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); - //ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); - //nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); - nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3 - ny = 1.f/6.f*(m3-m4); - nz = 1.f/6.f*(m5-m6); - - ElectricField[n] = nx; - ElectricField[Np+n] = ny; - ElectricField[2*Np+n] = nz; - } -} +//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC, +// int strideY, int strideZ,int start, int finish, int Np){ +// +// int n,nn; +// int ijk; +// int id; +// // distributions +// double m1,m2,m3,m4,m5,m6,m7,m8,m9; +// double m10,m11,m12,m13,m14,m15,m16,m17,m18; +// double nx,ny,nz; +// +// for (n=start; n0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1 +// //........................................................................ +// nn = ijk+1; // neighbor index (get convention) +// id = ID[nn]; +// m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2 +// //........................................................................ +// nn = ijk-strideY; // neighbor index (get convention) +// id = ID[nn]; +// m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3 +// //........................................................................ +// nn = ijk+strideY; // neighbor index (get convention) +// id = ID[nn]; +// m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4 +// //........................................................................ +// nn = ijk-strideZ; // neighbor index (get convention) +// id = ID[nn]; +// m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5 +// //........................................................................ +// nn = ijk+strideZ; // neighbor index (get convention) +// id = ID[nn]; +// m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6 +// //........................................................................ +// nn = ijk-strideY-1; // neighbor index (get convention) +// id = ID[nn]; +// m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7 +// //........................................................................ +// nn = ijk+strideY+1; // neighbor index (get convention) +// id = ID[nn]; +// m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8 +// //........................................................................ +// nn = ijk+strideY-1; // neighbor index (get convention) +// id = ID[nn]; +// m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9 +// //........................................................................ +// nn = ijk-strideY+1; // neighbor index (get convention) +// id = ID[nn]; +// m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10 +// //........................................................................ +// nn = ijk-strideZ-1; // neighbor index (get convention) +// id = ID[nn]; +// m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11 +// //........................................................................ +// nn = ijk+strideZ+1; // neighbor index (get convention) +// id = ID[nn]; +// m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12 +// //........................................................................ +// nn = ijk+strideZ-1; // neighbor index (get convention) +// id = ID[nn]; +// m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13 +// //........................................................................ +// nn = ijk-strideZ+1; // neighbor index (get convention) +// id = ID[nn]; +// m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14 +// //........................................................................ +// nn = ijk-strideZ-strideY; // neighbor index (get convention) +// id = ID[nn]; +// m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15 +// //........................................................................ +// nn = ijk+strideZ+strideY; // neighbor index (get convention) +// id = ID[nn]; +// m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16 +// //........................................................................ +// nn = ijk+strideZ-strideY; // neighbor index (get convention) +// id = ID[nn]; +// m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17 +// //........................................................................ +// nn = ijk-strideZ+strideY; // neighbor index (get convention) +// id = ID[nn]; +// m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18 +// //............Compute the Color Gradient................................... +// //nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14)); +// //ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18)); +// //nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18)); +// nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3 +// ny = 1.f/6.f*(m3-m4); +// nz = 1.f/6.f*(m5-m6); +// +// ElectricField[n] = nx; +// ElectricField[Np+n] = ny; +// ElectricField[2*Np+n] = nz; +// } +//} //extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np){ // int n; diff --git a/gpu/D3Q7BC.cu b/gpu/D3Q7BC.cu index 5f3ae92c..8d27f7d5 100644 --- a/gpu/D3Q7BC.cu +++ b/gpu/D3Q7BC.cu @@ -108,7 +108,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList } } -__global__ void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np) +__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np) { int idx, n; int nread,nr6; @@ -187,6 +187,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double int idx,n; double f0,f1,f2,f3,f4,f5,f6; idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ n = list[idx]; f0 = dist[n]; f1 = dist[2*Np+n]; @@ -232,7 +233,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList } } -extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np) +__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np) { int idx, n; int nread,nr6; @@ -264,940 +265,113 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, in } } -__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np) -{ - int idx, n; - int nread; - int nr5,nr11,nr14,nr15,nr18; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double ux,uy,uz,Cyz,Cxz; - ux = uy = 0.0; - - idx = blockIdx.x*blockDim.x + threadIdx.x; - - if (idx < count){ - - n = list[idx]; - f0 = dist[n]; - - nread = d_neighborList[n]; - f1 = dist[nread]; - - nread = d_neighborList[n+2*Np]; - f3 = dist[nread]; - - nread = d_neighborList[n+6*Np]; - f7 = dist[nread]; - - nread = d_neighborList[n+8*Np]; - f9 = dist[nread]; - - nread = d_neighborList[n+12*Np]; - f13 = dist[nread]; - - nread = d_neighborList[n+16*Np]; - f17 = dist[nread]; - - nread = d_neighborList[n+Np]; - f2 = dist[nread]; - - nread = d_neighborList[n+3*Np]; - f4 = dist[nread]; - - nread = d_neighborList[n+5*Np]; - f6 = dist[nread]; - - nread = d_neighborList[n+7*Np]; - f8 = dist[nread]; - - nread = d_neighborList[n+9*Np]; - f10 = dist[nread]; - - nread = d_neighborList[n+11*Np]; - f12 = dist[nread]; - - nread = d_neighborList[n+15*Np]; - f16 = dist[nread]; - - // Unknown distributions - nr5 = d_neighborList[n+4*Np]; - nr11 = d_neighborList[n+10*Np]; - nr15 = d_neighborList[n+14*Np]; - nr14 = d_neighborList[n+13*Np]; - nr18 = d_neighborList[n+17*Np]; - - //................................................... - //........Determine the inlet flow velocity......... - //ux = (f1-f2+f7-f8+f9-f10+f11-f12+f13-f14); - //uy = (f3-f4+f7-f8-f9+f10+f15-f16+f17-f18); - uz = din - (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); - - Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux; - Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy; - - f5 = f6 + 0.33333333333333338*uz; - f11 = f12 + 0.16666666666666678*(uz+ux)-Cxz; - f14 = f13 + 0.16666666666666678*(uz-ux)+Cxz; - f15 = f16 + 0.16666666666666678*(uy+uz)-Cyz; - f18 = f17 + 0.16666666666666678*(uz-uy)+Cyz; - //........Store in "opposite" memory location.......... - dist[nr5] = f5; - dist[nr11] = f11; - dist[nr14] = f14; - dist[nr15] = f15; - dist[nr18] = f18; - } -} - -__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *d_neighborList, int *list, double *dist, double dout, int count, int Np) -{ - int idx,n,nread; - int nr6,nr12,nr13,nr16,nr17; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double ux,uy,uz,Cyz,Cxz; - ux = uy = 0.0; - - idx = blockIdx.x*blockDim.x + threadIdx.x; - - // Loop over the boundary - threadblocks delineated by start...finish - if ( idx < count ){ - - n = list[idx]; - //........................................................................ - // Read distributions - //........................................................................ - f0 = dist[n]; - - nread = d_neighborList[n]; - f1 = dist[nread]; - - nread = d_neighborList[n+2*Np]; - f3 = dist[nread]; - - nread = d_neighborList[n+4*Np]; - f5 = dist[nread]; - - nread = d_neighborList[n+6*Np]; - f7 = dist[nread]; - - nread = d_neighborList[n+8*Np]; - f9 = dist[nread]; - - nread = d_neighborList[n+10*Np]; - f11 = dist[nread]; - - nread = d_neighborList[n+14*Np]; - f15 = dist[nread]; - - - nread = d_neighborList[n+Np]; - f2 = dist[nread]; - - nread = d_neighborList[n+3*Np]; - f4 = dist[nread]; - - nread = d_neighborList[n+7*Np]; - f8 = dist[nread]; - - nread = d_neighborList[n+9*Np]; - f10 = dist[nread]; - - nread = d_neighborList[n+13*Np]; - f14 = dist[nread]; - - nread = d_neighborList[n+17*Np]; - f18 = dist[nread]; - - // unknown distributions - nr6 = d_neighborList[n+5*Np]; - nr12 = d_neighborList[n+11*Np]; - nr16 = d_neighborList[n+15*Np]; - nr17 = d_neighborList[n+16*Np]; - nr13 = d_neighborList[n+12*Np]; - - - //........Determine the outlet flow velocity......... - //ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14; - //uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18; - uz = -dout + (f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f5+f11+f14+f15+f18)); - - Cxz = 0.5*(f1+f7+f9-f2-f10-f8) - 0.3333333333333333*ux; - Cyz = 0.5*(f3+f7+f10-f4-f9-f8) - 0.3333333333333333*uy; - - f6 = f5 - 0.33333333333333338*uz; - f12 = f11 - 0.16666666666666678*(uz+ux)+Cxz; - f13 = f14 - 0.16666666666666678*(uz-ux)-Cxz; - f16 = f15 - 0.16666666666666678*(uy+uz)+Cyz; - f17 = f18 - 0.16666666666666678*(uz-uy)-Cyz; - - //........Store in "opposite" memory location.......... - dist[nr6] = f6; - dist[nr12] = f12; - dist[nr13] = f13; - dist[nr16] = f16; - dist[nr17] = f17; - //................................................... - } -} - - -__global__ void dvc_ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double Area, - double *dvcsum, int count, int Np) -{ - int idx, n; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double factor = 1.f/(Area); - double sum = 0.f; - - idx = blockIdx.x*blockDim.x + threadIdx.x; - - if (idx < count){ - - n = list[idx]; - f0 = dist[n]; - f1 = dist[2*Np+n]; - f2 = dist[1*Np+n]; - f3 = dist[4*Np+n]; - f4 = dist[3*Np+n]; - f6 = dist[5*Np+n]; - f7 = dist[8*Np+n]; - f8 = dist[7*Np+n]; - f9 = dist[10*Np+n]; - f10 = dist[9*Np+n]; - f12 = dist[11*Np+n]; - f13 = dist[14*Np+n]; - f16 = dist[15*Np+n]; - f17 = dist[18*Np+n]; - sum = factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); - } - - //sum = blockReduceSum(sum); - //if (threadIdx.x==0) - // atomicAdd(dvcsum, sum); - - extern __shared__ double temp[]; - thread_group g = this_thread_block(); - double block_sum = reduce_sum(g, temp, sum); - - if (g.thread_rank() == 0) atomicAdd(dvcsum, block_sum); -} - - -__global__ void dvc_ScaLBL_D3Q19_AAodd_Flux_BC_z(int *d_neighborList, int *list, double *dist, double flux, - double Area, double *dvcsum, int count, int Np) -{ - int idx, n; - int nread; - - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double factor = 1.f/(Area); - double sum = 0.f; - - idx = blockIdx.x*blockDim.x + threadIdx.x; - - if (idx < count){ - - n = list[idx]; - - f0 = dist[n]; - - nread = d_neighborList[n]; - f1 = dist[nread]; - - nread = d_neighborList[n+2*Np]; - f3 = dist[nread]; - - nread = d_neighborList[n+6*Np]; - f7 = dist[nread]; - - nread = d_neighborList[n+8*Np]; - f9 = dist[nread]; - - nread = d_neighborList[n+12*Np]; - f13 = dist[nread]; - - nread = d_neighborList[n+16*Np]; - f17 = dist[nread]; - - nread = d_neighborList[n+Np]; - f2 = dist[nread]; - - nread = d_neighborList[n+3*Np]; - f4 = dist[nread]; - - nread = d_neighborList[n+5*Np]; - f6 = dist[nread]; - - nread = d_neighborList[n+7*Np]; - f8 = dist[nread]; - - nread = d_neighborList[n+9*Np]; - f10 = dist[nread]; - - nread = d_neighborList[n+11*Np]; - f12 = dist[nread]; - - nread = d_neighborList[n+15*Np]; - f16 = dist[nread]; - - sum = factor*(f0+f1+f2+f3+f4+f7+f8+f9+f10 + 2*(f6+f12+f13+f16+f17)); - - } - - //sum = blockReduceSum(sum); - //if (threadIdx.x==0) - // atomicAdd(dvcsum, sum); - - extern __shared__ double temp[]; - thread_group g = this_thread_block(); - double block_sum = reduce_sum(g, temp, sum); - - if (g.thread_rank() == 0) atomicAdd(dvcsum, block_sum); -} - - -__global__ void dvc_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz, - int Nx, int Ny, int Nz) -{ - int n,N; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double din; - - N = Nx*Ny*Nz; - n = Nx*Ny + blockIdx.x*blockDim.x + threadIdx.x; - - if (n < 2*Nx*Ny){ - //........................................................................ - // Read distributions from "opposite" memory convention - //........................................................................ - //........................................................................ - f1 = distodd[n]; - f3 = distodd[N+n]; - f5 = distodd[2*N+n]; - f7 = distodd[3*N+n]; - f9 = distodd[4*N+n]; - f11 = distodd[5*N+n]; - f13 = distodd[6*N+n]; - f15 = distodd[7*N+n]; - f17 = distodd[8*N+n]; - //........................................................................ - f0 = disteven[n]; - f2 = disteven[N+n]; - f4 = disteven[2*N+n]; - f6 = disteven[3*N+n]; - f8 = disteven[4*N+n]; - f10 = disteven[5*N+n]; - f12 = disteven[6*N+n]; - f14 = disteven[7*N+n]; - f16 = disteven[8*N+n]; - f18 = disteven[9*N+n]; - //................................................... - - // Determine the outlet flow velocity - // uz = 1.0 - (f0+f4+f3+f2+f1+f8+f7+f9+f10 + - // 2*(f5+f15+f18+f11+f14))/din; - din = (f0+f4+f3+f2+f1+f8+f7+f9+f10+2*(f5+f15+f18+f11+f14))/(1.0-uz); - // Set the unknown distributions: - f6 = f5 + 0.3333333333333333*din*uz; - f16 = f15 + 0.1666666666666667*din*uz; - f17 = f16 + f4 - f3-f15+f18+f8-f7 +f9-f10; - f12= (din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f2+f1-f14+f11-f8+f7+f9-f10)*0.5; - f13= din*uz+f5+ f15+f18+f11+f14-f6-f16-f17-f12; - - //........Store in "opposite" memory location.......... - disteven[3*N+n] = f6; - disteven[6*N+n] = f12; - distodd[6*N+n] = f13; - disteven[8*N+n] = f16; - distodd[8*N+n] = f17; - //................................................... - } -} - -__global__ void dvc_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, - int Nx, int Ny, int Nz, int outlet){ - int n,N; - // distributions - double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; - double f10,f11,f12,f13,f14,f15,f16,f17,f18; - double dout; - - N = Nx*Ny*Nz; - n = outlet + blockIdx.x*blockDim.x + threadIdx.x; - - // Loop over the boundary - threadblocks delineated by start...finish - if ( n 0 ){ - f_even[n] = 0 + 0.01*0; - f_odd[n] = 0+ 0.01*1; //double(100*n)+1.f; - f_even[N+n] = 1+ 0.01*2; //double(100*n)+2.f; - f_odd[N+n] = 1+ 0.01*3; //double(100*n)+3.f; - f_even[2*N+n] = 2+ 0.01*4; //double(100*n)+4.f; - f_odd[2*N+n] = 2+ 0.01*5; //double(100*n)+5.f; - f_even[3*N+n] = 3+ 0.01*6; //double(100*n)+6.f; - f_odd[3*N+n] = 3+ 0.01*7; //double(100*n)+7.f; - f_even[4*N+n] = 4+ 0.01*8; //double(100*n)+8.f; - f_odd[4*N+n] = 4+ 0.01*9; //double(100*n)+9.f; - f_even[5*N+n] = 5+ 0.01*10; //double(100*n)+10.f; - f_odd[5*N+n] = 5+ 0.01*11; //double(100*n)+11.f; - f_even[6*N+n] = 6+ 0.01*12; //double(100*n)+12.f; - f_odd[6*N+n] = 6+ 0.01*13; //double(100*n)+13.f; - f_even[7*N+n] = 7+ 0.01*14; //double(100*n)+14.f; - f_odd[7*N+n] = 7+ 0.01*15; //double(100*n)+15.f; - f_even[8*N+n] = 8+ 0.01*16; //double(100*n)+16.f; - f_odd[8*N+n] = 8+ 0.01*17; //double(100*n)+17.f; - f_even[9*N+n] = 9+ 0.01*18; //double(100*n)+18.f; - } - else{ - for(int q=0; q<9; q++){ - f_even[q*N+n] = -1.0; - f_odd[q*N+n] = -1.0; - } - f_even[9*N+n] = -1.0; - } - } - } -} - - //************************************************************************* -//extern "C" void ScaLBL_D3Q19_MapRecv(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, -// int *d3q19_recvlist, int Nx, int Ny, int Nz){ -// int GRID = count / 512 + 1; -// dvc_ScaLBL_D3Q19_Unpack <<>>(q, Cqx, Cqy, Cqz, list, start, count, d3q19_recvlist, Nx, Ny, Nz); -//} - -extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N){ +extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Pack <<>>(q, list, start, count, sendbuf, dist, N); + dvc_ScaLBL_Solid_Dirichlet_D3Q7<<>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",cudaGetErrorString(err)); + } } -extern "C" void ScaLBL_D3Q19_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N){ +extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Unpack <<>>(q, list, start, count, recvbuf, dist, N); -} -//************************************************************************* - -extern "C" void ScaLBL_D3Q19_AA_Init(double *f_even, double *f_odd, int Np){ - dvc_ScaLBL_D3Q19_AA_Init<<>>(f_even, f_odd, Np); + dvc_ScaLBL_Solid_Neumann_D3Q7<<>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){ - dvc_ScaLBL_D3Q19_Init<<>>(dist, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *dist, int Np, double Den){ - dvc_ScaLBL_D3Q19_GreyIMRT_Init<<>>(dist, Np, Den); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_GreyIMRT_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){ - dvc_ScaLBL_D3Q19_Swap<<>>(ID, disteven, distodd, Nx, Ny, Nz); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Swap: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np) -{ - - const int Q = 9; - // cudaStream_t streams[Q]; - // Launch the swap operation as different kernels - for (int q=0; q>>(neighborList, disteven, distodd, Np, q); - } - // cpu should wait for all kernels to finish (to avoid launch of dependent kernels) - //cudaDeviceSynchronize(); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Swap: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAeven_Compact(char * ID, double *d_dist, int Np) { - cudaFuncSetCacheConfig(dvc_ScaLBL_AAeven_Compact, cudaFuncCachePreferL1); - dvc_ScaLBL_AAeven_Compact<<>>(ID, d_dist, Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_AAodd_Compact(char * ID, int *d_neighborList, double *d_dist, int Np) { - cudaFuncSetCacheConfig(dvc_ScaLBL_AAodd_Compact, cudaFuncCachePreferL1); - dvc_ScaLBL_AAodd_Compact<<>>(ID,d_neighborList, d_dist,Np); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np){ - - dvc_ScaLBL_D3Q19_Momentum<<>>(dist, vel, Np); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Velocity: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_Pressure(double *fq, double *Pressure, int Np){ - dvc_ScaLBL_D3Q19_Pressure<<< NBLOCKS,NTHREADS >>>(fq, Pressure, Np); -} - -extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,int Nx, int Ny, int Nz){ - int GRID = Nx*Ny / 512 + 1; - dvc_D3Q19_Velocity_BC_z<<>>(disteven,distodd, uz, Nx, Ny, Nz); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_z: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, int Nx, int Ny, int Nz, int outlet){ - int GRID = Nx*Ny / 512 + 1; - dvc_D3Q19_Velocity_BC_Z<<>>(disteven, distodd, uz, Nx, Ny, Nz, outlet); - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_Z: %s \n",cudaGetErrorString(err)); - } -} - -extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, double flux,int Nx, int Ny, int Nz){ - - int GRID = Nx*Ny / 512 + 1; - - // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 - if (Nx*Ny > 512*512){ - printf("WARNING (ScaLBL_D3Q19_Flux_BC_z): CUDA reduction operation may fail if Nx*Ny > 512*512"); - } - - // Allocate memory to store the sums - double din; - double sum[1]; - double *dvcsum; - int sharedBytes = NTHREADS*sizeof(double); - cudaMalloc((void **)&dvcsum,sizeof(double)*Nx*Ny); - cudaMemset(dvcsum,0,sizeof(double)*Nx*Ny); - - cudaError_t err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); - } - - // compute the local flux and store the result - dvc_D3Q19_Flux_BC_z<<>>(disteven, distodd, flux, dvcsum, Nx, Ny, Nz); - - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 1): %s \n",cudaGetErrorString(err)); - } - - // Now read the total flux - cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); - din=sum[0]; - - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 2): %s \n",cudaGetErrorString(err)); - } - - // free the memory needed for reduction - cudaFree(dvcsum); - - return din; -} - - -extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int N){ +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_z<<>>(list, dist, din, count, N); + dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<>>(list, dist, Vin, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int N){ +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z<<>>(list, dist, dout, count, N); + dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<>>(list, dist, Vout, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int N){ +extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z<<>>(neighborList, list, dist, din, count, N); + dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<>>(d_neighborList, list, dist, Vin, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int N){ +extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_Z<<>>(neighborList, list, dist, dout, count, N); + dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<>>(d_neighborList, list, dist, Vout, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err)); } } -//******************************************************************************* -//******************************************************************************* -//******************************************************************************* - - -//******************************************************************************* -extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area, - int count, int N){ +extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){ int GRID = count / 512 + 1; - - // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 - if (count > 512*512){ - printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if count > 512*512"); - } - - // Allocate memory to store the sums - double din; - double sum[1]; - double *dvcsum; - cudaMalloc((void **)&dvcsum,sizeof(double)*count); - cudaMemset(dvcsum,0,sizeof(double)*count); - int sharedBytes = 512*sizeof(double); - + dvc_ScaLBL_Poisson_D3Q7_BC_z<<>>(list, Map, Psi, Vin, count); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",cudaGetErrorString(err)); } - - // compute the local flux and store the result - dvc_ScaLBL_D3Q19_AAeven_Flux_BC_z<<>>(list, dist, flux, area, dvcsum, count, N); - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); - } - - // Now read the total flux - cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); - din=sum[0]; - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err)); - } - - // free the memory needed for reduction - cudaFree(dvcsum); - - return din; } -extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, double *dist, double flux, - double area, int count, int N){ - +extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){ int GRID = count / 512 + 1; - - // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 - if (count > 512*512){ - printf("WARNING (ScaLBL_D3Q19_AAodd_Flux_BC_z): CUDA reduction operation may fail if count > 512*512"); - } - - // Allocate memory to store the sums - double din; - double sum[1]; - double *dvcsum; - cudaMalloc((void **)&dvcsum,sizeof(double)*count); - cudaMemset(dvcsum,0,sizeof(double)*count); - int sharedBytes = 512*sizeof(double); + dvc_ScaLBL_Poisson_D3Q7_BC_Z<<>>(list, Map, Psi, Vout, count); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",cudaGetErrorString(err)); } - - // compute the local flux and store the result - dvc_ScaLBL_D3Q19_AAodd_Flux_BC_z<<>>(neighborList, list, dist, flux, area, dvcsum, count, N); - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); - } - // Now read the total flux - cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); - din=sum[0]; - err = cudaGetLastError(); - if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err)); - } - - // free the memory needed for reduction - cudaFree(dvcsum); - - return din; } -extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, double flux, int Nx, int Ny, int Nz, int outlet){ - - int GRID = Nx*Ny / 512 + 1; - - // IMPORTANT -- this routine may fail if Nx*Ny > 512*512 - if (Nx*Ny > 512*512){ - printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if Nx*Ny > 512*512"); - } - - // Allocate memory to store the sums - double dout; - double sum[1]; - double *dvcsum; - cudaMalloc((void **)&dvcsum,sizeof(double)*Nx*Ny); - cudaMemset(dvcsum,0,sizeof(double)*Nx*Ny); - - // compute the local flux and store the result - dvc_D3Q19_Flux_BC_Z<<>>(disteven, distodd, flux, dvcsum, Nx, Ny, Nz, outlet); - - // Now read the total flux - cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost); - - // free the memory needed for reduction - - dout = sum[0]; - - cudaFree(dvcsum); - - return dout; - -} - -extern "C" double deviceReduce(double *in, double* out, int N) { - int threads = 512; - int blocks = min((N + threads - 1) / threads, 1024); - - double sum = 0.f; - deviceReduceKernel<<>>(in, out, N); - deviceReduceKernel<<<1, 1024>>>(out, out, blocks); - return sum; -} - -extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Reflection_BC_z<<>>(list, dist, count, Np); + dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<>>(list, dist, Cin, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){ int GRID = count / 512 + 1; - dvc_ScaLBL_D3Q19_Reflection_BC_Z<<>>(list, dist, count, Np); + dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<>>(list, dist, Cout, count, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, - double Fy, double Fz){ - - dvc_ScaLBL_AAeven_MRT<<>>(dist,start,finish,Np,rlx_setA,rlx_setB,Fx,Fy,Fz); - - cudaError_t err = cudaGetLastError(); +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<>>(d_neighborList, list, dist, Cin, count, Np); + cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_MRT: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err)); } } -extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborlist, double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, - double Fy, double Fz){ - - dvc_ScaLBL_AAodd_MRT<<>>(neighborlist,dist,start,finish,Np,rlx_setA,rlx_setB,Fx,Fy,Fz); - - cudaError_t err = cudaGetLastError(); +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<>>(d_neighborList, list, dist, Cout, count, Np); + cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ - printf("CUDA error in ScaLBL_D3Q19_AAeven_MRT: %s \n",cudaGetErrorString(err)); + printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err)); } } diff --git a/gpu/Ion.cu b/gpu/Ion.cu new file mode 100644 index 00000000..dc94d2e5 --- /dev/null +++ b/gpu/Ion.cu @@ -0,0 +1,344 @@ +#include +#include +//#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){ + int n,nread; + double fq,Ci; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + // q=2 + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + // q=4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + // q=6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + // q=0 + dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + + // q=2 + dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + } + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, + double Di, int zi, double rlx, double Vt, int start, int finish, int Np){ + int n; + double Ci; + double ux,uy,uz; + double uEPx,uEPy,uEPz;//electrochemical induced velocity + double Ex,Ey,Ez;//electrical field + double f0,f1,f2,f3,f4,f5,f6; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s0) + CD_tmp; + } + } +} + + +extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<>>(neighborList,dist,Den,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<>>(dist,Den,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, + double Di, int zi, double rlx, double Vt, int start, int finish, int Np){ + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAodd_Ion<<>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, + double Di, int zi, double rlx, double Vt, int start, int finish, int Np){ + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAeven_Ion<<>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_Ion_Init<<>>(dist,Den,DenInit,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Ion_Init: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} diff --git a/gpu/Poisson.cu b/gpu/Poisson.cu new file mode 100644 index 00000000..84a78330 --- /dev/null +++ b/gpu/Poisson.cu @@ -0,0 +1,330 @@ +#include +#include +//#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){ + int n; + double psi;//electric potential + double fq; + int nread; + int idx; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + f1 = dist[nr1]; // reading the f1 data into register fq + + nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + f2 = dist[nr2]; // reading the f2 data into register fq + + // q=3 + nr3 = neighborList[n+2*Np]; // neighbor 4 + f3 = dist[nr3]; + + // q = 4 + nr4 = neighborList[n+3*Np]; // neighbor 3 + f4 = dist[nr4]; + + // q=5 + nr5 = neighborList[n+4*Np]; + f5 = dist[nr5]; + + // q = 6 + nr6 = neighborList[n+5*Np]; + f6 = dist[nr6]; + + Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu + Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound + Ez = (f5-f6)*rlx*4.0; + ElectricField[n+0*Np] = Ex; + ElectricField[n+1*Np] = Ey; + ElectricField[n+2*Np] = Ez; + + // q = 0 + dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e); + + // q = 1 + dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + + // q = 2 + dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + + // q = 3 + dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + + // q = 4 + dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + + // q = 5 + dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + + // q = 6 + dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); + //........................................................................ + } + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){ + + int n; + double psi;//electric potential + double Ex,Ey,Ez;//electric field + double rho_e;//local charge density + double f0,f1,f2,f3,f4,f5,f6; + double rlx=1.0/tau; + int idx; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(neighborList,Map,dist,Psi,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential<<>>(Map,dist,Psi,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAodd_Poisson<<>>(neighborList,Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_AAeven_Poisson<<>>(Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q7_Poisson_Init<<>>(Map,dist,Psi,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Poisson_Init: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} diff --git a/gpu/Stokes.cu b/gpu/Stokes.cu new file mode 100644 index 00000000..d091b0b4 --- /dev/null +++ b/gpu/Stokes.cu @@ -0,0 +1,995 @@ +#include +#include +//#include + +#define NBLOCKS 1024 +#define NTHREADS 256 + +__global__ void dvc_ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np){ + + int n; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double ux,uy,uz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + int nread; + // body force due to electric field + double rhoE;//charge density + double Ex,Ey,Ez; + // total body force + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s 10Np => odd part of dist) + fq = dist[nread]; // reading the f1 data into register fq + //fp = dist[10*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jx = fq; + m4 = -4.0*fq; + m9 = 2.0*fq; + m10 = -4.0*fq; + + // f2 = dist[10*Np+n]; + nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist) + fq = dist[nread]; // reading the f2 data into register fq + //fq = dist[Np+n]; + rho += fq; + m1 -= 11.0*(fq); + m2 -= 4.0*(fq); + jx -= fq; + m4 += 4.0*(fq); + m9 += 2.0*(fq); + m10 -= 4.0*(fq); + + // q=3 + nread = neighborList[n+2*Np]; // neighbor 4 + fq = dist[nread]; + //fq = dist[11*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy = fq; + m6 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 = fq; + m12 = -2.0*fq; + + // q = 4 + nread = neighborList[n+3*Np]; // neighbor 3 + fq = dist[nread]; + //fq = dist[2*Np+n]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jy -= fq; + m6 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 += fq; + m12 -= 2.0*fq; + + // q=5 + nread = neighborList[n+4*Np]; + fq = dist[nread]; + //fq = dist[12*Np+n]; + rho += fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz = fq; + m8 = -4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + + // q = 6 + nread = neighborList[n+5*Np]; + fq = dist[nread]; + //fq = dist[3*Np+n]; + rho+= fq; + m1 -= 11.0*fq; + m2 -= 4.0*fq; + jz -= fq; + m8 += 4.0*fq; + m9 -= fq; + m10 += 2.0*fq; + m11 -= fq; + m12 += 2.0*fq; + + // q=7 + nread = neighborList[n+6*Np]; + fq = dist[nread]; + //fq = dist[13*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 = fq; + m16 = fq; + m17 = -fq; + + // q = 8 + nread = neighborList[n+7*Np]; + fq = dist[nread]; + //fq = dist[4*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 += fq; + m16 -= fq; + m17 += fq; + + // q=9 + nread = neighborList[n+8*Np]; + fq = dist[nread]; + //fq = dist[14*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jy -= fq; + m6 -= fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 += fq; + m17 += fq; + + // q = 10 + nread = neighborList[n+9*Np]; + fq = dist[nread]; + //fq = dist[5*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jy += fq; + m6 += fq; + m9 += fq; + m10 += fq; + m11 += fq; + m12 += fq; + m13 -= fq; + m16 -= fq; + m17 -= fq; + + // q=11 + nread = neighborList[n+10*Np]; + fq = dist[nread]; + //fq = dist[15*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 = fq; + m16 -= fq; + m18 = fq; + + // q=12 + nread = neighborList[n+11*Np]; + fq = dist[nread]; + //fq = dist[6*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 += fq; + m16 += fq; + m18 -= fq; + + // q=13 + nread = neighborList[n+12*Np]; + fq = dist[nread]; + //fq = dist[16*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx += fq; + m4 += fq; + jz -= fq; + m8 -= fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 -= fq; + m18 -= fq; + + // q=14 + nread = neighborList[n+13*Np]; + fq = dist[nread]; + //fq = dist[7*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jx -= fq; + m4 -= fq; + jz += fq; + m8 += fq; + m9 += fq; + m10 += fq; + m11 -= fq; + m12 -= fq; + m15 -= fq; + m16 += fq; + m18 += fq; + + // q=15 + nread = neighborList[n+14*Np]; + fq = dist[nread]; + //fq = dist[17*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 = fq; + m17 += fq; + m18 -= fq; + + // q=16 + nread = neighborList[n+15*Np]; + fq = dist[nread]; + //fq = dist[8*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 += fq; + m17 -= fq; + m18 += fq; + + // q=17 + //fq = dist[18*Np+n]; + nread = neighborList[n+16*Np]; + fq = dist[nread]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy += fq; + m6 += fq; + jz -= fq; + m8 -= fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 += fq; + m18 += fq; + + // q=18 + nread = neighborList[n+17*Np]; + fq = dist[nread]; + //fq = dist[9*Np+n]; + rho += fq; + m1 += 8.0*fq; + m2 += fq; + jy -= fq; + m6 -= fq; + jz += fq; + m8 += fq; + m9 -= 2.0*fq; + m10 -= 2.0*fq; + m14 -= fq; + m17 -= fq; + m18 -= fq; + + // write the velocity + ux = jx / rho0; + uy = jy / rho0; + uz = jz / rho0; + Velocity[n] = ux; + Velocity[Np+n] = uy; + Velocity[2*Np+n] = uz; + + //..............incorporate external force................................................ + //..............carry out relaxation process............................................... + m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1); + m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2); + m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4); + m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6); + m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8); + m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9); + m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10); + m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11); + m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12); + m13 = m13 + rlx_setA*((jx*jy/rho0) - m13); + m14 = m14 + rlx_setA*((jy*jz/rho0) - m14); + m15 = m15 + rlx_setA*((jx*jz/rho0) - m15); + m16 = m16 + rlx_setB*( - m16); + m17 = m17 + rlx_setB*( - m17); + m18 = m18 + rlx_setB*( - m18); + //....................................................................................................... + //.................inverse transformation...................................................... + + // q=0 + fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2; + dist[n] = fq; + + // q = 1 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx; + nread = neighborList[n+Np]; + dist[nread] = fq; + + // q=2 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx; + nread = neighborList[n]; + dist[nread] = fq; + + // q = 3 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy; + nread = neighborList[n+3*Np]; + dist[nread] = fq; + + // q = 4 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy; + nread = neighborList[n+2*Np]; + dist[nread] = fq; + + // q = 5 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz; + nread = neighborList[n+5*Np]; + dist[nread] = fq; + + // q = 6 + fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz; + nread = neighborList[n+4*Np]; + dist[nread] = fq; + + // q = 7 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy); + nread = neighborList[n+7*Np]; + dist[nread] = fq; + + // q = 8 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy); + nread = neighborList[n+6*Np]; + dist[nread] = fq; + + // q = 9 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy); + nread = neighborList[n+9*Np]; + dist[nread] = fq; + + // q = 10 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4) + +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11 + +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy); + nread = neighborList[n+8*Np]; + dist[nread] = fq; + + // q = 11 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz); + nread = neighborList[n+11*Np]; + dist[nread] = fq; + + // q = 12 + fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz); + nread = neighborList[n+10*Np]; + dist[nread]= fq; + + // q = 13 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz); + nread = neighborList[n+13*Np]; + dist[nread] = fq; + + // q= 14 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4) + +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11 + -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz); + nread = neighborList[n+12*Np]; + dist[nread] = fq; + + + // q = 15 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz); + nread = neighborList[n+15*Np]; + dist[nread] = fq; + + // q = 16 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8) + -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz); + nread = neighborList[n+14*Np]; + dist[nread] = fq; + + + // q = 17 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8) + -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz); + nread = neighborList[n+17*Np]; + dist[nread] = fq; + + // q = 18 + fq = mrt_V1*rho+mrt_V9*m1 + +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6) + -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz); + nread = neighborList[n+16*Np]; + dist[nread] = fq; + } + } +} + +__global__ void dvc_ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){ + + int n; + double fq; + // conserved momemnts + double rho,jx,jy,jz; + double ux,uy,uz; + // non-conserved moments + double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18; + // body force due to electric field + double rhoE;//charge density + double Ex,Ey,Ez; + // total body force + double Fx,Fy,Fz; + + const double mrt_V1=0.05263157894736842; + const double mrt_V2=0.012531328320802; + const double mrt_V3=0.04761904761904762; + const double mrt_V4=0.004594820384294068; + const double mrt_V5=0.01587301587301587; + const double mrt_V6=0.0555555555555555555555555; + const double mrt_V7=0.02777777777777778; + const double mrt_V8=0.08333333333333333; + const double mrt_V9=0.003341687552213868; + const double mrt_V10=0.003968253968253968; + const double mrt_V11=0.01388888888888889; + const double mrt_V12=0.04166666666666666; + + int S = Np/NBLOCKS/NTHREADS + 1; + for (int s=0; s>>(neighborList,dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAodd_StokesMRT: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + +extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){ + + //cudaProfilerStart(); + dvc_ScaLBL_D3Q19_AAeven_StokesMRT<<>>(dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q19_AAeven_StokesMRT: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + diff --git a/models/MultiPhysController.cpp b/models/MultiPhysController.cpp index a54223d3..fcfb5403 100644 --- a/models/MultiPhysController.cpp +++ b/models/MultiPhysController.cpp @@ -2,7 +2,7 @@ ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM): rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0), -analysis_interval(0),tolerance(0),comm(COMM) +analysis_interval(0),visualization_interval(0),tolerance(0),comm(COMM) { } @@ -23,6 +23,7 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){ num_iter_Stokes=1; num_iter_Ion.push_back(1); analysis_interval = 500; + visualization_interval = 10000; tolerance = 1.0e-6; // load input parameters @@ -32,6 +33,9 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){ if (study_db->keyExists( "analysis_interval" )){ analysis_interval = study_db->getScalar( "analysis_interval" ); } + if (study_db->keyExists( "visualization_interval" )){ + visualization_interval = study_db->getScalar( "visualization_interval" ); + } if (study_db->keyExists( "tolerance" )){ tolerance = study_db->getScalar( "tolerance" ); } @@ -76,15 +80,8 @@ int ScaLBL_Multiphys_Controller::getStokesNumIter_PNP_coupling(double StokesTime int num_iter_stokes; vector TimeConv; - printf("*****Debug; IonTimeConv size = %i\n",IonTimeConv.size()); - for (unsigned int i =0; i::iterator it_max = max_element(TimeConv.begin(),TimeConv.end()); int idx_max = distance(TimeConv.begin(),it_max); if (idx_max==0){ diff --git a/models/MultiPhysController.h b/models/MultiPhysController.h index f1c51f93..f217248f 100644 --- a/models/MultiPhysController.h +++ b/models/MultiPhysController.h @@ -33,6 +33,7 @@ public: int num_iter_Stokes; vector num_iter_Ion; int analysis_interval; + int visualization_interval; double tolerance; //double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 64232406..7c854c72 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,7 +4,7 @@ ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_dfh_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) diff --git a/tests/TestPNP_Stokes.cpp b/tests/TestPNP_Stokes.cpp index ee5d43c5..f6369b89 100644 --- a/tests/TestPNP_Stokes.cpp +++ b/tests/TestPNP_Stokes.cpp @@ -112,7 +112,7 @@ int main(int argc, char **argv) if (rank==0) printf("*************************************************************\n"); PROFILE_STOP("Main"); - PROFILE_SAVE("lbpm_electrokinetic_simulator",1); + PROFILE_SAVE("TestPNP_Stokes",1); // **************************************************** MPI_Barrier(comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize diff --git a/tests/lbpm_electrokinetic_dfh_simulator.cpp b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp similarity index 77% rename from tests/lbpm_electrokinetic_dfh_simulator.cpp rename to tests/lbpm_electrokinetic_SingleFluid_simulator.cpp index 75fe87e5..233889ac 100644 --- a/tests/lbpm_electrokinetic_dfh_simulator.cpp +++ b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp @@ -7,15 +7,15 @@ #include #include -#include "models/StokesModel.h" #include "models/IonModel.h" +#include "models/StokesModel.h" #include "models/PoissonSolver.h" #include "models/MultiPhysController.h" using namespace std; //*************************************************************************** -// Implementation of Multiphysics simulator using lattice-Boltzmann method +// Test lattice-Boltzmann Ion Model coupled with Poisson equation //*************************************************************************** int main(int argc, char **argv) @@ -35,7 +35,7 @@ int main(int argc, char **argv) { if (rank == 0){ printf("********************************************************\n"); - printf("Running Electrokinetic LBM Simulator \n"); + printf("Running LBPM electrokinetic single-fluid solver \n"); printf("********************************************************\n"); } //PROFILE_ENABLE_TRACE(); @@ -53,18 +53,24 @@ int main(int argc, char **argv) // Load controller information Study.ReadParams(filename); - // Initialize LB Navier-Stokes model - StokesModel.ReadParams(filename,Study.num_iter_Stokes); + // Load user input database files for Navier-Stokes and Ion solvers + StokesModel.ReadParams(filename); + IonModel.ReadParams(filename); + + // Setup other model specific structures StokesModel.SetDomain(); StokesModel.ReadInput(); StokesModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables - StokesModel.Initialize(); // initializing the model will set initial conditions for variables - // Initialize LB-Ion model - IonModel.ReadParams(filename,Study.num_iter_Ion); IonModel.SetDomain(); IonModel.ReadInput(); IonModel.Create(); + + // Get internal iteration number + StokesModel.timestepMax = Study.getStokesNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv); + StokesModel.Initialize(); // initializing the model will set initial conditions for variables + + IonModel.timestepMax = Study.getIonNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv); IonModel.Initialize(); // Initialize LB-Poisson model @@ -74,39 +80,36 @@ int main(int argc, char **argv) PoissonSolver.Create(); PoissonSolver.Initialize(); + int timestep=0; while (timestep < Study.timestepMax){ timestep++; - //if (rank==0) printf("timestep=%i; running Poisson solver\n",timestep); PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental - //PoissonSolver.getElectricPotential(timestep); - - //if (rank==0) printf("timestep=%i; running StokesModel\n",timestep); StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity - //StokesModel.getVelocity(timestep); - - //if (rank==0) printf("timestep=%i; running Ion model\n",timestep); IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential - //IonModel.getIonConcentration(timestep); - timestep++;//AA operations - //-------------------------------------------- - //potentially leave analysis module for future - //-------------------------------------------- + + if (timestep%Study.visualization_interval==0){ + PoissonSolver.getElectricPotential(timestep); + PoissonSolver.getElectricField(timestep); + IonModel.getIonConcentration(timestep); + StokesModel.getVelocity(timestep); + } } - StokesModel.getVelocity(timestep); + if (rank==0) printf("Save simulation raw data at maximum timestep\n"); PoissonSolver.getElectricPotential(timestep); PoissonSolver.getElectricField(timestep); IonModel.getIonConcentration(timestep); + StokesModel.getVelocity(timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); PROFILE_STOP("Main"); - PROFILE_SAVE("lbpm_electrokinetic_simulator",1); + PROFILE_SAVE("lbpm_electrokinetic_SingleFluid_simulator",1); // **************************************************** MPI_Barrier(comm); } // Limit scope so variables that contain communicators will free before MPI_Finialize From f0c7882639efabf8017a0c727a5fa1657d697dae Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 21 Sep 2020 23:54:28 -0400 Subject: [PATCH 226/270] add higher-order term in Ion equilibrium distribution --- gpu/Ion.cu | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/gpu/Ion.cu b/gpu/Ion.cu index dc94d2e5..2a3a0225 100644 --- a/gpu/Ion.cu +++ b/gpu/Ion.cu @@ -147,25 +147,32 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, doub f6 = dist[nr6]; // q=0 - dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 1 - dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + //dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q=2 - dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + //dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 3 - dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + //dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 4 - dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + //dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 5 - dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + //dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 6 - dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + //dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); } } } @@ -206,25 +213,32 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *V f6 = dist[5*Np+n]; // q=0 - dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 1 - dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + //dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q=2 - dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + //dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 3 - dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + //dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 4 - dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + //dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 5 - dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + //dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 6 - dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + //dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); } } } From 0d6231f1cb927824a3bc2f152a3df68f3750918a Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 22 Sep 2020 14:33:51 -0400 Subject: [PATCH 227/270] read BC from model database --- models/ColorModel.cpp | 7 +++++-- models/DFHModel.cpp | 11 +++++++++-- models/GreyscaleModel.cpp | 5 ++++- models/MRTModel.cpp | 5 ++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a8c21a75..28d2fbe2 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1,4 +1,4 @@ -/* + /* color lattice boltzmann model */ #include "models/ColorModel.h" @@ -121,7 +121,10 @@ void ScaLBL_ColorModel::ReadParams(string filename){ //if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details) BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ + if (color_db->keyExists( "BC" )){ + BoundaryCondition = color_db->getScalar( "BC" ); + } + else if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } diff --git a/models/DFHModel.cpp b/models/DFHModel.cpp index 4eb03bea..ad346271 100644 --- a/models/DFHModel.cpp +++ b/models/DFHModel.cpp @@ -81,13 +81,18 @@ void ScaLBL_DFHModel::ReadParams(string filename){ outletA=0.f; outletB=1.f; - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) + BoundaryCondition = domain_db->getScalar( "BC" ); + if (color_db->keyExists( "BC" )){ + BoundaryCondition = color_db->getScalar( "BC" ); + } + else if (domain_db->keyExists( "BC" )){ + BoundaryCondition = domain_db->getScalar( "BC" ); + } // Read domain parameters auto L = domain_db->getVector( "L" ); auto size = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); - BoundaryCondition = domain_db->getScalar( "BC" ); Nx = size[0]; Ny = size[1]; Nz = size[2]; @@ -97,6 +102,8 @@ void ScaLBL_DFHModel::ReadParams(string filename){ nprocx = nproc[0]; nprocy = nproc[1]; nprocz = nproc[2]; + + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) } void ScaLBL_DFHModel::SetDomain(){ diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index c28c88c5..5cdae905 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -88,7 +88,10 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){ //------------------------ Other Domain parameters ------------------------// BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ + if (greyscale_db->keyExists( "BC" )){ + BoundaryCondition = greyscale_db->getScalar( "BC" ); + } + else if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } // ------------------------------------------------------------------------// diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index acfb8821..b16e1705 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -57,7 +57,10 @@ void ScaLBL_MRTModel::ReadParams(string filename){ } // Read domain parameters - if (domain_db->keyExists( "BC" )){ + if (mrt_db->keyExists( "BoundaryCondition" )){ + BoundaryCondition = mrt_db->getScalar( "BC" ); + } + else if (domain_db->keyExists( "BC" )){ BoundaryCondition = domain_db->getScalar( "BC" ); } From 11be7935758421bbe030c75692f41c41e4b54a44 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 23 Sep 2020 12:30:22 -0400 Subject: [PATCH 228/270] use periodic BC in ScaLBL (model specific BC possible) --- common/ScaLBL.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 8db59597..49c19054 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -77,7 +77,8 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ nprocx = Dm->nprocx(); nprocy = Dm->nprocy(); nprocz = Dm->nprocz(); - BoundaryCondition = Dm->BoundaryCondition; + //BoundaryCondition = Dm->BoundaryCondition; + BoundaryCondition = 0; // default to periodic BC //...................................................................................... ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory From fd27b3138a422be44aa190a0e5eaf48b8a82753f Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 23 Sep 2020 14:53:46 -0400 Subject: [PATCH 229/270] tweak BC conventions --- common/ScaLBL.cpp | 4 ++-- models/PoissonSolver.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 49c19054..8b0abeba 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -77,8 +77,8 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ nprocx = Dm->nprocx(); nprocy = Dm->nprocy(); nprocz = Dm->nprocz(); - //BoundaryCondition = Dm->BoundaryCondition; - BoundaryCondition = 0; // default to periodic BC + BoundaryCondition = Dm->BoundaryCondition; + //BoundaryCondition = 0; // default to periodic BC //...................................................................................... ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index b74536c5..40020b94 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -113,6 +113,8 @@ void ScaLBL_Poisson::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); + Dm->BoundaryCondition = BoundaryCondition; + Mask->BoundaryCondition = BoundaryCondition; Dm->CommInit(); MPI_Barrier(comm); From 85844ec8e3e09696817697b3118ec43b20bcb414 Mon Sep 17 00:00:00 2001 From: James McClure Date: Wed, 23 Sep 2020 15:21:15 -0400 Subject: [PATCH 230/270] template for read from file --- common/Domain.cpp | 5 +++++ common/Domain.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/common/Domain.cpp b/common/Domain.cpp index 7a1a6230..396a6004 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -1286,3 +1286,8 @@ void ReadBinaryFile(char *FILENAME, double *Data, size_t N) File.close(); } +void ReadFromFile(const std::string& Filename, DoubleArray &Mesh){ + + +} + diff --git a/common/Domain.h b/common/Domain.h index df2812c1..920601fb 100755 --- a/common/Domain.h +++ b/common/Domain.h @@ -244,6 +244,8 @@ private: }; +void ReadFromFile(const std::string& Filename, DoubleArray &Mesh); + void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np); void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, size_t Np); From 226147a49830ad5464b0688c59aae6254b2e50f4 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 23 Sep 2020 17:47:02 -0400 Subject: [PATCH 231/270] save the work of read from user-input file --- common/Domain.cpp | 156 +++++++++++++++++++++++++++++++++++++- common/Domain.h | 2 + models/GreyscaleModel.cpp | 130 ++++++++++++++++++++++++++++++- models/GreyscaleModel.h | 1 + 4 files changed, 286 insertions(+), 3 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 7a1a6230..2dd8a481 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -56,7 +56,7 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL) + id(NULL),UserData(NULL) { NULL_USE( rnk ); NULL_USE( npy ); @@ -107,7 +107,7 @@ Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL) + id(NULL),UserData(NULL) { MPI_Comm_dup(Communicator,&Comm); @@ -165,6 +165,8 @@ Domain::~Domain() delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; // Free id delete [] id; + // Free user-defined input data + delete [] UserData; // Free the communicator if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { MPI_Comm_free(&Comm); @@ -238,6 +240,7 @@ void Domain::initialize( std::shared_ptr db ) id = new signed char[N]; memset(id,0,N); + UserData = new double[N]; BoundaryCondition = d_db->getScalar("BC"); int nprocs; MPI_Comm_size( Comm, &nprocs ); @@ -675,6 +678,155 @@ void Domain::Decomp( const std::string& Filename ) //......................................................... } +void Domain::Decomp_RegularFile(const std::string& Filename,const std::string& Datatype) +{ + //........................................................................................ + // Reading the user-defined input file + // NOTE: so far it only supports BC=0 (periodic) and BC=5 (mixed reflection) + // because if checkerboard or inlet/outlet buffer layers are added, the + // value of the void space is undefined. + // NOTE: if BC=5 is used, where the inlet and outlet layers of the domain are modified, + // user needs to modify the input file accordingly before LBPM simulator read + // the input file. + //........................................................................................ + int rank_offset = 0; + int RANK = rank(); + int nprocs, nprocx, nprocy, nprocz, nx, ny, nz; + int64_t global_Nx,global_Ny,global_Nz; + int64_t i,j,k,n; + //TODO These offset we may still need them + //int64_t xStart,yStart,zStart; + //xStart=yStart=zStart=0; + + // Read domain parameters + // TODO currently the size of the data is still read from Domain{}; + // but user may have a user-specified size + auto size = database->getVector( "n" ); + auto SIZE = database->getVector( "N" ); + auto nproc = database->getVector( "nproc" ); + //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain + //if (database->keyExists( "offset" )){ + // auto offset = database->getVector( "offset" ); + // xStart = offset[0]; + // yStart = offset[1]; + // zStart = offset[2]; + //} + + nx = size[0]; + ny = size[1]; + nz = size[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + global_Nx = SIZE[0]; + global_Ny = SIZE[1]; + global_Nz = SIZE[2]; + nprocs=nprocx*nprocy*nprocz; + + auto ReadType = Datatype.c_str(); + double *SegData = NULL; + if (RANK==0){ + printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str()); + printf("NOTE: currently only BC=0 or 5 supports user-defined input file!\n"); + // Rank=0 reads the entire segmented data and distributes to worker processes + printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz); + int64_t SIZE = global_Nx*global_Ny*global_Nz; + + if (ReadType == "double"){ + printf("Reading input data as double precision floating number\n"); + SegData = new double[SIZE]; + FILE *SEGDAT = fopen(Filename.c_str(),"rb"); + if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + size_t ReadSeg; + ReadSeg=fread(SegData,8,SIZE,SEGDAT); + if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + fclose(SEGDAT); + } + else{ + ERROR("Error: User-defined input file only supports double-precision floating number!\n"); + } + printf("Read file successfully from %s \n",Filename.c_str()); + } + + // Get the rank info + int64_t N = (nx+2)*(ny+2)*(nz+2); + + // number of sites to use for periodic boundary condition transition zone + //int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2; + //if (z_transition_size < 0) z_transition_size=0; + int64_t z_transition_size = 0; + + char LocalRankFilename[40];//just for debug + double *loc_id; + loc_id = new double [(nx+2)*(ny+2)*(nz+2)]; + + // Set up the sub-domains + if (RANK==0){ + printf("Decomposing user-defined input file\n"); + printf("Distributing subdomains across %i processors \n",nprocs); + printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz); + printf("Subdomain size: %i x %i x %i \n",nx,ny,nz); + printf("Size of transition region: %ld \n", z_transition_size); + + for (int kp=0; kp &File_poro,const vector &File_perm) +{ + size_t NLABELS=0; + signed char VALUE=0; + double POROSITY=0.f; + double PERMEABILITY=0.f; + + if (rank==0){ + printf("Input voxel porosity map: %s\n",Filename_poro.c_str()); + printf("Input voxel permeability map: %s\n",Filename_perm.c_str()); + printf("Relabeling %lu values\n",ReadValues.size()); + for (size_t idx=0; idxgetVector( "ComponentLabels" ); + auto PorosityList = greyscale_db->getVector( "PorosityList" ); + auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); + + NLABELS=LabelList.size(); + if (NLABELS != PorosityList.size()){ + ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); + } + + double label_count[NLABELS]; + double label_count_global[NLABELS]; + // Assign the labels + + for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (POROSITY<=0.0){ + ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); + } + else{ + Porosity[idx] = POROSITY; + } + } + } + } + } + + if (NLABELS != PermeabilityList.size()){ + ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); + } + for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component + } + } + int idx = Map(i,j,k); + if (!(idx < 0)){ + if (PERMEABILITY<=0.0){ + ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); + } + else{ + Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + } + } + } + } + } + + + // Set Dm to match Mask + for (int i=0; iid[i] = Mask->id[i]; + + for (int idx=0; idxComm, label_count[idx]); + //Initialize a weighted porosity after considering grey voxels + GreyPorosity=0.0; + for (unsigned int idx=0; idxvoxel_length); + printf("Number of component labels: %lu \n",NLABELS); + for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); + printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); + } + printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); + } +} void ScaLBL_GreyscaleModel::Create(){ /* @@ -341,7 +457,19 @@ void ScaLBL_GreyscaleModel::Create(){ double *Poros, *Perm; Poros = new double[Np]; Perm = new double[Np]; - AssignComponentLabels(Poros,Perm); + if (greyscale_db->keyExists("FileVoxelPorosityMap")){ + //NOTE: FileVoxel**Map is a vector, including "file_name, datatype" + auto File_poro = domain_db->getVector( "FileVoxelPorosityMap" ); + auto File_perm = domain_db->getVector( "FileVoxelPermeabilityMap" ); + AssignComponentLabels(Poros,Perm,File_poro,File_perm); + } + else if (greyscale_db->keyExists("PorosityList")){ + //initialize voxel porosity and perm from the input list + AssignComponentLabels(Poros,Perm); + } + else { + ERROR("Error: PorosityList or FilenameVoxelPorosityMap cannot be found! \n"); + } ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double)); ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double)); delete [] Poros; diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 151d49d6..0750b3ba 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -86,6 +86,7 @@ private: char LocalRestartFile[40]; void AssignComponentLabels(double *Porosity, double *Permeablity); + void AssignComponentLabels(double *Porosity, double *Permeablity,const std::string& Filename_poro,const std::string& Filename_perm); }; From 471f71d6906eebca6a20dee6f4259a1602380805 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Wed, 23 Sep 2020 22:15:05 -0400 Subject: [PATCH 232/270] save the work; untested; add a routine of read from file for greyscale simulator --- common/Domain.cpp | 306 +++++++++++++++++++------------------- common/Domain.h | 5 +- models/GreyscaleModel.cpp | 107 +++---------- models/GreyscaleModel.h | 3 +- 4 files changed, 172 insertions(+), 249 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index d53ea6f0..6e4e757d 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -56,7 +56,7 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL),UserData(NULL) + id(NULL) { NULL_USE( rnk ); NULL_USE( npy ); @@ -107,7 +107,7 @@ Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL),UserData(NULL) + id(NULL) { MPI_Comm_dup(Communicator,&Comm); @@ -165,8 +165,7 @@ Domain::~Domain() delete [] recvData_yZ; delete [] recvData_Yz; delete [] recvData_YZ; // Free id delete [] id; - // Free user-defined input data - delete [] UserData; + // Free the communicator if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { MPI_Comm_free(&Comm); @@ -240,7 +239,6 @@ void Domain::initialize( std::shared_ptr db ) id = new signed char[N]; memset(id,0,N); - UserData = new double[N]; BoundaryCondition = d_db->getScalar("BC"); int nprocs; MPI_Comm_size( Comm, &nprocs ); @@ -678,154 +676,6 @@ void Domain::Decomp( const std::string& Filename ) //......................................................... } -void Domain::Decomp_RegularFile(const std::string& Filename,const std::string& Datatype) -{ - //........................................................................................ - // Reading the user-defined input file - // NOTE: so far it only supports BC=0 (periodic) and BC=5 (mixed reflection) - // because if checkerboard or inlet/outlet buffer layers are added, the - // value of the void space is undefined. - // NOTE: if BC=5 is used, where the inlet and outlet layers of the domain are modified, - // user needs to modify the input file accordingly before LBPM simulator read - // the input file. - //........................................................................................ - int rank_offset = 0; - int RANK = rank(); - int nprocs, nprocx, nprocy, nprocz, nx, ny, nz; - int64_t global_Nx,global_Ny,global_Nz; - int64_t i,j,k,n; - //TODO These offset we may still need them - //int64_t xStart,yStart,zStart; - //xStart=yStart=zStart=0; - - // Read domain parameters - // TODO currently the size of the data is still read from Domain{}; - // but user may have a user-specified size - auto size = database->getVector( "n" ); - auto SIZE = database->getVector( "N" ); - auto nproc = database->getVector( "nproc" ); - //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain - //if (database->keyExists( "offset" )){ - // auto offset = database->getVector( "offset" ); - // xStart = offset[0]; - // yStart = offset[1]; - // zStart = offset[2]; - //} - - nx = size[0]; - ny = size[1]; - nz = size[2]; - nprocx = nproc[0]; - nprocy = nproc[1]; - nprocz = nproc[2]; - global_Nx = SIZE[0]; - global_Ny = SIZE[1]; - global_Nz = SIZE[2]; - nprocs=nprocx*nprocy*nprocz; - - auto ReadType = Datatype.c_str(); - double *SegData = NULL; - if (RANK==0){ - printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str()); - printf("NOTE: currently only BC=0 or 5 supports user-defined input file!\n"); - // Rank=0 reads the entire segmented data and distributes to worker processes - printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz); - int64_t SIZE = global_Nx*global_Ny*global_Nz; - - if (ReadType == "double"){ - printf("Reading input data as double precision floating number\n"); - SegData = new double[SIZE]; - FILE *SEGDAT = fopen(Filename.c_str(),"rb"); - if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading file: %s\n",Filename.c_str()); - size_t ReadSeg; - ReadSeg=fread(SegData,8,SIZE,SEGDAT); - if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str()); - fclose(SEGDAT); - } - else{ - ERROR("Error: User-defined input file only supports double-precision floating number!\n"); - } - printf("Read file successfully from %s \n",Filename.c_str()); - } - - // Get the rank info - int64_t N = (nx+2)*(ny+2)*(nz+2); - - // number of sites to use for periodic boundary condition transition zone - //int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2; - //if (z_transition_size < 0) z_transition_size=0; - int64_t z_transition_size = 0; - - char LocalRankFilename[40];//just for debug - double *loc_id; - loc_id = new double [(nx+2)*(ny+2)*(nz+2)]; - - // Set up the sub-domains - if (RANK==0){ - printf("Decomposing user-defined input file\n"); - printf("Distributing subdomains across %i processors \n",nprocs); - printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz); - printf("Subdomain size: %i x %i x %i \n",nx,ny,nz); - printf("Size of transition region: %ld \n", z_transition_size); - - for (int kp=0; kpgetVector( "n" ); + auto SIZE = database->getVector( "N" ); + auto nproc = database->getVector( "nproc" ); + //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain + //if (database->keyExists( "offset" )){ + // auto offset = database->getVector( "offset" ); + // xStart = offset[0]; + // yStart = offset[1]; + // zStart = offset[2]; + //} + nx = size[0]; + ny = size[1]; + nz = size[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + global_Nx = SIZE[0]; + global_Ny = SIZE[1]; + global_Nz = SIZE[2]; + nprocs=nprocx*nprocy*nprocz; + auto ReadType = Datatype.c_str(); + double *SegData = NULL; + if (RANK==0){ + printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str()); + printf("NOTE: currently only BC=0 or 5 supports user-defined input file!\n"); + // Rank=0 reads the entire segmented data and distributes to worker processes + printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz); + int64_t SIZE = global_Nx*global_Ny*global_Nz; + + if (ReadType == "double"){ + printf("Reading input data as double precision floating number\n"); + SegData = new double[SIZE]; + FILE *SEGDAT = fopen(Filename.c_str(),"rb"); + if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + size_t ReadSeg; + ReadSeg=fread(SegData,8,SIZE,SEGDAT); + if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + fclose(SEGDAT); + } + else{ + ERROR("Error: User-defined input file only supports double-precision floating number!\n"); + } + printf("Read file successfully from %s \n",Filename.c_str()); + } + + // Get the rank info + int64_t N = (nx+2)*(ny+2)*(nz+2); + + // number of sites to use for periodic boundary condition transition zone + //int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2; + //if (z_transition_size < 0) z_transition_size=0; + int64_t z_transition_size = 0; + + char LocalRankFilename[40];//just for debug + double *loc_id; + loc_id = new double [(nx+2)*(ny+2)*(nz+2)]; + + // Set up the sub-domains + if (RANK==0){ + printf("Decomposing user-defined input file\n"); + printf("Distributing subdomains across %i processors \n",nprocs); + printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz); + printf("Subdomain size: %i x %i x %i \n",nx,ny,nz); + printf("Size of transition region: %ld \n", z_transition_size); + + for (int kp=0; kp &File_poro,const vector &File_perm) { - size_t NLABELS=0; - signed char VALUE=0; + double *Porosity_host, Permeability_host; + Porosity_host = new double[N]; + Permeability_host = new double[N]; double POROSITY=0.f; double PERMEABILITY=0.f; + //Initialize a weighted porosity after considering grey voxels + double GreyPorosity=0.0; + //double label_count_loc = 0.0; + //double label_count_glb = 0.0; - if (rank==0){ - printf("Input voxel porosity map: %s\n",Filename_poro.c_str()); - printf("Input voxel permeability map: %s\n",Filename_perm.c_str()); - printf("Relabeling %lu values\n",ReadValues.size()); - for (size_t idx=0; idxgetVector( "ComponentLabels" ); - auto PorosityList = greyscale_db->getVector( "PorosityList" ); - auto PermeabilityList = greyscale_db->getVector( "PermeabilityList" ); - - NLABELS=LabelList.size(); - if (NLABELS != PorosityList.size()){ - ERROR("Error: ComponentLabels and PorosityList must be the same length! \n"); - } - - double label_count[NLABELS]; - double label_count_global[NLABELS]; - // Assign the labels - - for (int idx=0; idxid[n] = 0; // set mask to zero since this is an immobile component - } - } int idx = Map(i,j,k); if (!(idx < 0)){ + int n = k*Nx*Ny+j*Nx+i; + POROSITY = Porosity_host[n]; + PERMEABILITY = Permeability_host[n]; if (POROSITY<=0.0){ ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n"); } - else{ - Porosity[idx] = POROSITY; - } - } - } - } - } - - if (NLABELS != PermeabilityList.size()){ - ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n"); - } - for (int k=0;kid[n] = 0; // set mask to zero since this is an immobile component - } - } - int idx = Map(i,j,k); - if (!(idx < 0)){ - if (PERMEABILITY<=0.0){ + else if (PERMEABILITY<=0.0){ ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n"); } else{ - Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length; + Porosity[idx] = POROSITY; + Permeability[idx] = PERMEABILITY; + GreyPorosity += POROSITY; + //label_count_loc += 1.0; } } } } } - - - // Set Dm to match Mask - for (int i=0; iid[i] = Mask->id[i]; - - for (int idx=0; idxComm, label_count[idx]); - //Initialize a weighted porosity after considering grey voxels - GreyPorosity=0.0; - for (unsigned int idx=0; idxComm, label_count_loc); + GreyPorosity = GreyPorosity/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs); if (rank==0){ printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); - printf("Number of component labels: %lu \n",NLABELS); - for (unsigned int idx=0; idxvoxel_length/Dm->voxel_length,volume_fraction); - printf(" effective porosity=%.3g\n",volume_fraction*POROSITY); - } printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); } } diff --git a/models/GreyscaleModel.h b/models/GreyscaleModel.h index 0750b3ba..6fe1a108 100644 --- a/models/GreyscaleModel.h +++ b/models/GreyscaleModel.h @@ -86,7 +86,6 @@ private: char LocalRestartFile[40]; void AssignComponentLabels(double *Porosity, double *Permeablity); - void AssignComponentLabels(double *Porosity, double *Permeablity,const std::string& Filename_poro,const std::string& Filename_perm); - + void AssignComponentLabels(double *Porosity,double *Permeability,const vector &File_poro,const vector &File_perm); }; From 33fdfeb29d3e041d9ec41f9d77b7aba4ee4bf12e Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 24 Sep 2020 21:57:04 -0400 Subject: [PATCH 233/270] add a routine for single-fluid greyscale to read from file - voxel porosity and perm --- common/Domain.cpp | 33 ++++++++++++++++----------------- common/Domain.h | 3 ++- models/GreyscaleModel.cpp | 19 +++++++++++-------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 6e4e757d..5137ddd1 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -1289,7 +1289,7 @@ void ReadBinaryFile(char *FILENAME, double *Data, size_t N) } -void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData) +void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData) { //........................................................................................ // Reading the user-defined input file @@ -1306,8 +1306,8 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl int64_t global_Nx,global_Ny,global_Nz; int64_t i,j,k,n; //TODO These offset we may still need them - //int64_t xStart,yStart,zStart; - //xStart=yStart=zStart=0; + int64_t xStart,yStart,zStart; + xStart=yStart=zStart=0; // Read domain parameters // TODO currently the size of the data is still read from Domain{}; @@ -1316,12 +1316,12 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl auto SIZE = database->getVector( "N" ); auto nproc = database->getVector( "nproc" ); //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain - //if (database->keyExists( "offset" )){ - // auto offset = database->getVector( "offset" ); - // xStart = offset[0]; - // yStart = offset[1]; - // zStart = offset[2]; - //} + if (database->keyExists( "offset" )){ + auto offset = database->getVector( "offset" ); + xStart = offset[0]; + yStart = offset[1]; + zStart = offset[2]; + } nx = size[0]; ny = size[1]; @@ -1334,7 +1334,6 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl global_Nz = SIZE[2]; nprocs=nprocx*nprocy*nprocz; - auto ReadType = Datatype.c_str(); double *SegData = NULL; if (RANK==0){ printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str()); @@ -1343,11 +1342,11 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz); int64_t SIZE = global_Nx*global_Ny*global_Nz; - if (ReadType == "double"){ + if (Datatype == "double"){ printf("Reading input data as double precision floating number\n"); SegData = new double[SIZE]; FILE *SEGDAT = fopen(Filename.c_str(),"rb"); - if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading user-defined file!\n"); size_t ReadSeg; ReadSeg=fread(SegData,8,SIZE,SEGDAT); if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str()); @@ -1367,7 +1366,7 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl //if (z_transition_size < 0) z_transition_size=0; int64_t z_transition_size = 0; - char LocalRankFilename[40];//just for debug + //char LocalRankFilename[1000];//just for debug double *loc_id; loc_id = new double [(nx+2)*(ny+2)*(nz+2)]; @@ -1420,10 +1419,10 @@ void ReadFromFile(const std::string& Filename,const std::string& Datatype, doubl } // Write the data for this rank data // NOTE just for debug - sprintf(LocalRankFilename,"%s.%05i",Filename.c_str(),rnk+rank_offset); - FILE *ID = fopen(LocalRankFilename,"wb"); - fwrite(loc_id,1,(nx+2)*(ny+2)*(nz+2),ID); - fclose(ID); + //sprintf(LocalRankFilename,"%s.%05i",Filename.c_str(),rnk+rank_offset); + //FILE *ID = fopen(LocalRankFilename,"wb"); + //fwrite(loc_id,8,(nx+2)*(ny+2)*(nz+2),ID); + //fclose(ID); } } } diff --git a/common/Domain.h b/common/Domain.h index 2436ab42..e3b658f8 100755 --- a/common/Domain.h +++ b/common/Domain.h @@ -178,6 +178,7 @@ public: // Public variables (need to create accessors instead) void ReadIDs(); void Decomp( const std::string& filename ); + void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData); void CommunicateMeshHalo(DoubleArray &Mesh); void CommInit(); int PoreCount(); @@ -244,7 +245,7 @@ private: }; -void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData); +//void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData); //void ReadFromFile(const std::string& Filename, DoubleArray &Mesh); void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np); diff --git a/models/GreyscaleModel.cpp b/models/GreyscaleModel.cpp index 8bc62f52..69d41711 100644 --- a/models/GreyscaleModel.cpp +++ b/models/GreyscaleModel.cpp @@ -290,18 +290,19 @@ void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Perm void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity,double *Permeability,const vector &File_poro,const vector &File_perm) { - double *Porosity_host, Permeability_host; + double *Porosity_host, *Permeability_host; Porosity_host = new double[N]; Permeability_host = new double[N]; double POROSITY=0.f; double PERMEABILITY=0.f; //Initialize a weighted porosity after considering grey voxels - double GreyPorosity=0.0; + double GreyPorosity_loc=0.0; + GreyPorosity=0.0; //double label_count_loc = 0.0; //double label_count_glb = 0.0; - ReadFromFile(File_poro[0],File_poro[1],Porosity_host); - ReadFromFile(File_perm[0],File_perm[1],Permeability_host); + Mask->ReadFromFile(File_poro[0],File_poro[1],Porosity_host); + Mask->ReadFromFile(File_perm[0],File_perm[1],Permeability_host); for (int k=0;kComm, label_count_loc); + GreyPorosity = sumReduce( Dm->Comm, GreyPorosity_loc); GreyPorosity = GreyPorosity/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs); if (rank==0){ printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length); printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity); } + delete [] Porosity_host; + delete [] Permeability_host; } void ScaLBL_GreyscaleModel::Create(){ @@ -390,8 +393,8 @@ void ScaLBL_GreyscaleModel::Create(){ Perm = new double[Np]; if (greyscale_db->keyExists("FileVoxelPorosityMap")){ //NOTE: FileVoxel**Map is a vector, including "file_name, datatype" - auto File_poro = domain_db->getVector( "FileVoxelPorosityMap" ); - auto File_perm = domain_db->getVector( "FileVoxelPermeabilityMap" ); + auto File_poro = greyscale_db->getVector( "FileVoxelPorosityMap" ); + auto File_perm = greyscale_db->getVector( "FileVoxelPermeabilityMap" ); AssignComponentLabels(Poros,Perm,File_poro,File_perm); } else if (greyscale_db->keyExists("PorosityList")){ From e529caf6feb1d7c463147c050a42f1023d8d4943 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 25 Sep 2020 17:02:16 -0400 Subject: [PATCH 234/270] finish BC tweak --- models/IonModel.cpp | 2 ++ models/PoissonSolver.cpp | 2 -- models/StokesModel.cpp | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 0a8f779a..7366c0b4 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -418,6 +418,8 @@ void ScaLBL_IonModel::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); + Dm->BoundaryCondition = BoundaryCondition; + Mask->BoundaryCondition = BoundaryCondition; Dm->CommInit(); MPI_Barrier(comm); diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 40020b94..fccb1feb 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -416,14 +416,12 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ timestep++; SolveElectricPotentialAAodd();//update electric potential - //SolveElectricField(); //deprecated - compute electric field SolvePoissonAAodd(ChargeDensity);//perform collision ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP*************// timestep++; SolveElectricPotentialAAeven();//update electric potential - //SolveElectricField();//deprecated - compute electric field SolvePoissonAAeven(ChargeDensity);//perform collision ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************/ diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 3b0b2d3a..09528567 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -41,10 +41,6 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ //--------------------------------------------------------------------------// // Read domain parameters - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } @@ -53,6 +49,10 @@ void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){ //if (stokes_db->keyExists( "timestepMax" )){ // timestepMax = stokes_db->getScalar( "timestepMax" ); //} + BoundaryCondition = 0; + if (stokes_db->keyExists( "BC" )){ + BoundaryCondition = stokes_db->getScalar( "BC" ); + } if (stokes_db->keyExists( "tolerance" )){ tolerance = stokes_db->getScalar( "tolerance" ); } @@ -117,10 +117,6 @@ void ScaLBL_StokesModel::ReadParams(string filename){ //--------------------------------------------------------------------------// // Read domain parameters - BoundaryCondition = 0; - if (domain_db->keyExists( "BC" )){ - BoundaryCondition = domain_db->getScalar( "BC" ); - } if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu h = domain_db->getScalar( "voxel_length" ); } @@ -129,6 +125,10 @@ void ScaLBL_StokesModel::ReadParams(string filename){ //if (stokes_db->keyExists( "timestepMax" )){ // timestepMax = stokes_db->getScalar( "timestepMax" ); //} + BoundaryCondition = 0; + if (stokes_db->keyExists( "BC" )){ + BoundaryCondition = stokes_db->getScalar( "BC" ); + } if (stokes_db->keyExists( "tolerance" )){ tolerance = stokes_db->getScalar( "tolerance" ); } @@ -190,6 +190,8 @@ void ScaLBL_StokesModel::SetDomain(){ for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); + Dm->BoundaryCondition = BoundaryCondition; + Mask->BoundaryCondition = BoundaryCondition; Dm->CommInit(); MPI_Barrier(comm); From 4c1ce54a6dcdaba776f554dab19c5d7a88057445 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 25 Sep 2020 17:09:25 -0400 Subject: [PATCH 235/270] found that higher-order terms in Ion BGK not very useful --- gpu/Ion.cu | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/gpu/Ion.cu b/gpu/Ion.cu index 2a3a0225..877e5591 100644 --- a/gpu/Ion.cu +++ b/gpu/Ion.cu @@ -147,32 +147,32 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, doub f6 = dist[nr6]; // q=0 - //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; - dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 1 - //dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); - dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + //dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q=2 - //dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); - dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + //dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 3 - //dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); - dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + //dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 4 - //dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); - dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + //dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 5 - //dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); - dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + //dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 6 - //dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); - dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + //dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); } } } @@ -213,32 +213,32 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *V f6 = dist[5*Np+n]; // q=0 - //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; - dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci; + //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 1 - //dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); - dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)); + //dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q=2 - //dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); - dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)); + //dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 3 - //dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); - dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)); + //dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 4 - //dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); - dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)); + //dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 5 - //dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); - dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)); + //dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); // q = 6 - //dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); - dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); + dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)); + //dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz))); } } } From 4657adbc94675b410b60069f6471bb06df81e92e Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 28 Sep 2020 17:08:49 -0400 Subject: [PATCH 236/270] add routine for ion model to read from file --- common/Domain.cpp | 147 ++++++++++++++++++++++++++++++++++++++++++++ common/Domain.h | 1 + common/ScaLBL.h | 1 + cpu/Ion.cpp | 16 +++++ gpu/Ion.cu | 33 ++++++++++ models/IonModel.cpp | 55 +++++++++++++++-- models/IonModel.h | 1 + 7 files changed, 249 insertions(+), 5 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index d75ec4a5..fc797a8e 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -1286,3 +1286,150 @@ void ReadBinaryFile(char *FILENAME, double *Data, size_t N) File.close(); } +void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData) +{ + //........................................................................................ + // Reading the user-defined input file + // NOTE: so far it only supports BC=0 (periodic) and BC=5 (mixed reflection) + // because if checkerboard or inlet/outlet buffer layers are added, the + // value of the void space is undefined. + // NOTE: if BC=5 is used, where the inlet and outlet layers of the domain are modified, + // user needs to modify the input file accordingly before LBPM simulator read + // the input file. + //........................................................................................ + int rank_offset = 0; + int RANK = rank(); + int nprocs, nprocx, nprocy, nprocz, nx, ny, nz; + int64_t global_Nx,global_Ny,global_Nz; + int64_t i,j,k,n; + //TODO These offset we may still need them + int64_t xStart,yStart,zStart; + xStart=yStart=zStart=0; + + // Read domain parameters + // TODO currently the size of the data is still read from Domain{}; + // but user may have a user-specified size + auto size = database->getVector( "n" ); + auto SIZE = database->getVector( "N" ); + auto nproc = database->getVector( "nproc" ); + //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain + if (database->keyExists( "offset" )){ + auto offset = database->getVector( "offset" ); + xStart = offset[0]; + yStart = offset[1]; + zStart = offset[2]; + } + + nx = size[0]; + ny = size[1]; + nz = size[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + global_Nx = SIZE[0]; + global_Ny = SIZE[1]; + global_Nz = SIZE[2]; + nprocs=nprocx*nprocy*nprocz; + + double *SegData = NULL; + if (RANK==0){ + printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str()); + printf("NOTE: currently only BC=0 or 5 supports user-defined input file!\n"); + // Rank=0 reads the entire segmented data and distributes to worker processes + printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz); + int64_t SIZE = global_Nx*global_Ny*global_Nz; + + if (Datatype == "double"){ + printf("Reading input data as double precision floating number\n"); + SegData = new double[SIZE]; + FILE *SEGDAT = fopen(Filename.c_str(),"rb"); + if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading user-defined file!\n"); + size_t ReadSeg; + ReadSeg=fread(SegData,8,SIZE,SEGDAT); + if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str()); + fclose(SEGDAT); + } + else{ + ERROR("Error: User-defined input file only supports double-precision floating number!\n"); + } + printf("Read file successfully from %s \n",Filename.c_str()); + } + + // Get the rank info + int64_t N = (nx+2)*(ny+2)*(nz+2); + + // number of sites to use for periodic boundary condition transition zone + //int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2; + //if (z_transition_size < 0) z_transition_size=0; + int64_t z_transition_size = 0; + + //char LocalRankFilename[1000];//just for debug + double *loc_id; + loc_id = new double [(nx+2)*(ny+2)*(nz+2)]; + + // Set up the sub-domains + if (RANK==0){ + printf("Decomposing user-defined input file\n"); + printf("Distributing subdomains across %i processors \n",nprocs); + printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz); + printf("Subdomain size: %i x %i x %i \n",nx,ny,nz); + printf("Size of transition region: %ld \n", z_transition_size); + + for (int kp=0; kp>>(dist,Den,Np); + + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",cudaGetErrorString(err)); + } + //cudaProfilerStop(); +} + extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){ //cudaProfilerStart(); diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 7366c0b4..ac53d5cc 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -136,7 +136,7 @@ void ScaLBL_IonModel::ReadParams(string filename,vector &num_iter){ } //read initial ion concentration list; INPUT unit [mol/m^3] //it must be converted to LB unit [mol/lu^3] - if (ion_db->keyExists("IonConcentrationList")){ + if (ion_db->keyExists("IonConcentrationList")){ IonConcentration.clear(); IonConcentration = ion_db->getVector( "IonConcentrationList" ); if (IonConcentration.size()!=number_ion_species){ @@ -544,6 +544,35 @@ void ScaLBL_IonModel::AssignSolidBoundary(double *ion_solid) } } +void ScaLBL_IonModel::AssignIonConcentration_FromFile(double *Ci,const vector &File_ion) +{ + double *Ci_host; + Ci_host = new double[N]; + double VALUE=0.f; + + Mask->ReadFromFile(File_ion[0],File_ion[1],Ci_host); + + for (int k=0;kkeyExists("IonConcentrationFile")){ + //TODO: Need to figure out how to deal with multi-species concentration initialization + //NOTE: "IonConcentrationFile" is a vector, including "file_name, datatype" + auto File_ion = ion_db->getVector( "IonConcentrationFile" ); + double *Ci_host; + Ci_host = new double[number_ion_species*Np]; + for (int ic=0; ic db0); void AssignSolidBoundary(double *ion_solid); + void AssignIonConcentration_FromFile(double *Ci,const vector &File_ion); void IonConcentration_LB_to_Phys(DoubleArray &Den_reg); }; From 471e78703a54846bc62eef740bbc07fe62c3594d Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Tue, 29 Sep 2020 13:22:25 -0400 Subject: [PATCH 237/270] add some print-out for debugging --- models/IonModel.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index ac53d5cc..5cf38e64 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -697,6 +697,12 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ for (double item : rlx){ item = 1.0/item; } + //**debug + if (rank==0){ + for (unsigned int ic=0;ic Date: Tue, 29 Sep 2020 15:48:39 -0400 Subject: [PATCH 238/270] fix dumb bug of asssigning inverse of tau --- models/IonModel.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 5cf38e64..bc58a649 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -693,15 +693,9 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ //2. ElectricField is from Poisson model //LB-related parameter - vector rlx(tau.begin(),tau.end()); - for (double item : rlx){ - item = 1.0/item; - } - //**debug - if (rank==0){ - for (unsigned int ic=0;ic rlx; + for (unsigned int ic=0;ic Date: Thu, 1 Oct 2020 16:27:46 -0400 Subject: [PATCH 239/270] save the work;upgrade output data writing by writing single file instead of decomposed data --- common/Domain.cpp | 91 +++++++++++++++++++++++++++++++++++++++++++++ common/Domain.h | 1 + models/IonModel.cpp | 13 ++++--- models/IonModel.h | 1 + 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index fc797a8e..9cf9a1ca 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -1433,3 +1433,94 @@ void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatyp //Comm.barrier(); MPI_Barrier(Comm); } + +void Domain::AggregateLabels( const std::string& filename, DoubleArray &UserData ){ + + int nx = Nx; + int ny = Ny; + int nz = Nz; + + int npx = nprocx(); + int npy = nprocy(); + int npz = nprocz(); + + int ipx = iproc(); + int ipy = jproc(); + int ipz = kproc(); + + int nprocs = nprocx()*nprocy()*nprocz(); + + int full_nx = npx*(nx-2); + int full_ny = npy*(ny-2); + int full_nz = npz*(nz-2); + int local_size = (nx-2)*(ny-2)*(nz-2); + unsigned long int full_size = long(full_nx)*long(full_ny)*long(full_nz); + + double *LocalID; + LocalID = new double [local_size]; + + //printf("aggregate labels: local size=%i, global size = %i",local_size, full_size); + // assign the ID for the local sub-region + for (int k=1; kAggregateLabels(OutputFilename,PhaseField); } } diff --git a/models/IonModel.h b/models/IonModel.h index 995898bd..0f527c3a 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -85,6 +85,7 @@ private: char LocalRankString[8]; char LocalRankFilename[40]; char LocalRestartFile[40]; + char OutputFilename[200]; //int rank,nprocs; void LoadParams(std::shared_ptr db0); From d40de38c488da9c1f37f2cbed9e0cc5fab05edae Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 5 Oct 2020 11:03:35 -0400 Subject: [PATCH 240/270] save the work;results needs to be validated --- models/IonModel.cpp | 23 +++++++++++------ models/IonModel.h | 1 + models/PoissonSolver.cpp | 54 ++++++++++++++++++++++++++++++++-------- models/PoissonSolver.h | 3 +++ models/StokesModel.cpp | 26 +++++++++++++++++++ models/StokesModel.h | 2 ++ 6 files changed, 92 insertions(+), 17 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 0d977416..24c4d8bd 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -787,23 +787,32 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ } void ScaLBL_IonModel::getIonConcentration(int timestep){ - + //This function wirte out the data in a normal layout (by aggregating all decomposed domains) DoubleArray PhaseField(Nx,Ny,Nz); for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); IonConcentration_LB_to_Phys(PhaseField); - //FILE *OUTFILE; - //sprintf(LocalRankFilename,"Ion%02i_Time_%i.%05i.raw",ic+1,timestep,rank); - //OUTFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,OUTFILE); - //fclose(OUTFILE); - sprintf(OutputFilename,"Ion%02i_Time_%i.raw",ic+1,timestep); Mask->AggregateLabels(OutputFilename,PhaseField); } +} +void ScaLBL_IonModel::getIonConcentration_debug(int timestep){ + //This function write out decomposed data + DoubleArray PhaseField(Nx,Ny,Nz); + for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + IonConcentration_LB_to_Phys(PhaseField); + + FILE *OUTFILE; + sprintf(LocalRankFilename,"Ion%02i_Time_%i.%05i.raw",ic+1,timestep,rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + } } void ScaLBL_IonModel::IonConcentration_LB_to_Phys(DoubleArray &Den_reg){ diff --git a/models/IonModel.h b/models/IonModel.h index 0f527c3a..5a568182 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -31,6 +31,7 @@ public: void Initialize(); void Run(double *Velocity, double *ElectricField); void getIonConcentration(int timestep); + void getIonConcentration_debug(int timestep); void DummyFluidVelocity(); void DummyElectricField(); double CalIonDenConvergence(vector &ci_avg_previous); diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index fccb1feb..3e11de0a 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -544,21 +544,55 @@ void ScaLBL_Poisson::DummyChargeDensity(){ delete [] ChargeDensity_host; } -void ScaLBL_Poisson::getElectricPotential(int timestep){ +void ScaLBL_Poisson::getElectricPotential_debug(int timestep){ + //This function write out decomposed data + DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); + ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz); + //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + FILE *OUTFILE; + sprintf(LocalRankFilename,"Electric_Potential_Time_%i.%05i.raw",timestep,rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); +} - DoubleArray PhaseField(Nx,Ny,Nz); - //ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); - ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz); - //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - FILE *OUTFILE; - sprintf(LocalRankFilename,"Electric_Potential_Time_%i.%05i.raw",timestep,rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,OUTFILE); - fclose(OUTFILE); +void ScaLBL_Poisson::getElectricPotential(int timestep){ + //This function wirte out the data in a normal layout (by aggregating all decomposed domains) + DoubleArray PhaseField(Nx,Ny,Nz); + //ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); + ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + sprintf(OutputFilename,"Electric_Potential_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); } void ScaLBL_Poisson::getElectricField(int timestep){ + DoubleArray PhaseField(Nx,Ny,Nz); + + ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"ElectricField_X_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); + + ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"ElectricField_Y_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); + + ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],PhaseField); + ElectricField_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"ElectricField_Z_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); +} + +void ScaLBL_Poisson::getElectricField_debug(int timestep){ + //ScaLBL_D3Q7_Poisson_getElectricField(fq,ElectricField,tau,Np); //ScaLBL_DeviceBarrier(); MPI_Barrier(comm); diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 921963ed..dfd098d5 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -29,7 +29,9 @@ public: void Initialize(); void Run(double *ChargeDensity); void getElectricPotential(int timestep); + void getElectricPotential_debug(int timestep); void getElectricField(int timestep); + void getElectricField_debug(int timestep); void DummyChargeDensity();//for debugging //bool Restart,pBC; @@ -76,6 +78,7 @@ private: char LocalRankString[8]; char LocalRankFilename[40]; char LocalRestartFile[40]; + char OutputFilename[200]; //int rank,nprocs; void LoadParams(std::shared_ptr db0); diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 09528567..964baaae 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -375,6 +375,32 @@ void ScaLBL_StokesModel::getVelocity(int timestep){ ScaLBL_D3Q19_Momentum(fq, Velocity, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + DoubleArray PhaseField(Nx,Ny,Nz); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); + Velocity_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"Velocity_X_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); + Velocity_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"Velocity_Y_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); + + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); + Velocity_LB_to_Phys(PhaseField); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + sprintf(OutputFilename,"Velocity_Z_Time_%i.raw",timestep); + Mask->AggregateLabels(OutputFilename,PhaseField); +} + +void ScaLBL_StokesModel::getVelocity_debug(int timestep){ + //get velocity in physical unit [m/sec] + ScaLBL_D3Q19_Momentum(fq, Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + DoubleArray PhaseField(Nx,Ny,Nz); ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); Velocity_LB_to_Phys(PhaseField); diff --git a/models/StokesModel.h b/models/StokesModel.h index 346d75c3..b7ad345e 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -32,6 +32,7 @@ public: void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); void getVelocity(int timestep); + void getVelocity_debug(int timestep); double CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField); bool Restart,pBC; @@ -79,6 +80,7 @@ private: char LocalRankString[8]; char LocalRankFilename[40]; char LocalRestartFile[40]; + char OutputFilename[200]; //int rank,nprocs; void LoadParams(std::shared_ptr db0); From fd3663628d972220f3c8db95a1c5b3cb24864a3c Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 8 Oct 2020 13:13:31 -0400 Subject: [PATCH 241/270] adding grey analysis tools --- analysis/GreyPhase.cpp | 275 +++++++++++++++++++++++++++++++++++++++++ analysis/GreyPhase.h | 52 ++++++++ 2 files changed, 327 insertions(+) create mode 100644 analysis/GreyPhase.cpp create mode 100644 analysis/GreyPhase.h diff --git a/analysis/GreyPhase.cpp b/analysis/GreyPhase.cpp new file mode 100644 index 00000000..6e7a958f --- /dev/null +++ b/analysis/GreyPhase.cpp @@ -0,0 +1,275 @@ +#include "analysis/SubPhase.h" + +// Constructor +GreyPhase::GreyPhase(std::shared_ptr dm): + Dm(dm) +{ + Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; + Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; + + // Global arrays + PhaseID.resize(Nx,Ny,Nz); PhaseID.fill(0); + Rho_n.resize(Nx,Ny,Nz); Rho_n.fill(0); + Rho_w.resize(Nx,Ny,Nz); Rho_w.fill(0); + Pressure.resize(Nx,Ny,Nz); Pressure.fill(0); + Phi.resize(Nx,Ny,Nz); Phi.fill(0); + DelPhi.resize(Nx,Ny,Nz); DelPhi.fill(0); + Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field + Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0); + Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0); + //......................................... + + if (Dm->rank()==0){ + bool WriteHeader=false; + TIMELOG = fopen("timelog.csv","r"); + if (TIMELOG != NULL) + fclose(TIMELOG); + else + WriteHeader=true; + + TIMELOG = fopen("timelog.csv","a+"); + if (WriteHeader) + { + // If timelog is empty, write a short header to list the averages + //fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n"); + fprintf(TIMELOG,"sw krw krn vw vn pw pn\n"); + } + } +} + + +// Destructor +GreyPhase::~GreyPhase() +{ + +} + +void GreyPhase::Write(int timestep) +{ + +} + +void GreyPhase::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B) +{ + Fx = force_x; + Fy = force_y; + Fz = force_z; + rho_n = rhoA; + rho_w = rhoB; + nu_n = (tauA-0.5)/3.f; + nu_w = (tauB-0.5)/3.f; + gamma_wn = 5.796*alpha; + beta = B; +} + +void GreyPhase::Basic(){ + int i,j,k,n,imin,jmin,kmin,kmax; + + // If external boundary conditions are set, do not average over the inlet + kmin=1; kmax=Nz-1; + imin=jmin=1; + + double count_w = 0.0; + double count_n = 0.0; + + for (k=0; kid[n] > 0 ){ + // compute density + double nA = Rho_n(n); + double nB = Rho_w(n); + double phi = (nA-nB)/(nA+nB); + Phi(n) = phi; + } + } + } + } + + for (k=kmin; kid[n] > 0 ){ + // compute density + double nA = Rho_n(n); + double nB = Rho_w(n); + double phi = (nA-nB)/(nA+nB); + /* if ( phi > 0.0 ){ + nA = 1.0; + nb.V += 1.0; + nb.M += nA*rho_n; + // velocity + nb.Px += rho_n*nA*Vel_x(n); + nb.Py += rho_n*nA*Vel_y(n); + nb.Pz += rho_n*nA*Vel_z(n); + } + else{ + nB = 1.0; + wb.M += nB*rho_w; + wb.V += 1.0; + + // velocity + wb.Px += rho_w*nB*Vel_x(n); + wb.Py += rho_w*nB*Vel_y(n); + wb.Pz += rho_w*nB*Vel_z(n); + } + if ( phi > 0.99 ){ + nb.p += Pressure(n); + count_n += 1.0; + } + else if ( phi < -0.99 ){ + wb.p += Pressure(n); + count_w += 1.0; + } + */ + } + } + } + } + /* gwb.V=sumReduce( Dm->Comm, wb.V); + gnb.V=sumReduce( Dm->Comm, nb.V); + gwb.M=sumReduce( Dm->Comm, wb.M); + gnb.M=sumReduce( Dm->Comm, nb.M); + gwb.Px=sumReduce( Dm->Comm, wb.Px); + gwb.Py=sumReduce( Dm->Comm, wb.Py); + gwb.Pz=sumReduce( Dm->Comm, wb.Pz); + gnb.Px=sumReduce( Dm->Comm, nb.Px); + gnb.Py=sumReduce( Dm->Comm, nb.Py); + gnb.Pz=sumReduce( Dm->Comm, nb.Pz); + + count_w=sumReduce( Dm->Comm, count_w); + count_n=sumReduce( Dm->Comm, count_n); + if (count_w > 0.0) + gwb.p=sumReduce( Dm->Comm, wb.p) / count_w; + else + gwb.p = 0.0; + if (count_n > 0.0) + gnb.p=sumReduce( Dm->Comm, nb.p) / count_n; + else + gnb.p = 0.0; + + // check for NaN + bool err=false; + if (gwb.V != gwb.V) err=true; + if (gnb.V != gnb.V) err=true; + if (gwb.p != gwb.p) err=true; + if (gnb.p != gnb.p) err=true; + if (gwb.Px != gwb.Px) err=true; + if (gwb.Py != gwb.Py) err=true; + if (gwb.Pz != gwb.Pz) err=true; + if (gnb.Px != gnb.Px) err=true; + if (gnb.Py != gnb.Py) err=true; + if (gnb.Pz != gnb.Pz) err=true; + + if (Dm->rank() == 0){ + double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); + double dir_x = 0.0; + double dir_y = 0.0; + double dir_z = 0.0; + if (force_mag > 0.0){ + dir_x = Fx/force_mag; + dir_y = Fy/force_mag; + dir_z = Fz/force_mag; + } + else { + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + } + if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){ + // compute the pressure drop + double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0; + double length = ((Nz-2)*Dm->nprocz()); + force_mag -= pressure_drop/length; + } + if (force_mag == 0.0){ + // default to z direction + dir_x = 0.0; + dir_y = 0.0; + dir_z = 1.0; + force_mag = 1.0; + } + double saturation=gwb.V/(gwb.V + gnb.V); + double water_flow_rate=gwb.V*(gwb.Px*dir_x + gwb.Py*dir_y + gwb.Pz*dir_z)/gwb.M / Dm->Volume; + double not_water_flow_rate=gnb.V*(gnb.Px*dir_x + gnb.Py*dir_y + gnb.Pz*dir_z)/gnb.M/ Dm->Volume; + //double total_flow_rate = water_flow_rate + not_water_flow_rate; + //double fractional_flow = water_flow_rate / total_flow_rate; + + double h = Dm->voxel_length; + double krn = h*h*nu_n*not_water_flow_rate / force_mag ; + double krw = h*h*nu_w*water_flow_rate / force_mag; + //printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow); + fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p); + fflush(TIMELOG); + } +*/ + if (err==true){ + // exception if simulation produceds NaN + printf("GreyPhase.cpp: NaN encountered, may need to check simulation parameters \n"); + } + ASSERT(err==false); +} +/* +inline void InterfaceTransportMeasures( double beta, double rA, double rB, double nA, double nB, + double nx, double ny, double nz, double ux, double uy, double uz, interface &I){ + + double A1,A2,A3,A4,A5,A6; + double B1,B2,B3,B4,B5,B6; + double nAB,delta; + // Instantiate mass transport distributions + // Stationary value - distribution 0 + nAB = 1.0/(nA+nB); + //............................................... + // q = 0,2,4 + // Cq = {1,0,0}, {0,1,0}, {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nx; + if (!(nA*nB*nAB>0)) delta=0; + A1 = nA*(0.1111111111111111*(1+4.5*ux))+delta; + B1 = nB*(0.1111111111111111*(1+4.5*ux))-delta; + A2 = nA*(0.1111111111111111*(1-4.5*ux))-delta; + B2 = nB*(0.1111111111111111*(1-4.5*ux))+delta; + + //............................................... + // Cq = {0,1,0} + delta = beta*nA*nB*nAB*0.1111111111111111*ny; + if (!(nA*nB*nAB>0)) delta=0; + A3 = nA*(0.1111111111111111*(1+4.5*uy))+delta; + B3 = nB*(0.1111111111111111*(1+4.5*uy))-delta; + A4 = nA*(0.1111111111111111*(1-4.5*uy))-delta; + B4 = nB*(0.1111111111111111*(1-4.5*uy))+delta; + + //............................................... + // q = 4 + // Cq = {0,0,1} + delta = beta*nA*nB*nAB*0.1111111111111111*nz; + if (!(nA*nB*nAB>0)) delta=0; + A5 = nA*(0.1111111111111111*(1+4.5*uz))+delta; + B5 = nB*(0.1111111111111111*(1+4.5*uz))-delta; + A6 = nA*(0.1111111111111111*(1-4.5*uz))-delta; + B6 = nB*(0.1111111111111111*(1-4.5*uz))+delta; + + double unx = (A1-A2); + double uny = (A3-A4); + double unz = (A5-A6); + double uwx = (B1-B2); + double uwy = (B3-B4); + double uwz = (B5-B6); + + I.Mn += rA*nA; + I.Mw += rB*nB; + I.Pnx += rA*nA*unx; + I.Pny += rA*nA*uny; + I.Pnz += rA*nA*unz; + I.Pwx += rB*nB*uwx; + I.Pwy += rB*nB*uwy; + I.Pwz += rB*nB*uwz; + I.Kn += rA*nA*(unx*unx + uny*uny + unz*unz); + I.Kw += rB*nB*(uwx*uwx + uwy*uwy + uwz*uwz); + +} +*/ diff --git a/analysis/GreyPhase.h b/analysis/GreyPhase.h new file mode 100644 index 00000000..ddf43261 --- /dev/null +++ b/analysis/GreyPhase.h @@ -0,0 +1,52 @@ +/* + * Sub-phase averaging tools + */ + +#ifndef GreyPhase_INC +#define GreyPhase_INC + +#include +#include "common/Domain.h" +#include "common/Communication.h" +#include "analysis/analysis.h" +#include "common/Utilities.h" +#include "common/MPI_Helpers.h" +#include "IO/MeshDatabase.h" +#include "IO/Reader.h" +#include "IO/Writer.h" + +class GreyPhase{ +public: + std::shared_ptr Dm; + double Volume; + // input variables + double rho_n, rho_w; + double nu_n, nu_w; + double gamma_wn, beta; + double Fx, Fy, Fz; + + //........................................................................... + int Nx,Ny,Nz; + IntArray PhaseID; // Phase ID array + DoubleArray Rho_n; // density field + DoubleArray Rho_w; // density field + DoubleArray Phi; // phase indicator field + DoubleArray DelPhi; // Magnitude of Gradient of the phase indicator field + DoubleArray Pressure; // pressure field + DoubleArray Vel_x; // velocity field + DoubleArray Vel_y; + DoubleArray Vel_z; + + GreyPhase(std::shared_ptr Dm); + ~GreyPhase(); + + void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta); + void Basic(); + void Write(int time); + +private: + FILE *TIMELOG; +}; + +#endif + From 896a3617b8268d2e68bca90cd1beb91930f25529 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 8 Oct 2020 13:14:01 -0400 Subject: [PATCH 242/270] adding dedicated analysis for grey model (broken) --- models/GreyscaleColorModel.cpp | 40 +--------------------------------- models/GreyscaleColorModel.h | 3 +-- 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index d97c844d..e35aff4a 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -19,43 +19,6 @@ Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0), ScaLBL_GreyscaleColorModel::~ScaLBL_GreyscaleColorModel(){ } - -/*void ScaLBL_GreyscaleColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np) -{ - int q,n; - double value; - ofstream File(FILENAME,ios::binary); - for (n=0; n( filename ); @@ -172,8 +135,7 @@ void ScaLBL_GreyscaleColorModel::SetDomain(){ N = Nx*Ny*Nz; id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way - //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object - Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); Dm->CommInit(); MPI_Barrier(comm); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index d7dde941..b9e8d11f 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -50,8 +50,7 @@ public: std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; std::shared_ptr ScaLBL_Comm_Regular; - //std::shared_ptr Averages; - std::shared_ptr Averages; + std::shared_ptr Averages; // input database std::shared_ptr db; From 390556ceb0785be07e0466cce4d62f2261b713e0 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 8 Oct 2020 14:45:28 -0400 Subject: [PATCH 243/270] greyscale model with simple analysis tool --- analysis/GreyPhase.cpp | 151 +++++------ analysis/GreyPhase.h | 26 +- models/GreyscaleColorModel.cpp | 441 ++------------------------------- models/GreyscaleColorModel.h | 5 +- 4 files changed, 105 insertions(+), 518 deletions(-) diff --git a/analysis/GreyPhase.cpp b/analysis/GreyPhase.cpp index 6e7a958f..ce1e2547 100644 --- a/analysis/GreyPhase.cpp +++ b/analysis/GreyPhase.cpp @@ -1,13 +1,15 @@ -#include "analysis/SubPhase.h" +#include "analysis/GreyPhase.h" // Constructor -GreyPhase::GreyPhase(std::shared_ptr dm): +GreyPhaseAnalysis::GreyPhaseAnalysis(std::shared_ptr dm): Dm(dm) { Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; // Global arrays + SDs.resize(Nx,Ny,Nz); SDs.fill(0); + Porosity.resize(Nx,Ny,Nz); Porosity.fill(0); PhaseID.resize(Nx,Ny,Nz); PhaseID.fill(0); Rho_n.resize(Nx,Ny,Nz); Rho_n.fill(0); Rho_w.resize(Nx,Ny,Nz); Rho_w.fill(0); @@ -39,17 +41,17 @@ GreyPhase::GreyPhase(std::shared_ptr dm): // Destructor -GreyPhase::~GreyPhase() +GreyPhaseAnalysis::~GreyPhaseAnalysis() { } -void GreyPhase::Write(int timestep) +void GreyPhaseAnalysis::Write(int timestep) { } -void GreyPhase::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B) +void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B) { Fx = force_x; Fy = force_y; @@ -62,7 +64,7 @@ void GreyPhase::SetParams(double rhoA, double rhoB, double tauA, double tauB, do beta = B; } -void GreyPhase::Basic(){ +void GreyPhaseAnalysis::Basic(){ int i,j,k,n,imin,jmin,kmin,kmax; // If external boundary conditions are set, do not average over the inlet @@ -72,22 +74,8 @@ void GreyPhase::Basic(){ double count_w = 0.0; double count_n = 0.0; - for (k=0; kid[n] > 0 ){ - // compute density - double nA = Rho_n(n); - double nB = Rho_w(n); - double phi = (nA-nB)/(nA+nB); - Phi(n) = phi; - } - } - } - } - + Water_local.reset(); + Oil_local.reset(); for (k=kmin; kid[n] > 0 ){ // compute density - double nA = Rho_n(n); - double nB = Rho_w(n); - double phi = (nA-nB)/(nA+nB); - /* if ( phi > 0.0 ){ - nA = 1.0; - nb.V += 1.0; - nb.M += nA*rho_n; - // velocity - nb.Px += rho_n*nA*Vel_x(n); - nb.Py += rho_n*nA*Vel_y(n); - nb.Pz += rho_n*nA*Vel_z(n); - } - else{ - nB = 1.0; - wb.M += nB*rho_w; - wb.V += 1.0; + double pressure = Pressure(n); + double rho_n = Rho_n(n); + double rho_w = Rho_w(n); + double porosity = Porosity(n); + double vel_x = Vel_x(n); + double vel_y = Vel_y(n); + double vel_z = Vel_z(n); + Water_local.p += pressure*(rho_w)/(rho_n+rho_w); + Water_local.M += rho_w*porosity; + Water_local.Px += rho_w*vel_x; + Water_local.Py += rho_w*vel_y; + Water_local.Pz += rho_w*vel_z; + Oil_local.p += pressure*(rho_n)/(rho_n+rho_w); + Oil_local.M += rho_n*porosity; + Oil_local.Px += rho_n*vel_x; + Oil_local.Py += rho_n*vel_y; + Oil_local.Pz += rho_n*vel_z; - // velocity - wb.Px += rho_w*nB*Vel_x(n); - wb.Py += rho_w*nB*Vel_y(n); - wb.Pz += rho_w*nB*Vel_z(n); - } - if ( phi > 0.99 ){ - nb.p += Pressure(n); - count_n += 1.0; - } - else if ( phi < -0.99 ){ - wb.p += Pressure(n); - count_w += 1.0; - } - */ } } } } - /* gwb.V=sumReduce( Dm->Comm, wb.V); - gnb.V=sumReduce( Dm->Comm, nb.V); - gwb.M=sumReduce( Dm->Comm, wb.M); - gnb.M=sumReduce( Dm->Comm, nb.M); - gwb.Px=sumReduce( Dm->Comm, wb.Px); - gwb.Py=sumReduce( Dm->Comm, wb.Py); - gwb.Pz=sumReduce( Dm->Comm, wb.Pz); - gnb.Px=sumReduce( Dm->Comm, nb.Px); - gnb.Py=sumReduce( Dm->Comm, nb.Py); - gnb.Pz=sumReduce( Dm->Comm, nb.Pz); - - count_w=sumReduce( Dm->Comm, count_w); - count_n=sumReduce( Dm->Comm, count_n); - if (count_w > 0.0) - gwb.p=sumReduce( Dm->Comm, wb.p) / count_w; - else - gwb.p = 0.0; - if (count_n > 0.0) - gnb.p=sumReduce( Dm->Comm, nb.p) / count_n; - else - gnb.p = 0.0; + Oil.M=sumReduce( Dm->Comm, Oil_local.M); + Oil.p=sumReduce( Dm->Comm, Oil_local.p); + Oil.Px=sumReduce( Dm->Comm, Oil_local.Px); + Oil.Py=sumReduce( Dm->Comm, Oil_local.Py); + Oil.Pz=sumReduce( Dm->Comm, Oil_local.Pz); + + Water.M=sumReduce( Dm->Comm, Water_local.M); + Water.p=sumReduce( Dm->Comm, Water_local.p); + Water.Px=sumReduce( Dm->Comm, Water_local.Px); + Water.Py=sumReduce( Dm->Comm, Water_local.Py); + Water.Pz=sumReduce( Dm->Comm, Water_local.Pz); + + Oil.p /= Oil.M; + Water.p /= Water.M; // check for NaN bool err=false; - if (gwb.V != gwb.V) err=true; - if (gnb.V != gnb.V) err=true; - if (gwb.p != gwb.p) err=true; - if (gnb.p != gnb.p) err=true; - if (gwb.Px != gwb.Px) err=true; - if (gwb.Py != gwb.Py) err=true; - if (gwb.Pz != gwb.Pz) err=true; - if (gnb.Px != gnb.Px) err=true; - if (gnb.Py != gnb.Py) err=true; - if (gnb.Pz != gnb.Pz) err=true; + if (Water.M != Water.M) err=true; + if (Water.p != Water.p) err=true; + if (Water.Px != Water.Px) err=true; + if (Water.Py != Water.Py) err=true; + if (Water.Pz != Water.Pz) err=true; + + if (Oil.M != Oil.M) err=true; + if (Oil.p != Oil.p) err=true; + if (Oil.Px != Oil.Px) err=true; + if (Oil.Py != Oil.Py) err=true; + if (Oil.Pz != Oil.Pz) err=true; if (Dm->rank() == 0){ double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); @@ -194,23 +163,21 @@ void GreyPhase::Basic(){ dir_z = 1.0; force_mag = 1.0; } - double saturation=gwb.V/(gwb.V + gnb.V); - double water_flow_rate=gwb.V*(gwb.Px*dir_x + gwb.Py*dir_y + gwb.Pz*dir_z)/gwb.M / Dm->Volume; - double not_water_flow_rate=gnb.V*(gnb.Px*dir_x + gnb.Py*dir_y + gnb.Pz*dir_z)/gnb.M/ Dm->Volume; - //double total_flow_rate = water_flow_rate + not_water_flow_rate; - //double fractional_flow = water_flow_rate / total_flow_rate; + saturation=Water.M/(Water.M + Oil.M); // assume constant density + water_flow_rate=saturation*(Water.Px*dir_x + Water.Py*dir_y + Water.Pz*dir_z)/Water.M; + oil_flow_rate=(1.0-saturation)*(Oil.Px*dir_x + Oil.Py*dir_y + Oil.Pz*dir_z)/Oil.M; double h = Dm->voxel_length; - double krn = h*h*nu_n*not_water_flow_rate / force_mag ; + double krn = h*h*nu_n*oil_flow_rate / force_mag ; double krw = h*h*nu_w*water_flow_rate / force_mag; //printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow); - fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p); + fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*oil_flow_rate, Water.p, Oil.p); fflush(TIMELOG); } -*/ + if (err==true){ // exception if simulation produceds NaN - printf("GreyPhase.cpp: NaN encountered, may need to check simulation parameters \n"); + printf("GreyPhaseAnalysis.cpp: NaN encountered, may need to check simulation parameters \n"); } ASSERT(err==false); } diff --git a/analysis/GreyPhase.h b/analysis/GreyPhase.h index ddf43261..3019648e 100644 --- a/analysis/GreyPhase.h +++ b/analysis/GreyPhase.h @@ -6,7 +6,7 @@ #define GreyPhase_INC #include -#include "common/Domain.h" +#include "common/ScaLBL.h" #include "common/Communication.h" #include "analysis/analysis.h" #include "common/Utilities.h" @@ -16,6 +16,17 @@ #include "IO/Writer.h" class GreyPhase{ + public: + double p; + double M,Px,Py,Pz; + void reset(){ + p=M=Px=Py=Pz=0.0; + } + + private: +}; + +class GreyPhaseAnalysis{ public: std::shared_ptr Dm; double Volume; @@ -24,9 +35,16 @@ public: double nu_n, nu_w; double gamma_wn, beta; double Fx, Fy, Fz; + // outputs + double saturation,water_flow_rate, oil_flow_rate; + //simulation outputs (averaged values) + GreyPhase Water, Oil; + GreyPhase Water_local, Oil_local; //........................................................................... - int Nx,Ny,Nz; + int Nx,Ny,Nz; + IntArray SDs; // contains porosity map + IntArray Porosity; // contains porosity map IntArray PhaseID; // Phase ID array DoubleArray Rho_n; // density field DoubleArray Rho_w; // density field @@ -37,8 +55,8 @@ public: DoubleArray Vel_y; DoubleArray Vel_z; - GreyPhase(std::shared_ptr Dm); - ~GreyPhase(); + GreyPhaseAnalysis(std::shared_ptr Dm); + ~GreyPhaseAnalysis(); void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta); void Basic(); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index e35aff4a..b83c0c53 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -135,7 +135,7 @@ void ScaLBL_GreyscaleColorModel::SetDomain(){ N = Nx*Ny*Nz; id = new signed char [N]; for (int i=0; iid[i] = 1; // initialize this way - Averages = std::shared_ptr ( new SubPhase(Dm) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new GreyPhaseAnalysis(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); Dm->CommInit(); MPI_Barrier(comm); @@ -831,15 +831,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ USE_SEED = true; USE_MORPH = true; } - else if (protocol == "open connected oil"){ - morph_delta = -0.05; - USE_MORPH = true; - USE_MORPHOPEN_OIL = true; - } - else if (protocol == "shell aggregation"){ - morph_delta = -0.05; - USE_MORPH = true; - } + if (greyscaleColor_db->keyExists( "capillary_number" )){ capillary_number = greyscaleColor_db->getScalar( "capillary_number" ); SET_CAPILLARY_NUMBER=true; @@ -868,11 +860,6 @@ void ScaLBL_GreyscaleColorModel::Run(){ morph_interval = analysis_db->getScalar( "morph_interval" ); USE_MORPH = true; } - if (analysis_db->keyExists( "use_morphopen_oil" )){ - USE_MORPHOPEN_OIL = analysis_db->getScalar( "use_morphopen_oil" ); - if (rank == 0 && USE_MORPHOPEN_OIL) printf("Volume change by morphological opening \n"); - USE_MORPH = true; - } if (analysis_db->keyExists( "tolerance" )){ tolerance = analysis_db->getScalar( "tolerance" ); } @@ -908,20 +895,6 @@ void ScaLBL_GreyscaleColorModel::Run(){ printf(" morph_delta = %f \n",morph_delta); printf(" seed_water = %f \n",seed_water); } - else if (protocol == "open connected oil"){ - printf(" using protocol = open connected oil \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - printf(" morph_delta = %f \n",morph_delta); - } - else if (protocol == "shell aggregation"){ - printf(" using protocol = shell aggregation \n"); - printf(" min_steady_timesteps = %i \n",MIN_STEADY_TIMESTEPS); - printf(" max_steady_timesteps = %i \n",MAX_STEADY_TIMESTEPS); - printf(" tolerance = %f \n",tolerance); - printf(" morph_delta = %f \n",morph_delta); - } printf("No. of timesteps: %i \n", timestepMax); fflush(stdout); } @@ -938,7 +911,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ //std::shared_ptr analysis_db; bool Regular = false; auto current_db = db->cloneDatabase(); - runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); + //runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map ); //analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } @@ -1076,26 +1049,20 @@ void ScaLBL_GreyscaleColorModel::Run(){ if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ printf("%i %f \n",timestep,din); + ScaLBL_Comm->RegularLayout(Map,Pressure,Averages->Pressure); + ScaLBL_Comm->RegularLayout(Map,&Den[0],Averages->Rho_n); + ScaLBL_Comm->RegularLayout(Map,&Den[Np],Averages->Rho_w); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Averages->Vel_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Averages->Vel_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); + Averages->Basic(); } - // Run the analysis - analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); // allow initial ramp-up to get closer to steady state if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ - analysis.finish(); + //analysis.finish(); CURRENT_STEADY_TIMESTEPS += analysis_interval; - double volB = Averages->gwb.V; - double volA = Averages->gnb.V; - volA /= Dm->Volume; - volB /= Dm->Volume;; - //initial_volume = volA*Dm->Volume; - double vA_x = Averages->gnb.Px/Averages->gnb.M; - double vA_y = Averages->gnb.Py/Averages->gnb.M; - double vA_z = Averages->gnb.Pz/Averages->gnb.M; - double vB_x = Averages->gwb.Px/Averages->gwb.M; - double vB_y = Averages->gwb.Py/Averages->gwb.M; - double vB_z = Averages->gwb.Pz/Averages->gwb.M; double muA = rhoA*(tauA-0.5)/3.f; double muB = rhoB*(tauB-0.5)/3.f; double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz); @@ -1109,10 +1076,12 @@ void ScaLBL_GreyscaleColorModel::Run(){ dir_z = 1.0; force_mag = 1.0; } - double current_saturation = volB/(volA+volB); - double flow_rate_A = volA*(vA_x*dir_x + vA_y*dir_y + vA_z*dir_z); - double flow_rate_B = volB*(vB_x*dir_x + vB_y*dir_y + vB_z*dir_z); - double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(5.796*alpha); + double current_saturation = Averages->saturation; + double volA = current_saturation*Mask->Porosity()*Mask->Volume; + double volB = (1.0-current_saturation)*Mask->Porosity()*Mask->Volume; + double flow_rate_A = Averages->oil_flow_rate; + double flow_rate_B = Averages->water_flow_rate; + double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(6.0*alpha); if ( morph_timesteps > morph_interval ){ @@ -1165,17 +1134,17 @@ void ScaLBL_GreyscaleColorModel::Run(){ volA_prev = volA; //******************************** **/ /** compute averages & write data **/ - Averages->Full(); + /*Averages->Full(); Averages->Write(timestep); analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den ); analysis.finish(); - + */ if (rank==0){ printf("** WRITE STEADY POINT *** "); printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); double h = Dm->voxel_length; // pressures - double pA = Averages->gnb.p; + /*double pA = Averages->gnb.p; double pB = Averages->gwb.p; double pAc = Averages->gnc.p; double pBc = Averages->gwc.p; @@ -1231,7 +1200,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); fclose(kr_log_file); - + */ printf(" Measured capillary number %f \n ",Ca); } if (SET_CAPILLARY_NUMBER ){ @@ -1283,16 +1252,6 @@ void ScaLBL_GreyscaleColorModel::Run(){ double massChange = SeedPhaseField(seed_water); if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); } - else if (USE_MORPHOPEN_OIL){ - delta_volume = volA*Dm->Volume - initial_volume; - if (rank==0) printf("***Morphological opening of connected oil, target volume change %f ***\n", delta_volume_target); - MorphOpenConnected(delta_volume_target); - } - else { - if (rank==0) printf("***Shell aggregation, target volume change %f ***\n", delta_volume_target); - //double delta_volume_target = volB - (volA + volB)*TARGET_SATURATION; // change in volume to A - delta_volume += MorphInit(beta,delta_volume_target-delta_volume); - } if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ MORPH_ADAPT = false; @@ -1316,7 +1275,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ } MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); } - analysis.finish(); + //analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ @@ -1380,149 +1339,6 @@ double ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ return saturation; } - -double ScaLBL_GreyscaleColorModel::MorphOpenConnected(double target_volume_change){ - - int nx = Nx; - int ny = Ny; - int nz = Nz; - int n; - int N = nx*ny*nz; - double volume_change=0.0; - - if (target_volume_change < 0.0){ - Array id_solid(nx,ny,nz); - Array phase_label(nx,ny,nz); - DoubleArray distance(Nx,Ny,Nz); - DoubleArray phase(nx,ny,nz); - signed char *id_connected; - id_connected = new signed char [nx*ny*nz]; - - ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); - - // Extract only the connected part of NWP - BlobIDstruct new_index; - double vF=0.0; double vS=0.0; - ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm); - MPI_Barrier(Dm->Comm); - - long long count_connected=0; - long long count_porespace=0; - long long count_water=0; - for (int k=1; k 0){ - count_porespace++; - } - if (id[n] == 2){ - count_water++; - } - } - } - } - count_connected=sumReduce( Dm->Comm, count_connected); - count_porespace=sumReduce( Dm->Comm, count_porespace); - count_water=sumReduce( Dm->Comm, count_water); - - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } - } - } - - int count_morphopen=0.0; - for (int k=1; kComm, count_morphopen); - volume_change = double(count_morphopen - count_connected); - - if (rank==0) printf(" opening of connected oil %f \n",volume_change/count_connected); - - ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - } - } - return(volume_change); -} double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ srand(time(NULL)); double mass_loss =0.f; @@ -1598,219 +1414,6 @@ double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil return(mass_loss); } -double ScaLBL_GreyscaleColorModel::MorphInit(const double beta, const double target_delta_volume){ - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - - double vF = 0.f; - double vS = 0.f; - double delta_volume; - double WallFactor = 0.0; - bool USE_CONNECTED_NWP = false; - - DoubleArray phase(Nx,Ny,Nz); - IntArray phase_label(Nx,Ny,Nz);; - DoubleArray phase_distance(Nx,Ny,Nz); - Array phase_id(Nx,Ny,Nz); - fillHalo fillDouble(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); - - - // Basic algorithm to - // 1. Copy phase field to CPU - ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double)); - - double count = 0.f; - for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f) count+=1.f; - } - } - } - double volume_initial = sumReduce( Dm->Comm, count); - /* - sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank); - FILE *INPUT = fopen(LocalRankFilename,"wb"); - fwrite(phase.data(),8,N,INPUT); - fclose(INPUT); - */ - // 2. Identify connected components of phase field -> phase_label - - double volume_connected = 0.0; - double second_biggest = 0.0; - if (USE_CONNECTED_NWP){ - BlobIDstruct new_index; - ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm); - MPI_Barrier(Dm->Comm); - - // only operate on component "0" - count = 0.0; - - for (int k=0; kComm, count); - second_biggest = sumReduce( Dm->Comm, second_biggest); - } - else { - // use the whole NWP - for (int k=0; kSDs(i,j,k) > 0.f){ - if (phase(i,j,k) > 0.f ){ - phase_id(i,j,k) = 0; - } - else { - phase_id(i,j,k) = 1; - } - } - else { - phase_id(i,j,k) = 1; - } - } - } - } - } - - /*int reach_x, reach_y, reach_z; - for (int k=0; k phase_distance - CalcDist(phase_distance,phase_id,*Dm); - - double temp,value; - double factor=0.5/beta; - for (int k=0; k 1.f) value=1.f; - if (value < -1.f) value=-1.f; - // temp -- distance based on analytical form McClure, Prins et al, Comp. Phys. Comm. - temp = -factor*log((1.0+value)/(1.0-value)); - /// use this approximation close to the object - if (fabs(value) < 0.8 && Averages->SDs(i,j,k) > 1.f ){ - phase_distance(i,j,k) = temp; - } - // erase the original object - phase(i,j,k) = -1.0; - } - } - } - } - - if (USE_CONNECTED_NWP){ - if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){ - // if connected volume is less than 2% just delete the whole thing - if (rank==0) printf("Connected region has shrunk! \n"); - REVERSE_FLOW_DIRECTION = true; - } - -/* else{*/ - if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest ); - } - if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial); - double target_delta_volume_incremental = target_delta_volume; - if (fabs(target_delta_volume) > 0.01*volume_initial) - target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume); - delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor); - - for (int k=0; kSDs(i,j,k) > 0.f){ - if (d < 3.f){ - //phase(i,j,k) = -1.0; - phase(i,j,k) = (2.f*(exp(-2.f*beta*d))/(1.f+exp(-2.f*beta*d))-1.f); - } - } - } - } - } - fillDouble.fill(phase); - //} - - count = 0.f; - for (int k=1; k 0.f && Averages->SDs(i,j,k) > 0.f){ - count+=1.f; - } - } - } - } - double volume_final= sumReduce( Dm->Comm, count); - - delta_volume = (volume_final-volume_initial); - if (rank == 0) printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial); - if (rank == 0) printf(" new saturation = %f \n", volume_final/(0.238323*double((Nx-2)*(Ny-2)*(Nz-2)*nprocs))); - - // 6. copy back to the device - //if (rank==0) printf("MorphInit: copy data back to device\n"); - ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double)); - /* - sprintf(LocalRankFilename,"dist_final.%05i.raw",rank); - FILE *DIST = fopen(LocalRankFilename,"wb"); - fwrite(phase_distance.data(),8,N,DIST); - fclose(DIST); - - sprintf(LocalRankFilename,"phi_final.%05i.raw",rank); - FILE *PHI = fopen(LocalRankFilename,"wb"); - fwrite(phase.data(),8,N,PHI); - fclose(PHI); - */ - // 7. Re-initialize phase field and density - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - } - return delta_volume; -} - void ScaLBL_GreyscaleColorModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index b9e8d11f..d7298c10 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -10,8 +10,7 @@ Implementation of two-fluid greyscale color lattice boltzmann model #include #include "common/Communication.h" -#include "analysis/TwoPhase.h" -#include "analysis/runAnalysis.h" +#include "analysis/GreyPhase.h" #include "common/MPI_Helpers.h" #include "ProfilerApp.h" #include "threadpool/thread_pool.h" @@ -50,7 +49,7 @@ public: std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; std::shared_ptr ScaLBL_Comm_Regular; - std::shared_ptr Averages; + std::shared_ptr Averages; // input database std::shared_ptr db; From 33f6f836870f301b230c55d4a97379f4d35914a7 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Thu, 8 Oct 2020 14:52:06 -0400 Subject: [PATCH 244/270] fix array type bug --- analysis/GreyPhase.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analysis/GreyPhase.h b/analysis/GreyPhase.h index 3019648e..5583bc1a 100644 --- a/analysis/GreyPhase.h +++ b/analysis/GreyPhase.h @@ -43,9 +43,9 @@ public: GreyPhase Water_local, Oil_local; //........................................................................... int Nx,Ny,Nz; - IntArray SDs; // contains porosity map - IntArray Porosity; // contains porosity map IntArray PhaseID; // Phase ID array + DoubleArray SDs; // contains porosity map + DoubleArray Porosity; // contains porosity map DoubleArray Rho_n; // density field DoubleArray Rho_w; // density field DoubleArray Phi; // phase indicator field From 21d9e6c9007c1bd54c11ca45651574608620a830 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 8 Oct 2020 22:13:28 -0400 Subject: [PATCH 245/270] save the work;wait for validation results --- tests/TestIonModel.cpp | 35 +++++++++++-------- tests/TestNernstPlanck.cpp | 35 +++++++++++-------- tests/TestPNP_Stokes.cpp | 35 +++++++++++-------- tests/TestPoissonSolver.cpp | 33 ++++++++++------- ...m_electrokinetic_SingleFluid_simulator.cpp | 35 +++++++++++-------- 5 files changed, 104 insertions(+), 69 deletions(-) diff --git a/tests/TestIonModel.cpp b/tests/TestIonModel.cpp index 32f8302b..0b57ff1c 100644 --- a/tests/TestIonModel.cpp +++ b/tests/TestIonModel.cpp @@ -9,6 +9,7 @@ #include "models/IonModel.h" #include "models/MultiPhysController.h" +#include "common/Utilities.h" using namespace std; @@ -18,24 +19,27 @@ using namespace std; int main(int argc, char **argv) { - // Initialize MPI - int provided_thread_support = -1; - MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - if ( rank==0 && provided_thread_support #include "models/PoissonSolver.h" +#include "common/Utilities.h" using namespace std; @@ -18,23 +19,26 @@ using namespace std; int main(int argc, char **argv) { // Initialize MPI - int provided_thread_support = -1; - MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - if ( rank==0 && provided_thread_support Date: Fri, 9 Oct 2020 13:07:06 -0400 Subject: [PATCH 246/270] post-polish; make greyscaleColor analysis consistent with normal color;build test passed --- analysis/GreyPhase.cpp | 52 +++---- analysis/GreyPhase.h | 11 +- common/ScaLBL.h | 4 +- cpu/Greyscale.cpp | 4 +- cpu/GreyscaleColor.cpp | 6 +- gpu/Greyscale.cu | 4 +- gpu/GreyscaleColor.cu | 15 +- models/GreyscaleColorModel.cpp | 268 +++++++++++++++------------------ models/GreyscaleColorModel.h | 3 +- 9 files changed, 173 insertions(+), 194 deletions(-) diff --git a/analysis/GreyPhase.cpp b/analysis/GreyPhase.cpp index ce1e2547..1e341a1a 100644 --- a/analysis/GreyPhase.cpp +++ b/analysis/GreyPhase.cpp @@ -8,14 +8,14 @@ GreyPhaseAnalysis::GreyPhaseAnalysis(std::shared_ptr dm): Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; // Global arrays - SDs.resize(Nx,Ny,Nz); SDs.fill(0); - Porosity.resize(Nx,Ny,Nz); Porosity.fill(0); - PhaseID.resize(Nx,Ny,Nz); PhaseID.fill(0); + SDs.resize(Nx,Ny,Nz); SDs.fill(0); + Porosity.resize(Nx,Ny,Nz); Porosity.fill(0); + //PhaseID.resize(Nx,Ny,Nz); PhaseID.fill(0); Rho_n.resize(Nx,Ny,Nz); Rho_n.fill(0); Rho_w.resize(Nx,Ny,Nz); Rho_w.fill(0); Pressure.resize(Nx,Ny,Nz); Pressure.fill(0); - Phi.resize(Nx,Ny,Nz); Phi.fill(0); - DelPhi.resize(Nx,Ny,Nz); DelPhi.fill(0); + //Phi.resize(Nx,Ny,Nz); Phi.fill(0); + //DelPhi.resize(Nx,Ny,Nz); DelPhi.fill(0); Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0); Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0); @@ -51,7 +51,7 @@ void GreyPhaseAnalysis::Write(int timestep) } -void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B) +void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B, double GreyPorosity) { Fx = force_x; Fy = force_y; @@ -60,8 +60,9 @@ void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double rho_w = rhoB; nu_n = (tauA-0.5)/3.f; nu_w = (tauB-0.5)/3.f; - gamma_wn = 5.796*alpha; + gamma_wn = 6.0*alpha; beta = B; + grey_porosity = GreyPorosity; } void GreyPhaseAnalysis::Basic(){ @@ -71,9 +72,6 @@ void GreyPhaseAnalysis::Basic(){ kmin=1; kmax=Nz-1; imin=jmin=1; - double count_w = 0.0; - double count_n = 0.0; - Water_local.reset(); Oil_local.reset(); for (k=kmin; kid[n] > 0 ){ // compute density - double pressure = Pressure(n); - double rho_n = Rho_n(n); - double rho_w = Rho_w(n); + double pressure = Pressure(n); + double nA = Rho_n(n); + double nB = Rho_w(n); double porosity = Porosity(n); - double vel_x = Vel_x(n); - double vel_y = Vel_y(n); - double vel_z = Vel_z(n); - Water_local.p += pressure*(rho_w)/(rho_n+rho_w); - Water_local.M += rho_w*porosity; - Water_local.Px += rho_w*vel_x; - Water_local.Py += rho_w*vel_y; - Water_local.Pz += rho_w*vel_z; - Oil_local.p += pressure*(rho_n)/(rho_n+rho_w); - Oil_local.M += rho_n*porosity; - Oil_local.Px += rho_n*vel_x; - Oil_local.Py += rho_n*vel_y; - Oil_local.Pz += rho_n*vel_z; + Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB); + Water_local.M += rho_w*nB*porosity; + Water_local.Px += rho_w*nB*Vel_x(n); + Water_local.Py += rho_w*nB*Vel_y(n); + Water_local.Pz += rho_w*nB*Vel_z(n); + Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB); + Oil_local.M += rho_n*nA*porosity; + Oil_local.Px += rho_n*nA*Vel_x(n); + Oil_local.Py += rho_n*nA*Vel_y(n); + Oil_local.Pz += rho_n*nA*Vel_z(n); } } @@ -164,10 +159,11 @@ void GreyPhaseAnalysis::Basic(){ force_mag = 1.0; } saturation=Water.M/(Water.M + Oil.M); // assume constant density - water_flow_rate=saturation*(Water.Px*dir_x + Water.Py*dir_y + Water.Pz*dir_z)/Water.M; - oil_flow_rate=(1.0-saturation)*(Oil.Px*dir_x + Oil.Py*dir_y + Oil.Pz*dir_z)/Oil.M; + water_flow_rate=grey_porosity*saturation*(Water.Px*dir_x + Water.Py*dir_y + Water.Pz*dir_z)/Water.M; + oil_flow_rate =grey_porosity*(1.0-saturation)*(Oil.Px*dir_x + Oil.Py*dir_y + Oil.Pz*dir_z)/Oil.M; double h = Dm->voxel_length; + //TODO check if need greyporosity or domain porosity ? - compare to analytical solution double krn = h*h*nu_n*oil_flow_rate / force_mag ; double krw = h*h*nu_w*water_flow_rate / force_mag; //printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow); diff --git a/analysis/GreyPhase.h b/analysis/GreyPhase.h index 5583bc1a..4aca756d 100644 --- a/analysis/GreyPhase.h +++ b/analysis/GreyPhase.h @@ -35,6 +35,7 @@ public: double nu_n, nu_w; double gamma_wn, beta; double Fx, Fy, Fz; + double grey_porosity; // outputs double saturation,water_flow_rate, oil_flow_rate; @@ -42,14 +43,14 @@ public: GreyPhase Water, Oil; GreyPhase Water_local, Oil_local; //........................................................................... - int Nx,Ny,Nz; - IntArray PhaseID; // Phase ID array + int Nx,Ny,Nz; + //IntArray PhaseID; // Phase ID array DoubleArray SDs; // contains porosity map DoubleArray Porosity; // contains porosity map DoubleArray Rho_n; // density field DoubleArray Rho_w; // density field - DoubleArray Phi; // phase indicator field - DoubleArray DelPhi; // Magnitude of Gradient of the phase indicator field + //DoubleArray Phi; // phase indicator field + //DoubleArray DelPhi; // Magnitude of Gradient of the phase indicator field DoubleArray Pressure; // pressure field DoubleArray Vel_x; // velocity field DoubleArray Vel_y; @@ -58,7 +59,7 @@ public: GreyPhaseAnalysis(std::shared_ptr Dm); ~GreyPhaseAnalysis(); - void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta); + void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta, double GreyPorosity); void Basic(); void Write(int time); diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 2e8eef1a..a5efe816 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -177,12 +177,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist // double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index a2395e61..5f6d3633 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1959,7 +1959,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist } //Calculate pressure for MRT model - pressure=rho/3.f; + pressure=rho/3.f/porosity; //-------------------- MRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); @@ -2457,7 +2457,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int f } //Calculate pressure for Incompressible-MRT model - pressure=rho/3.f; + pressure=rho/3.f/porosity; //-------------------- IMRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); diff --git a/cpu/GreyscaleColor.cpp b/cpu/GreyscaleColor.cpp index 48f84e09..a04fb22a 100644 --- a/cpu/GreyscaleColor.cpp +++ b/cpu/GreyscaleColor.cpp @@ -1,7 +1,7 @@ #include extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Velocity, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Velocity,double *Pressure, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ @@ -494,6 +494,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, d Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; + Pressure[n] = rho/3.f/porosity; //........................................................................ //..............carry out relaxation process.............................. @@ -709,7 +710,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, d } extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Velocity, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Velocity,double *Pressure, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ @@ -1148,6 +1149,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; + Pressure[n] = rho/3.f/porosity; //........................................................................ //..............carry out relaxation process.............................. diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 63b3ccc8..57452bbb 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -1990,7 +1990,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double * } //Calculate pressure for MRT model - pressure=rho/3.f; + pressure=rho/3.f/porosity; //-------------------- MRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); @@ -2496,7 +2496,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, i } //Calculate pressure for Incompressible-MRT model - pressure=rho/3.f; + pressure=rho/3.f/porosity; //-------------------- IMRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index 2af5760b..b3398d32 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -6,7 +6,7 @@ //Model-1 & 4 __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double *Pressure, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ @@ -512,6 +512,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; + Pressure[n] = rho/3.f/porosity; //........................................................................ //..............carry out relaxation process.............................. @@ -767,7 +768,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma //Model-1 & 4 __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, double *Pressure, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Gx, double Gy, double Gz, int strideY, int strideZ, int start, int finish, int Np){ int ijk,nn,n; @@ -1217,6 +1218,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; + Pressure[n] = rho/3.f/porosity; //........................................................................ //..............carry out relaxation process.............................. @@ -2918,14 +2920,14 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, //Model-1 & 4 extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, double *Pressure, double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, + dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor<<>>(Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm, Vel, Pressure, rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff, alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); if (cudaSuccess != err){ @@ -2937,14 +2939,15 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl //Model-1 & 4 extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, - double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, + double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np){ //cudaProfilerStart(); //cudaFuncSetCacheConfig(dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor, cudaFuncCachePreferL1); - dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel, + dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor<<>>(d_neighborList, Map, dist, Aq, Bq, Den, Phi, GreySolidGrad, Poros, Perm,Vel,Pressure, + rhoA, rhoB, tauA, tauB, tauA_eff, tauB_eff,alpha, beta, Fx, Fy, Fz, strideY, strideZ, start, finish, Np); cudaError_t err = cudaGetLastError(); diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index b83c0c53..6685943e 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -12,7 +12,7 @@ Two-fluid greyscale color lattice boltzmann model ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0), -Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),greyMode(0),Lx(0),Ly(0),Lz(0),comm(COMM) +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { REVERSE_FLOW_DIRECTION = false; } @@ -37,7 +37,6 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ Restart=false; din=dout=1.0; flux=0.0; - greyMode=false; // Color Model parameters if (greyscaleColor_db->keyExists( "timestepMax" )){ @@ -80,12 +79,6 @@ void ScaLBL_GreyscaleColorModel::ReadParams(string filename){ if (greyscaleColor_db->keyExists( "flux" )){ flux = greyscaleColor_db->getScalar( "flux" ); } - if (greyscaleColor_db->keyExists("GreySolidLabels")&& - greyscaleColor_db->keyExists("GreySolidAffinity")&& - greyscaleColor_db->keyExists("PorosityList")&& - greyscaleColor_db->keyExists("PermeabilityList")){ - greyMode = true; - } inletA=1.f; inletB=0.f; outletA=0.f; @@ -211,7 +204,6 @@ void ScaLBL_GreyscaleColorModel::ReadInput(){ if (rank == 0) cout << "Domain set." << endl; - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); } void ScaLBL_GreyscaleColorModel::AssignComponentLabels() @@ -624,13 +616,11 @@ void ScaLBL_GreyscaleColorModel::Create(){ ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - if (greyMode==true){ - //ScaLBL_AllocateDeviceMemory((void **) &GreySolidPhi, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &GreySolidGrad, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Porosity_dvc, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Permeability_dvc, sizeof(double)*Np); - } //ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + //ScaLBL_AllocateDeviceMemory((void **) &GreySolidPhi, sizeof(double)*Nx*Ny*Nz); + ScaLBL_AllocateDeviceMemory((void **) &GreySolidGrad, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Porosity_dvc, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Permeability_dvc, sizeof(double)*Np); //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); @@ -670,10 +660,9 @@ void ScaLBL_GreyscaleColorModel::Create(){ // initialize phi based on PhaseLabel (include solid component labels) AssignComponentLabels();//do open/black/grey nodes initialization - if (greyMode==true){ - AssignGreySolidLabels(); - AssignGreyPoroPermLabels(); - } + AssignGreySolidLabels(); + AssignGreyPoroPermLabels(); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); } void ScaLBL_GreyscaleColorModel::Initialize(){ @@ -761,7 +750,7 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); } } - ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); + //ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double)); } void ScaLBL_GreyscaleColorModel::Run(){ @@ -934,20 +923,14 @@ void ScaLBL_GreyscaleColorModel::Run(){ } // Halo exchange for phase field ScaLBL_Comm_Regular->SendHalo(Phi); - if (greyMode==true){ - //Model-1&4 - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, - rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ////Model-2&3 - //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, - // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } - else{ - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } + //Model-1&4 + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity,Pressure, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(Phi); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -964,20 +947,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - if (greyMode==true){ - //Model-1&4 - ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, - rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ////Model-2&3 - //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, - // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - } - else{ - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - } + + //Model-1&4 + ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity,Pressure, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAodd_GreyscaleColor(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); @@ -998,20 +976,14 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } ScaLBL_Comm_Regular->SendHalo(Phi); - if (greyMode==true){ - //Model-1&4 - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, - rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - ////Model-2&3 - //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, - // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } - else{ - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); - } + //Model-1&4 + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity,Pressure, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm_Regular->RecvHalo(Phi); ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); @@ -1028,20 +1000,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->D3Q19_Reflection_BC_z(fq); ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq); } - if (greyMode==true){ - //Model-1&4 - ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity, - rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - ////Model-2&3 - //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, - // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, - // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - } - else{ - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); - } + + //Model-1&4 + ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidGrad,Porosity_dvc,Permeability_dvc,Velocity,Pressure, + rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); + ////Model-2&3 + //ScaLBL_D3Q19_AAeven_GreyscaleColor(dvcMap, fq, Aq, Bq, Den, Phi,GreySolidPhi,Porosity_dvc,Permeability_dvc,Velocity, + // rhoA, rhoB, tauA, tauB,tauA_eff, tauB_eff, + // alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); //************************************************************************ @@ -1049,6 +1016,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ printf("%i %f \n",timestep,din); + } + + if (timestep%analysis_interval == 0){ + ScaLBL_Comm->RegularLayout(Map,Porosity_dvc,Averages->Porosity); ScaLBL_Comm->RegularLayout(Map,Pressure,Averages->Pressure); ScaLBL_Comm->RegularLayout(Map,&Den[0],Averages->Rho_n); ScaLBL_Comm->RegularLayout(Map,&Den[Np],Averages->Rho_w); @@ -1056,7 +1027,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Averages->Vel_y); ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); Averages->Basic(); - } + } // allow initial ramp-up to get closer to steady state if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){ @@ -1077,8 +1048,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ force_mag = 1.0; } double current_saturation = Averages->saturation; - double volA = current_saturation*Mask->Porosity()*Mask->Volume; - double volB = (1.0-current_saturation)*Mask->Porosity()*Mask->Volume; + double volA = current_saturation*GreyPorosity; + double volB = (1.0-current_saturation)*GreyPorosity; double flow_rate_A = Averages->oil_flow_rate; double flow_rate_B = Averages->water_flow_rate; double Ca = fabs(muA*flow_rate_A + muB*flow_rate_B)/(6.0*alpha); @@ -1105,7 +1076,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ Fz *= 1e-3/force_mag; } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } if ( isSteady ){ @@ -1143,44 +1114,48 @@ void ScaLBL_GreyscaleColorModel::Run(){ printf("** WRITE STEADY POINT *** "); printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); double h = Dm->voxel_length; + // pressures - /*double pA = Averages->gnb.p; - double pB = Averages->gwb.p; - double pAc = Averages->gnc.p; - double pBc = Averages->gwc.p; + double pA = Averages->Oil.p; + double pB = Averages->Water.p; double pAB = (pA-pB)/(h*6.0*alpha); - double pAB_connected = (pAc-pBc)/(h*6.0*alpha); - // connected contribution - double Vol_nc = Averages->gnc.V/Dm->Volume; - double Vol_wc = Averages->gwc.V/Dm->Volume; - double Vol_nd = Averages->gnd.V/Dm->Volume; - double Vol_wd = Averages->gwd.V/Dm->Volume; - double Mass_n = Averages->gnc.M + Averages->gnd.M; - double Mass_w = Averages->gwc.M + Averages->gwd.M; - double vAc_x = Averages->gnc.Px/Mass_n; - double vAc_y = Averages->gnc.Py/Mass_n; - double vAc_z = Averages->gnc.Pz/Mass_n; - double vBc_x = Averages->gwc.Px/Mass_w; - double vBc_y = Averages->gwc.Py/Mass_w; - double vBc_z = Averages->gwc.Pz/Mass_w; - // disconnected contribution - double vAd_x = Averages->gnd.Px/Mass_n; - double vAd_y = Averages->gnd.Py/Mass_n; - double vAd_z = Averages->gnd.Pz/Mass_n; - double vBd_x = Averages->gwd.Px/Mass_w; - double vBd_y = Averages->gwd.Py/Mass_w; - double vBd_z = Averages->gwd.Pz/Mass_w; - - double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); - double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); - double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); - double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); - - double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); - double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); - - double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); - double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); + + // -------- The following quantities may not make sense for greyscale simulation -----------// +// double pAc = Averages->gnc.p; +// double pBc = Averages->gwc.p; +// double pAB_connected = (pAc-pBc)/(h*6.0*alpha); +// // connected contribution +// double Vol_nc = Averages->gnc.V/Dm->Volume; +// double Vol_wc = Averages->gwc.V/Dm->Volume; +// double Vol_nd = Averages->gnd.V/Dm->Volume; +// double Vol_wd = Averages->gwd.V/Dm->Volume; +// double Mass_n = Averages->gnc.M + Averages->gnd.M; +// double Mass_w = Averages->gwc.M + Averages->gwd.M; +// double vAc_x = Averages->gnc.Px/Mass_n; +// double vAc_y = Averages->gnc.Py/Mass_n; +// double vAc_z = Averages->gnc.Pz/Mass_n; +// double vBc_x = Averages->gwc.Px/Mass_w; +// double vBc_y = Averages->gwc.Py/Mass_w; +// double vBc_z = Averages->gwc.Pz/Mass_w; +// // disconnected contribution +// double vAd_x = Averages->gnd.Px/Mass_n; +// double vAd_y = Averages->gnd.Py/Mass_n; +// double vAd_z = Averages->gnd.Pz/Mass_n; +// double vBd_x = Averages->gwd.Px/Mass_w; +// double vBd_y = Averages->gwd.Py/Mass_w; +// double vBd_z = Averages->gwd.Pz/Mass_w; +// +// double flow_rate_A_connected = Vol_nc*(vAc_x*dir_x + vAc_y*dir_y + vAc_z*dir_z); +// double flow_rate_B_connected = Vol_wc*(vBc_x*dir_x + vBc_y*dir_y + vBc_z*dir_z); +// double flow_rate_A_disconnected = (Vol_nd)*(vAd_x*dir_x + vAd_y*dir_y + vAd_z*dir_z); +// double flow_rate_B_disconnected = (Vol_wd)*(vBd_x*dir_x + vBd_y*dir_y + vBd_z*dir_z); +// +// double kAeff_connected = h*h*muA*flow_rate_A_connected/(force_mag); +// double kBeff_connected = h*h*muB*flow_rate_B_connected/(force_mag); +// +// double kAeff_disconnected = h*h*muA*flow_rate_A_disconnected/(force_mag); +// double kBeff_disconnected = h*h*muB*flow_rate_B_disconnected/(force_mag); +// //---------------------------------------------------------------------------------------// double kAeff = h*h*muA*(flow_rate_A)/(force_mag); double kBeff = h*h*muB*(flow_rate_B)/(force_mag); @@ -1196,11 +1171,12 @@ void ScaLBL_GreyscaleColorModel::Run(){ WriteHeader=true; kr_log_file = fopen("relperm.csv","a"); if (WriteHeader) - fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n"); + fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water cap.pressure.norm pressure.drop Ca M\n"); - fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility); + fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n", + CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,pAB,viscous_pressure_drop,Ca,Mobility); fclose(kr_log_file); - */ + printf(" Measured capillary number %f \n ",Ca); } if (SET_CAPILLARY_NUMBER ){ @@ -1213,7 +1189,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ Fz *= 1e-3/force_mag; } if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); - Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } @@ -1298,45 +1274,47 @@ void ScaLBL_GreyscaleColorModel::Run(){ // ************************************************************************ } -double ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ - +void ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str()); Mask->Decomp(Filename); for (int i=0; iid[i]; // save what was read for (int i=0; iid[i] = Mask->id[i]; // save what was read AssignComponentLabels(); + AssignGreySolidLabels(); + AssignGreyPoroPermLabels(); + Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); - double Count = 0.0; - double PoreCount = 0.0; - for (int k=1; kComm, Count); - PoreCount=sumReduce( Dm->Comm, PoreCount); - - if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); + //NOTE in greyscale simulations, water may have multiple labels (e.g. 2, 21, 22, etc) + //so the saturaiton calculation is not that straightforward +// double Count = 0.0; +// double PoreCount = 0.0; +// for (int k=1; kComm, Count); +// PoreCount=sumReduce( Dm->Comm, PoreCount); +// if (rank==0) printf(" new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount); ScaLBL_D3Q19_Init(fq, Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL); - ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); + //ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double)); - double saturation = Count/PoreCount; - return saturation; + //double saturation = Count/PoreCount; + //return saturation; } double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil){ diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index d7298c10..d7043257 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -39,7 +39,6 @@ public: double Fx,Fy,Fz,flux; double din,dout,inletA,inletB,outletA,outletB; double GreyPorosity; - bool greyMode;//run greyColor model if true int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -87,7 +86,7 @@ private: void AssignComponentLabels(); void AssignGreySolidLabels(); void AssignGreyPoroPermLabels(); - double ImageInit(std::string filename); + void ImageInit(std::string filename); double MorphInit(const double beta, const double morph_delta); double SeedPhaseField(const double seed_water_in_oil); double MorphOpenConnected(double target_volume_change); From 78d17e053858dc1e7050e3938cf1a75856234870 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 11 Oct 2020 21:08:39 -0400 Subject: [PATCH 247/270] made a few small changes --- analysis/GreyPhase.cpp | 12 ++++++------ models/GreyscaleColorModel.cpp | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/analysis/GreyPhase.cpp b/analysis/GreyPhase.cpp index 1e341a1a..68e0c01b 100644 --- a/analysis/GreyPhase.cpp +++ b/analysis/GreyPhase.cpp @@ -87,14 +87,14 @@ void GreyPhaseAnalysis::Basic(){ double porosity = Porosity(n); Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB); Water_local.M += rho_w*nB*porosity; - Water_local.Px += rho_w*nB*Vel_x(n); - Water_local.Py += rho_w*nB*Vel_y(n); - Water_local.Pz += rho_w*nB*Vel_z(n); + Water_local.Px += porosity*rho_w*nB*Vel_x(n); + Water_local.Py += porosity*rho_w*nB*Vel_y(n); + Water_local.Pz += porosity*rho_w*nB*Vel_z(n); Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB); Oil_local.M += rho_n*nA*porosity; - Oil_local.Px += rho_n*nA*Vel_x(n); - Oil_local.Py += rho_n*nA*Vel_y(n); - Oil_local.Pz += rho_n*nA*Vel_z(n); + Oil_local.Px += porosity*rho_n*nA*Vel_x(n); + Oil_local.Py += porosity*rho_n*nA*Vel_y(n); + Oil_local.Pz += porosity*rho_n*nA*Vel_z(n); } } diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 6685943e..9cb011e2 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -663,6 +663,7 @@ void ScaLBL_GreyscaleColorModel::Create(){ AssignGreySolidLabels(); AssignGreyPoroPermLabels(); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); + ScaLBL_Comm->RegularLayout(Map,Porosity_dvc,Averages->Porosity);//porosity doesn't change over time } void ScaLBL_GreyscaleColorModel::Initialize(){ @@ -1015,11 +1016,10 @@ void ScaLBL_GreyscaleColorModel::Run(){ PROFILE_STOP("Update"); if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){ - printf("%i %f \n",timestep,din); + printf("%i %.5g \n",timestep,din); } if (timestep%analysis_interval == 0){ - ScaLBL_Comm->RegularLayout(Map,Porosity_dvc,Averages->Porosity); ScaLBL_Comm->RegularLayout(Map,Pressure,Averages->Pressure); ScaLBL_Comm->RegularLayout(Map,&Den[0],Averages->Rho_n); ScaLBL_Comm->RegularLayout(Map,&Den[Np],Averages->Rho_w); @@ -1075,7 +1075,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ Fy *= 1e-3/force_mag; Fz *= 1e-3/force_mag; } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + if (rank == 0) printf(" -- adjust force by factor %.5g \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } @@ -1097,8 +1097,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev)); delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume)); if (rank==0){ - printf(" Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP); - printf(" log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); + printf(" Enabling endpoint adaptation: krA = %.5g, krB = %.5g \n",krA_TMP,krB_TMP); + printf(" log(kr)=%.5g, volume=%.5g, TARGET log(kr)=%.5g, volume change=%.5g \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume)); } } log_krA_prev = log_krA; @@ -1112,7 +1112,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ */ if (rank==0){ printf("** WRITE STEADY POINT *** "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + printf("Ca = %.5g, (previous = %.5g) \n",Ca,Ca_previous); double h = Dm->voxel_length; // pressures @@ -1177,7 +1177,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,pAB,viscous_pressure_drop,Ca,Mobility); fclose(kr_log_file); - printf(" Measured capillary number %f \n ",Ca); + printf(" Measured capillary number %.5g \n ",Ca); } if (SET_CAPILLARY_NUMBER ){ Fx *= capillary_number / Ca; @@ -1188,7 +1188,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ Fy *= 1e-3/force_mag; Fz *= 1e-3/force_mag; } - if (rank == 0) printf(" -- adjust force by factor %f \n ",capillary_number / Ca); + if (rank == 0) printf(" -- adjust force by factor %.5g \n ",capillary_number / Ca); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); greyscaleColor_db->putVector("F",{Fx,Fy,Fz}); } @@ -1198,7 +1198,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ else{ if (rank==0){ printf("** Continue to simulate steady *** \n "); - printf("Ca = %f, (previous = %f) \n",Ca,Ca_previous); + printf("Ca = %.5g, (previous = %.5g) \n",Ca,Ca_previous); } } morph_timesteps=0; @@ -1226,7 +1226,7 @@ void ScaLBL_GreyscaleColorModel::Run(){ delta_volume = volA*Dm->Volume - initial_volume; CURRENT_MORPH_TIMESTEPS += analysis_interval; double massChange = SeedPhaseField(seed_water); - if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target); + if (rank==0) printf("***Seed water in oil %.5g, volume change %.5g / %.5g ***\n", massChange, delta_volume, delta_volume_target); } if ( (delta_volume - delta_volume_target)/delta_volume_target > 0.0 ){ @@ -1284,6 +1284,7 @@ void ScaLBL_GreyscaleColorModel::ImageInit(std::string Filename){ AssignGreySolidLabels(); AssignGreyPoroPermLabels(); Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta,GreyPorosity); + ScaLBL_Comm->RegularLayout(Map,Porosity_dvc,Averages->Porosity); //NOTE in greyscale simulations, water may have multiple labels (e.g. 2, 21, 22, etc) //so the saturaiton calculation is not that straightforward @@ -1382,7 +1383,7 @@ double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil count= sumReduce( Dm->Comm, count); mass_loss= sumReduce( Dm->Comm, mass_loss); - if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count); + if (rank == 0) printf("Remove mass %.5g from %.5g voxels \n",mass_loss,count); // Need to initialize Aq, Bq, Den, Phi directly //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double)); From 2934872910db13ca34817f7ea9f7038ba22f15b0 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 11 Oct 2020 23:52:07 -0400 Subject: [PATCH 248/270] correct how pressure is averaged --- analysis/GreyPhase.cpp | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/analysis/GreyPhase.cpp b/analysis/GreyPhase.cpp index 68e0c01b..dab2ad9b 100644 --- a/analysis/GreyPhase.cpp +++ b/analysis/GreyPhase.cpp @@ -71,9 +71,13 @@ void GreyPhaseAnalysis::Basic(){ // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; imin=jmin=1; + if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; + if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; Water_local.reset(); Oil_local.reset(); + double count_w = 0.0; + double count_n = 0.0; for (k=kmin; kid[n] > 0 ){ // compute density - double pressure = Pressure(n); double nA = Rho_n(n); double nB = Rho_w(n); + double phi = (nA-nB)/(nA+nB); double porosity = Porosity(n); - Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB); Water_local.M += rho_w*nB*porosity; Water_local.Px += porosity*rho_w*nB*Vel_x(n); Water_local.Py += porosity*rho_w*nB*Vel_y(n); Water_local.Pz += porosity*rho_w*nB*Vel_z(n); - Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB); Oil_local.M += rho_n*nA*porosity; Oil_local.Px += porosity*rho_n*nA*Vel_x(n); Oil_local.Py += porosity*rho_n*nA*Vel_y(n); Oil_local.Pz += porosity*rho_n*nA*Vel_z(n); + if ( phi > 0.99 ){ + Oil_local.p += Pressure(n); + //Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB); + count_n += 1.0; + } + else if ( phi < -0.99 ){ + Water_local.p += Pressure(n); + //Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB); + count_w += 1.0; + } } } } } Oil.M=sumReduce( Dm->Comm, Oil_local.M); - Oil.p=sumReduce( Dm->Comm, Oil_local.p); Oil.Px=sumReduce( Dm->Comm, Oil_local.Px); Oil.Py=sumReduce( Dm->Comm, Oil_local.Py); Oil.Pz=sumReduce( Dm->Comm, Oil_local.Pz); Water.M=sumReduce( Dm->Comm, Water_local.M); - Water.p=sumReduce( Dm->Comm, Water_local.p); Water.Px=sumReduce( Dm->Comm, Water_local.Px); Water.Py=sumReduce( Dm->Comm, Water_local.Py); Water.Pz=sumReduce( Dm->Comm, Water_local.Pz); - Oil.p /= Oil.M; - Water.p /= Water.M; + + //Oil.p /= Oil.M; + //Water.p /= Water.M; + count_w=sumReduce( Dm->Comm, count_w); + count_n=sumReduce( Dm->Comm, count_n); + if (count_w > 0.0) + Water.p=sumReduce( Dm->Comm, Water_local.p) / count_w; + else + Water.p = 0.0; + if (count_n > 0.0) + Oil.p=sumReduce( Dm->Comm, Oil_local.p) / count_n; + else + Oil.p = 0.0; // check for NaN bool err=false; From a245d9c9d6910b81733b4d3c83ef9d7dce1f28a2 Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Mon, 12 Oct 2020 09:11:08 -0400 Subject: [PATCH 249/270] update electro tests --- tests/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c536a7ec..a9869130 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,11 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest ) #ADD_LBPM_EXECUTABLE( DataAggregator ) #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel ) ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar ) +ADD_LBPM_EXECUTABLE( TestPoissonSolver ) +ADD_LBPM_EXECUTABLE( TestIonModel ) +ADD_LBPM_EXECUTABLE( TestNernstPlanck ) +ADD_LBPM_EXECUTABLE( TestPNP_Stokes ) + CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_DIR}/cylindertest COPYONLY ) From 057f7e4f574bddbd1aa9f69fc76139d4f2c30e0a Mon Sep 17 00:00:00 2001 From: JamesEMcclure Date: Mon, 12 Oct 2020 09:32:39 -0400 Subject: [PATCH 250/270] fix CMakelists --- tests/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a9869130..0a561abc 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,10 +36,6 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest ) #ADD_LBPM_EXECUTABLE( DataAggregator ) #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel ) ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar ) -ADD_LBPM_EXECUTABLE( TestPoissonSolver ) -ADD_LBPM_EXECUTABLE( TestIonModel ) -ADD_LBPM_EXECUTABLE( TestNernstPlanck ) -ADD_LBPM_EXECUTABLE( TestPNP_Stokes ) CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_DIR}/cylindertest COPYONLY ) From b5f9ad5a6c65826b097616a6977665bafc2385c7 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 15 Oct 2020 11:54:26 -0400 Subject: [PATCH 251/270] PoissonSolver: remove extra setSlice for halo layer in Initialize() --- models/PoissonSolver.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 3e11de0a..a84b0916 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -390,14 +390,15 @@ void ScaLBL_Poisson::Initialize(){ delete [] psi_host; //extra treatment for halo layer - if (BoundaryCondition==1){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); - } - } + //maybe not useful + //if (BoundaryCondition==1){ + // if (Dm->kproc()==0){ + // ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); + // } + // if (Dm->kproc() == nprocz-1){ + // ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); + // } + //} } void ScaLBL_Poisson::Run(double *ChargeDensity){ From 21a0ec8c0ba1ad852e0bd5158558e2d7dd437125 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 16 Oct 2020 12:50:28 -0400 Subject: [PATCH 252/270] add a routine to write convergence log for Poisson solver --- models/PoissonSolver.cpp | 55 ++++++++++++++++++++++++++++++++-------- models/PoissonSolver.h | 4 +++ 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index a84b0916..aca6ec95 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -8,7 +8,7 @@ ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0), epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0), -chargeDen_dummy(0), +chargeDen_dummy(0),WriteLog(0), nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM) { @@ -36,6 +36,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ Vin = 1.0; //Boundary-z (inlet) electric potential Vout = 1.0; //Boundary-Z (outlet) electric potential chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3] + WriteLog = 0; // LB-Poisson Model parameters if (electric_db->keyExists( "timestepMax" )){ @@ -53,6 +54,15 @@ void ScaLBL_Poisson::ReadParams(string filename){ if (electric_db->keyExists( "DummyChargeDen" )){ chargeDen_dummy = electric_db->getScalar( "DummyChargeDen" ); } + if (electric_db->keyExists( "WriteLog" )){ + auto writelog = electric_db->getScalar( "WriteLog" ); + if (writelog !="True" && writelog !="False"){ + ERROR("Error: LB-Poisson Solver: WriteLog cannot be identified! Uesage: WriteLog is either True or False.\n"); + } + else if (writelog =="True"){ + WriteLog = 1; + } + } // Read solid boundary condition specific to Poisson equation BoundaryConditionSolid = 1; @@ -390,15 +400,14 @@ void ScaLBL_Poisson::Initialize(){ delete [] psi_host; //extra treatment for halo layer - //maybe not useful - //if (BoundaryCondition==1){ - // if (Dm->kproc()==0){ - // ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); - // } - // if (Dm->kproc() == nprocz-1){ - // ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); - // } - //} + if (BoundaryCondition==1){ + if (Dm->kproc()==0){ + ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); + } + if (Dm->kproc() == nprocz-1){ + ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); + } + } } void ScaLBL_Poisson::Run(double *ChargeDensity){ @@ -457,6 +466,9 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ psi_avg_previous = psi_avg; } } + if(WriteLog==1){ + getConvergenceLog(timestep,error); + } //************************************************************************/ //stoptime = MPI_Wtime(); @@ -476,6 +488,29 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ } +void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){ + if (rank==0){ + bool WriteHeader=false; + TIMELOG = fopen("PoissonSolver_Convergence.csv","r"); + if (TIMELOG != NULL) + fclose(TIMELOG); + else + WriteHeader=true; + + TIMELOG = fopen("PoissonSolver_Convergence.csv","a+"); + if (WriteHeader) + { + fprintf(TIMELOG,"Timestep Error\n"); + fprintf(TIMELOG,"%i %.5g\n",timestep,error); + fflush(TIMELOG); + } + else { + fprintf(TIMELOG,"%i %.5g\n",timestep,error); + fflush(TIMELOG); + } + } +} + void ScaLBL_Poisson::SolveElectricPotentialAAodd(){ ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index dfd098d5..f4d2efd7 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -3,6 +3,7 @@ */ #include #include +#include #include #include #include @@ -45,6 +46,7 @@ public: double epsilon0,epsilon0_LB,epsilonR,epsilon_LB; double Vin, Vout; double chargeDen_dummy;//for debugging + short WriteLog; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; @@ -79,6 +81,7 @@ private: char LocalRankFilename[40]; char LocalRestartFile[40]; char OutputFilename[200]; + FILE *TIMELOG; //int rank,nprocs; void LoadParams(std::shared_ptr db0); @@ -90,5 +93,6 @@ private: //void SolveElectricField(); void SolvePoissonAAodd(double *ChargeDensity); void SolvePoissonAAeven(double *ChargeDensity); + void getConvergenceLog(int timestep,double error); }; From 7cc2be826e60e6456fd5df507017004f3ce2f41f Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 18 Oct 2020 12:20:15 -0400 Subject: [PATCH 253/270] PoissonSolver: remove extra setSlice for halo layer in Initialize() --- models/PoissonSolver.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index aca6ec95..0ef13c84 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -400,14 +400,14 @@ void ScaLBL_Poisson::Initialize(){ delete [] psi_host; //extra treatment for halo layer - if (BoundaryCondition==1){ - if (Dm->kproc()==0){ - ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); - } - } + //if (BoundaryCondition==1){ + // if (Dm->kproc()==0){ + // ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0); + // } + // if (Dm->kproc() == nprocz-1){ + // ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1); + // } + //} } void ScaLBL_Poisson::Run(double *ChargeDensity){ From 69f319ab5c27baca55f535959c4c7d8dcf1d5741 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 5 Nov 2020 21:51:29 -0500 Subject: [PATCH 254/270] work in progress; add CPU ion flux BC --- common/ScaLBL.cpp | 22 ++++ common/ScaLBL.h | 10 ++ cpu/D3Q7BC.cpp | 52 ++++++++ models/IonModel.cpp | 287 ++++++++++++++++++++++++++++---------------- models/IonModel.h | 7 +- 5 files changed, 272 insertions(+), 106 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 8b0abeba..e946e975 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2109,3 +2109,25 @@ void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double } } } + +void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, int time){ + if (kproc == 0) { + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, sendCount_z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, sendCount_z, N); + } + } +} + +void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, int time){ + if (kproc == nprocz-1){ + if (time%2==0){ + ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, sendCount_Z, N); + } + else{ + ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, sendCount_Z, N); + } + } +} diff --git a/common/ScaLBL.h b/common/ScaLBL.h index edd601e8..4b34655d 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -214,6 +214,14 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, in extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np); + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np); + class ScaLBL_Communicator{ public: //...................................................................................... @@ -279,6 +287,8 @@ public: void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout); void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time); void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time); + void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, int time); + void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, int time); // Debugging and unit testing functions void PrintD3Q19(); diff --git a/cpu/D3Q7BC.cpp b/cpu/D3Q7BC.cpp index 161e6a5c..85012df0 100644 --- a/cpu/D3Q7BC.cpp +++ b/cpu/D3Q7BC.cpp @@ -223,3 +223,55 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, in dist[nr6] = f6; } } + +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, int count, int Np){ + //NOTE: FluxIn is the inward flux + for (int idx=0; idx &num_iter){ IonDiffusivity.push_back(1.0e-9);//user-input diffusivity has physical unit [m^2/sec] IonValence.push_back(1);//algebraic valence charge IonConcentration.push_back(1.0e-3);//user-input ion concentration has physical unit [mol/m^3] - Cin.push_back(1.0e-3);//user-input inlet boundary ion concentration;unit [mol/m^3] - Cout.push_back(1.0e-3);//user-input outlet boundary ion concentration;unit [mol/m^3] //tau.push_back(0.5+k2_inv*time_conv/(h*1.0e-6)/(h*1.0e-6)*IonDiffusivity[0]); time_conv.push_back((tau[0]-0.5)/k2_inv*(h*h*1.0e-12)/IonDiffusivity[0]); fluidVelx_dummy = 0.0;//for debugging, unit [m/sec] @@ -162,54 +160,78 @@ void ScaLBL_IonModel::ReadParams(string filename,vector &num_iter){ } // Read boundary condition for ion transport // BC = 0: normal periodic BC - // BC = 1: fixed inlet and outlet ion concentration - BoundaryCondition = 0; - if (ion_db->keyExists( "BC" )){ - BoundaryCondition = ion_db->getScalar( "BC" ); + // BC = 1: fixed ion concentration; unit=[mol/m^3] + // BC = 2: fixed ion flux (inward flux); unit=[mol/m^2/sec] + BoundaryConditionInlet.push_back(0); + BoundaryConditionOutlet.push_back(0); + //Inlet + if (ion_db->keyExists( "BC_InletList" )){ + BoundaryConditionInlet = ion_db->getVector( "BC_InletList" ); + if (BoundaryConditionInlet.size()!=number_ion_species){ + ERROR("Error: number_ion_species and BC_InletList must be of the same length! \n"); + } + unsigned short int BC_inlet_min = *min_element(BoundaryConditionInlet.begin(),BoundaryConditionInlet.end()); + unsigned short int BC_inlet_max = *max_element(BoundaryConditionInlet.begin(),BoundaryConditionInlet.end()); + if (BC_inlet_min == 0 && BC_inlet_max>0){ + ERROR("Error: BC_InletList: mix of periodic, ion concentration and flux BC is not supported! \n"); + } + if (BC_inlet_min>0){ + //read in inlet values Cin + if (ion_db->keyExists("InletValueList")){ + Cin = ion_db->getVector( "InletValueList" ); + if (Cin.size()!=number_ion_species){ + ERROR("Error: number_ion_species and InletValueList must be the same length! \n"); + } + } + else { + ERROR("Error: Non-periodic BCs are specified but InletValueList cannot be found! \n"); + } + for (unsigned int i=0;ikeyExists("CinList")){ - Cin.clear(); - Cin = ion_db->getVector( "CinList" ); - if (Cin.size()!=number_ion_species){ - ERROR("Error: number_ion_species and CinList must be the same length! \n"); + //Outlet + if (ion_db->keyExists( "BC_OutletList" )){ + BoundaryConditionOutlet = ion_db->getVector( "BC_OutletList" ); + if (BoundaryConditionOutlet.size()!=number_ion_species){ + ERROR("Error: number_ion_species and BC_OutletList must be of the same length! \n"); + } + unsigned short int BC_outlet_min = *min_element(BoundaryConditionOutlet.begin(),BoundaryConditionOutlet.end()); + unsigned short int BC_outlet_max = *max_element(BoundaryConditionOutlet.begin(),BoundaryConditionOutlet.end()); + if (BC_outlet_min == 0 && BC_outlet_max>0){ + ERROR("Error: BC_OutletList: mix of periodic, ion concentration and flux BC is not supported! \n"); + } + if (BC_outlet_min>0){ + //read in outlet values Cout + if (ion_db->keyExists("OutletValueList")){ + Cout = ion_db->getVector( "OutletValueList" ); + if (Cout.size()!=number_ion_species){ + ERROR("Error: number_ion_species and OutletValueList must be the same length! \n"); + } } - else{ - for (int i=0; ikeyExists("CoutList")){ - Cout.clear(); - Cout = ion_db->getVector( "CoutList" ); - if (Cout.size()!=number_ion_species){ - ERROR("Error: number_ion_species and CoutList must be the same length! \n"); - } - else{ - for (int i=0; ikeyExists( "BC" )){ - BoundaryCondition = ion_db->getScalar( "BC" ); + // BC = 1: fixed ion concentration; unit=[mol/m^3] + // BC = 2: fixed ion flux (inward flux); unit=[mol/m^2/sec] + BoundaryConditionInlet.push_back(0); + BoundaryConditionOutlet.push_back(0); + //Inlet + if (ion_db->keyExists( "BC_InletList" )){ + BoundaryConditionInlet = ion_db->getVector( "BC_InletList" ); + if (BoundaryConditionInlet.size()!=number_ion_species){ + ERROR("Error: number_ion_species and BC_InletList must be of the same length! \n"); + } + unsigned short int BC_inlet_min = *min_element(BoundaryConditionInlet.begin(),BoundaryConditionInlet.end()); + unsigned short int BC_inlet_max = *max_element(BoundaryConditionInlet.begin(),BoundaryConditionInlet.end()); + if (BC_inlet_min == 0 && BC_inlet_max>0){ + ERROR("Error: BC_InletList: mix of periodic, ion concentration and flux BC is not supported! \n"); + } + if (BC_inlet_min>0){ + //read in inlet values Cin + if (ion_db->keyExists("InletValueList")){ + Cin = ion_db->getVector( "InletValueList" ); + if (Cin.size()!=number_ion_species){ + ERROR("Error: number_ion_species and InletValueList must be the same length! \n"); + } + } + else { + ERROR("Error: Non-periodic BCs are specified but InletValueList cannot be found! \n"); + } + for (unsigned int i=0;ikeyExists("CinList")){ - Cin.clear(); - Cin = ion_db->getVector( "CinList" ); - if (Cin.size()!=number_ion_species){ - ERROR("Error: number_ion_species and CinList must be the same length! \n"); + //Outlet + if (ion_db->keyExists( "BC_OutletList" )){ + BoundaryConditionOutlet = ion_db->getVector( "BC_OutletList" ); + if (BoundaryConditionOutlet.size()!=number_ion_species){ + ERROR("Error: number_ion_species and BC_OutletList must be of the same length! \n"); + } + unsigned short int BC_outlet_min = *min_element(BoundaryConditionOutlet.begin(),BoundaryConditionOutlet.end()); + unsigned short int BC_outlet_max = *max_element(BoundaryConditionOutlet.begin(),BoundaryConditionOutlet.end()); + if (BC_outlet_min == 0 && BC_outlet_max>0){ + ERROR("Error: BC_OutletList: mix of periodic, ion concentration and flux BC is not supported! \n"); + } + if (BC_outlet_min>0){ + //read in outlet values Cout + if (ion_db->keyExists("OutletValueList")){ + Cout = ion_db->getVector( "OutletValueList" ); + if (Cout.size()!=number_ion_species){ + ERROR("Error: number_ion_species and OutletValueList must be the same length! \n"); + } } - else{ - for (int i=0; ikeyExists("CoutList")){ - Cout.clear(); - Cout = ion_db->getVector( "CoutList" ); - if (Cout.size()!=number_ion_species){ - ERROR("Error: number_ion_species and CoutList must be the same length! \n"); - } - else{ - for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); - Dm->BoundaryCondition = BoundaryCondition; - Mask->BoundaryCondition = BoundaryCondition; + + unsigned short int BC_inlet_min = *min_element(BoundaryConditionInlet.begin(),BoundaryConditionInlet.end()); + unsigned short int BC_outlet_min = *min_element(BoundaryConditionOutlet.begin(),BoundaryConditionOutlet.end()); + if (BC_inlet_min==0 && BC_outlet_min==0){ + Dm->BoundaryCondition = 0; + Mask->BoundaryCondition = 0; + } + else if (BC_inlet_min>0 && BC_outlet_min>0){ + Dm->BoundaryCondition = 1; + Mask->BoundaryCondition = 1; + } + else { //i.e. periodic and non-periodic BCs are mixed + ERROR("Error: check the type of inlet and outlet boundary condition! Mixed periodic and non-periodic BCs are found. \n"); + } + Dm->CommInit(); MPI_Barrier(comm); @@ -714,12 +771,24 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + //--------------------------------------- Set boundary conditions -------------------------------------// + switch (BoundaryConditionInlet[i]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; } - //-------------------------// + switch (BoundaryConditionOutlet[i]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + } + //----------------------------------------------------------------------------------------------------// ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); @@ -742,12 +811,24 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np],ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); - // Set boundary conditions - if (BoundaryCondition == 1){ - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + //--------------------------------------- Set boundary conditions -------------------------------------// + switch (BoundaryConditionInlet[i]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; } - //-------------------------// + switch (BoundaryConditionOutlet[i]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + } + //----------------------------------------------------------------------------------------------------// ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); diff --git a/models/IonModel.h b/models/IonModel.h index 5a568182..59382002 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -39,7 +39,6 @@ public: //bool Restart,pBC; int timestep; vector timestepMax; - int BoundaryCondition; int BoundaryConditionSolid; double h;//domain resolution, unit [um/lu] double kb,electron_charge,T,Vt; @@ -49,11 +48,13 @@ public: double Ex_dummy,Ey_dummy,Ez_dummy; int number_ion_species; + vector BoundaryConditionInlet; + vector BoundaryConditionOutlet; vector IonDiffusivity;//User input unit [m^2/sec] vector IonValence; vector IonConcentration;//unit [mol/m^3] - vector Cin;//unit [mol/m^3] - vector Cout;//unit [mol/m^3] + vector Cin;//inlet boundary value, can be either concentration [mol/m^3] or flux [mol/m^2/sec] + vector Cout;//outlet boundary value, can be either concentration [mol/m^3] or flux [mol/m^2/sec] vector tau; vector time_conv; From f9c32855e5d51eaf45096707e6eb7d43b8c2fbfe Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 7 Nov 2020 16:41:07 -0500 Subject: [PATCH 255/270] save the work; validation to be continued --- models/IonModel.cpp | 107 +++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/models/IonModel.cpp b/models/IonModel.cpp index ec004967..cfb69366 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -721,15 +721,6 @@ void ScaLBL_IonModel::Initialize(){ ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np); } - if (rank==0) printf("*****************************************************\n"); - if (rank==0) printf("LB Ion Transport Solver: \n"); - for (int i=0; iRecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); //--------------------------------------- Set boundary conditions -------------------------------------// - switch (BoundaryConditionInlet[i]){ - case 1: - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - break; - case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - break; + if (BoundaryConditionInlet[ic]>0){ + switch (BoundaryConditionInlet[ic]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + } } - switch (BoundaryConditionOutlet[i]){ - case 1: - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); - break; - case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); - break; + if (BoundaryConditionOutlet[ic]>0){ + switch (BoundaryConditionOutlet[ic]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + } } //----------------------------------------------------------------------------------------------------// ScaLBL_D3Q7_AAodd_IonConcentration(NeighborList, &fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); @@ -812,21 +841,25 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->RecvD3Q7AA(fq, ic); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); //--------------------------------------- Set boundary conditions -------------------------------------// - switch (BoundaryConditionInlet[i]){ - case 1: - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - break; - case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); - break; + if (BoundaryConditionInlet[ic]>0){ + switch (BoundaryConditionInlet[ic]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + break; + } } - switch (BoundaryConditionOutlet[i]){ - case 1: - ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); - break; - case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); - break; + if (BoundaryConditionOutlet[ic]>0){ + switch (BoundaryConditionOutlet[ic]){ + case 1: + ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + case 2: + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + break; + } } //----------------------------------------------------------------------------------------------------// ScaLBL_D3Q7_AAeven_IonConcentration(&fq[ic*Np*7],&Ci[ic*Np], 0, ScaLBL_Comm->LastExterior(), Np); From e93b941d9edb7931e9950df654f8ecc2c3f88fd9 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 19 Nov 2020 13:17:31 -0500 Subject: [PATCH 256/270] save the work; fluxBC for Ion solver still does not fully agree COMSOL --- common/ScaLBL.cpp | 12 +++--- common/ScaLBL.h | 12 +++--- cpu/D3Q7BC.cpp | 102 +++++++++++++++++++++++++++++++++++++------- models/IonModel.cpp | 12 +++--- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index e946e975..ca4e89b6 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -2110,24 +2110,24 @@ void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double } } -void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, int time){ +void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){ if (kproc == 0) { if (time%2==0){ - ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, sendCount_z, N); + ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N); } else{ - ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, sendCount_z, N); + ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N); } } } -void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, int time){ +void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){ if (kproc == nprocz-1){ if (time%2==0){ - ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, sendCount_Z, N); + ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N); } else{ - ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, sendCount_Z, N); + ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N); } } } diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 4b34655d..3ef8c31c 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -214,13 +214,13 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, in extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, int count, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np); -extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, int count, int Np); +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np); -extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np); +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np); class ScaLBL_Communicator{ public: @@ -287,8 +287,8 @@ public: void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout); void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time); void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time); - void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, int time); - void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, int time); + void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time); + void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time); // Debugging and unit testing functions void PrintD3Q19(); diff --git a/cpu/D3Q7BC.cpp b/cpu/D3Q7BC.cpp index 85012df0..39c95172 100644 --- a/cpu/D3Q7BC.cpp +++ b/cpu/D3Q7BC.cpp @@ -224,54 +224,124 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, in } } -extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, int count, int Np){ +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){ //NOTE: FluxIn is the inward flux + double f0,f1,f2,f3,f4,f5,f6; + double fsum_partial; + int n; + double uz; for (int idx=0; idxD3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); break; case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], tau[ic], &Velocity[2*Np], timestep); break; } } @@ -813,7 +813,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); break; case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], tau[ic], &Velocity[2*Np], timestep); break; } } @@ -847,7 +847,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->D3Q7_Ion_Concentration_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); break; case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Flux_BC_z(NeighborList, &fq[ic*Np*7], Cin[ic], tau[ic], &Velocity[2*Np], timestep); break; } } @@ -857,7 +857,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ ScaLBL_Comm->D3Q7_Ion_Concentration_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); break; case 2: - ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], timestep); + ScaLBL_Comm->D3Q7_Ion_Flux_BC_Z(NeighborList, &fq[ic*Np*7], Cout[ic], tau[ic], &Velocity[2*Np], timestep); break; } } From 9693bb94400567d4f8e455c0202ff16e5dca6fec Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 3 Dec 2020 02:15:20 -0500 Subject: [PATCH 257/270] CPU only; revert the definition of pressure back to normal, without the factor of voxel porosity --- cpu/GreyscaleColor.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpu/GreyscaleColor.cpp b/cpu/GreyscaleColor.cpp index a04fb22a..cf01cd84 100644 --- a/cpu/GreyscaleColor.cpp +++ b/cpu/GreyscaleColor.cpp @@ -494,7 +494,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, d Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; - Pressure[n] = rho/3.f/porosity; + //Pressure[n] = rho/3.f/porosity; + Pressure[n] = rho/3.f; //........................................................................ //..............carry out relaxation process.............................. @@ -1149,7 +1150,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; - Pressure[n] = rho/3.f/porosity; + //Pressure[n] = rho/3.f/porosity; + Pressure[n] = rho/3.f; //........................................................................ //..............carry out relaxation process.............................. From 1528aa7435115ac7a835fca293d62e9ef79d0ed9 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 3 Dec 2020 19:21:03 -0500 Subject: [PATCH 258/270] CPU only: flux BC validated and done --- cpu/D3Q7BC.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpu/D3Q7BC.cpp b/cpu/D3Q7BC.cpp index 39c95172..2917bab4 100644 --- a/cpu/D3Q7BC.cpp +++ b/cpu/D3Q7BC.cpp @@ -242,7 +242,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double uz = VelocityZ[n]; //................................................... - f5 =(FluxIn+(1.0-0.5/tau)*f6-uz*fsum_partial)/(1.0-0.5/tau+uz); + f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); dist[6*Np+n] = f5; } } @@ -266,7 +266,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double uz = VelocityZ[n]; //................................................... - f6 =(FluxIn+(1.0-0.5/tau)*f5+uz*fsum_partial)/(1.0-0.5/tau-uz); + f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); dist[5*Np+n] = f6; } } @@ -300,7 +300,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, fsum_partial = f0+f1+f2+f3+f4+f6; uz = VelocityZ[n]; //................................................... - f5 =(FluxIn+(1.0-0.5/tau)*f6-uz*fsum_partial)/(1.0-0.5/tau+uz); + f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); // Unknown distributions nr5 = d_neighborList[n+4*Np]; @@ -338,7 +338,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, fsum_partial = f0+f1+f2+f3+f4+f5; uz = VelocityZ[n]; //................................................... - f6 =(FluxIn+(1.0-0.5/tau)*f5+uz*fsum_partial)/(1.0-0.5/tau-uz); + f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); // unknown distributions nr6 = d_neighborList[n+5*Np]; From de60bae8fcfc3c9e6538c72095742eae98fb0971 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Thu, 3 Dec 2020 20:28:09 -0500 Subject: [PATCH 259/270] GPU fluxBC is ready too --- gpu/D3Q7BC.cu | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/gpu/D3Q7BC.cu b/gpu/D3Q7BC.cu index 8d27f7d5..63aee2bb 100644 --- a/gpu/D3Q7BC.cu +++ b/gpu/D3Q7BC.cu @@ -265,6 +265,131 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList } } +__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np) +{ + //NOTE: FluxIn is the inward flux + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + double fsum_partial; + double uz; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f6 = dist[5*Np+n]; + fsum_partial = f0+f1+f2+f3+f4+f6; + uz = VelocityZ[n]; + //................................................... + f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); + dist[6*Np+n] = f5; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np) +{ + //NOTE: FluxIn is the inward flux + int idx,n; + double f0,f1,f2,f3,f4,f5,f6; + double fsum_partial; + double uz; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + f1 = dist[2*Np+n]; + f2 = dist[1*Np+n]; + f3 = dist[4*Np+n]; + f4 = dist[3*Np+n]; + f5 = dist[6*Np+n]; + fsum_partial = f0+f1+f2+f3+f4+f5; + uz = VelocityZ[n]; + //................................................... + f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); + dist[5*Np+n] = f6; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np) +{ + //NOTE: FluxIn is the inward flux + int idx, n; + int nread,nr5; + double f0,f1,f2,f3,f4,f5,f6; + double fsum_partial; + double uz; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + nread = d_neighborList[n+5*Np]; + f6 = dist[nread]; + + fsum_partial = f0+f1+f2+f3+f4+f6; + uz = VelocityZ[n]; + //................................................... + f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); + + // Unknown distributions + nr5 = d_neighborList[n+4*Np]; + dist[nr5] = f5; + } +} + +__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np) +{ + //NOTE: FluxIn is the inward flux + int idx, n; + int nread,nr6; + double f0,f1,f2,f3,f4,f5,f6; + double fsum_partial; + double uz; + idx = blockIdx.x*blockDim.x + threadIdx.x; + if (idx < count){ + n = list[idx]; + f0 = dist[n]; + + nread = d_neighborList[n]; + f1 = dist[nread]; + + nread = d_neighborList[n+2*Np]; + f3 = dist[nread]; + + nread = d_neighborList[n+4*Np]; + f5 = dist[nread]; + + nread = d_neighborList[n+Np]; + f2 = dist[nread]; + + nread = d_neighborList[n+3*Np]; + f4 = dist[nread]; + + fsum_partial = f0+f1+f2+f3+f4+f5; + uz = VelocityZ[n]; + //................................................... + f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); + + // unknown distributions + nr6 = d_neighborList[n+5*Np]; + dist[nr6] = f6; + } +} //************************************************************************* extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){ @@ -375,3 +500,38 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, in } } +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<>>(list, dist, FluxIn, tau, VelocityZ, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<>>(list, dist, FluxIn, tau, VelocityZ, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err)); + } +} + +extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){ + int GRID = count / 512 + 1; + dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err)); + } +} From 7ef18bfea3e1390e506d431e898757728bd2b975 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 4 Dec 2020 00:37:02 -0500 Subject: [PATCH 260/270] make the WriteLog in PoissonSolver a bool type to facilitate input --- models/PoissonSolver.cpp | 12 +++--------- models/PoissonSolver.h | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index 0ef13c84..c50d8816 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -36,7 +36,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ Vin = 1.0; //Boundary-z (inlet) electric potential Vout = 1.0; //Boundary-Z (outlet) electric potential chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3] - WriteLog = 0; + WriteLog = false; // LB-Poisson Model parameters if (electric_db->keyExists( "timestepMax" )){ @@ -55,13 +55,7 @@ void ScaLBL_Poisson::ReadParams(string filename){ chargeDen_dummy = electric_db->getScalar( "DummyChargeDen" ); } if (electric_db->keyExists( "WriteLog" )){ - auto writelog = electric_db->getScalar( "WriteLog" ); - if (writelog !="True" && writelog !="False"){ - ERROR("Error: LB-Poisson Solver: WriteLog cannot be identified! Uesage: WriteLog is either True or False.\n"); - } - else if (writelog =="True"){ - WriteLog = 1; - } + WriteLog = electric_db->getScalar( "WriteLog" ); } // Read solid boundary condition specific to Poisson equation @@ -466,7 +460,7 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){ psi_avg_previous = psi_avg; } } - if(WriteLog==1){ + if(WriteLog==true){ getConvergenceLog(timestep,error); } diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index f4d2efd7..241e871a 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -46,7 +46,7 @@ public: double epsilon0,epsilon0_LB,epsilonR,epsilon_LB; double Vin, Vout; double chargeDen_dummy;//for debugging - short WriteLog; + bool WriteLog; int Nx,Ny,Nz,N,Np; int rank,nprocx,nprocy,nprocz,nprocs; From cb0efd10e362dc29d077e4f843c458afe9d2ae7b Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Fri, 4 Dec 2020 17:54:57 -0500 Subject: [PATCH 261/270] CPU/GPU single/two-fluid, MRT only; revert the definition of pressure back to normal --- cpu/Greyscale.cpp | 6 ++++-- gpu/Greyscale.cu | 6 ++++-- gpu/GreyscaleColor.cu | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cpu/Greyscale.cpp b/cpu/Greyscale.cpp index 5f6d3633..4aab97e2 100644 --- a/cpu/Greyscale.cpp +++ b/cpu/Greyscale.cpp @@ -1959,7 +1959,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist } //Calculate pressure for MRT model - pressure=rho/3.f/porosity; + //pressure=rho/3.f/porosity; + pressure=rho/3.f; //-------------------- MRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); @@ -2457,7 +2458,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int f } //Calculate pressure for Incompressible-MRT model - pressure=rho/3.f/porosity; + //pressure=rho/3.f/porosity; + pressure=rho/3.f; //-------------------- IMRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); diff --git a/gpu/Greyscale.cu b/gpu/Greyscale.cu index 57452bbb..edf80a16 100644 --- a/gpu/Greyscale.cu +++ b/gpu/Greyscale.cu @@ -1990,7 +1990,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double * } //Calculate pressure for MRT model - pressure=rho/3.f/porosity; + //pressure=rho/3.f/porosity; + pressure=rho/3.f; //-------------------- MRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); @@ -2496,7 +2497,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, i } //Calculate pressure for Incompressible-MRT model - pressure=rho/3.f/porosity; + //pressure=rho/3.f/porosity; + pressure=rho/3.f; //-------------------- IMRT collison where body force has NO higher-order terms -------------// m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1); diff --git a/gpu/GreyscaleColor.cu b/gpu/GreyscaleColor.cu index b3398d32..69c9a080 100644 --- a/gpu/GreyscaleColor.cu +++ b/gpu/GreyscaleColor.cu @@ -512,7 +512,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; - Pressure[n] = rho/3.f/porosity; + //Pressure[n] = rho/3.f/porosity; + Pressure[n] = rho/3.f; //........................................................................ //..............carry out relaxation process.............................. @@ -1218,7 +1219,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, Velocity[n] = ux; Velocity[Np+n] = uy; Velocity[2*Np+n] = uz; - Pressure[n] = rho/3.f/porosity; + //Pressure[n] = rho/3.f/porosity; + Pressure[n] = rho/3.f; //........................................................................ //..............carry out relaxation process.............................. From f3ba90337a33836366888edda4290bbc45f29744 Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sun, 13 Dec 2020 19:34:56 -0500 Subject: [PATCH 262/270] to be compiled: add basic restart and write visualization functionality --- models/GreyscaleColorModel.cpp | 169 +++++++++++++++++++++++++++++++++ models/GreyscaleColorModel.h | 1 + 2 files changed, 170 insertions(+) diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 9cb011e2..9ebad61c 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -9,6 +9,12 @@ Two-fluid greyscale color lattice boltzmann model #include #include +template +void DeleteArray( const TYPE *p ) +{ + delete [] p; +} + ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM): rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0), @@ -786,6 +792,15 @@ void ScaLBL_GreyscaleColorModel::Run(){ double initial_volume = 0.0; double delta_volume = 0.0; double delta_volume_target = 0.0; + + //TODO -------- For temporary use - should be included in the analysis framework later ------------- + if (analysis_db->keyExists( "visualization_interval" )){ + visualization_interval = analysis_db->getScalar( "visualization_interval" ); + } + if (analysis_db->keyExists( "restart_interval" )){ + restart_interval = analysis_db->getScalar( "restart_interval" ); + } + //------------------------------------------------------------------------------------------------- /* history for morphological algoirthm */ double KRA_MORPH_FACTOR=0.5; @@ -1015,6 +1030,51 @@ void ScaLBL_GreyscaleColorModel::Run(){ //************************************************************************ PROFILE_STOP("Update"); + //TODO For temporary use - writing Restart and Vis files should be included in the analysis framework in the future + if (timestep%restart_interval==0){ + //Use rank=0 write out Restart.db + if (rank==0) { + greyscaleColor_db->putScalar("timestep",timestep); + greyscaleColor_db->putScalar( "Restart", true ); + current_db->putDatabase("Color", greyscaleColor_db); + std::ofstream OutStream("Restart.db"); + current_db->print(OutStream, ""); + OutStream.close(); + + } + //Write out Restart data. + std::shared_ptr cDen; + std::shared_ptr cfq; + cDen = std::shared_ptr(new double[2*Np], DeleteArray); + cfq = std::shared_ptr(new double[19*Np],DeleteArray); + ScaLBL_CopyToHost(cDen.get(),Den,2*Np*sizeof(double));// Copy restart data to the CPU + ScaLBL_CopyToHost(cfq.get(), fq,19*Np*sizeof(double));// Copy restart data to the CPU + + ofstream RESTARTFILE(LocalRestartFile,ios::binary); + double value; + for (int n=0; n visData; + fillHalo fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1); + + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + auto SignDistVar = std::make_shared(); + auto PressureVar = std::make_shared(); + auto PhaseVar = std::make_shared(); + + // Create the MeshDataStruct + IO::initialize("","silo","false"); + visData.resize(1); + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + + // create a temp data for copy from device + DoubleArray DataTemp(Nx,Ny,Nz); + + if (vis_db->getWithDefault( "save_phase_field", true )){ + + PhaseVar->name = "Phase"; + PhaseVar->type = IO::VariableType::VolumeVariable; + PhaseVar->dim = 1; + PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PaseVar); + + ASSERT(visData[0].vars[0]->name=="Phase"); + Array& PhaseData = visData[0].vars[0]->data; + ScaLBL_Comm->RegularLayout(Map,Phase,DataTemp); + fillData.copy(DataTemp,PhaseData); + } + + if (vis_db->getWithDefault( "save_pressure", false )){ + + PressureVar->name = "Pressure"; + PressureVar->type = IO::VariableType::VolumeVariable; + PressureVar->dim = 1; + PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(PressureVar); + + ASSERT(visData[0].vars[1]->name=="Pressure"); + Array& PressData = visData[0].vars[1]->data; + ScaLBL_Comm->RegularLayout(Map,Pressure,DataTemp); + fillData.copy(DataTemp,PressData); + } + + if (vis_db->getWithDefault( "save_velocity", false )){ + + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + + ASSERT(visData[0].vars[2]->name=="Velocity_x"); + ASSERT(visData[0].vars[3]->name=="Velocity_y"); + ASSERT(visData[0].vars[4]->name=="Velocity_z"); + Array& VelxData = visData[0].vars[2]->data; + Array& VelyData = visData[0].vars[3]->data; + Array& VelzData = visData[0].vars[4]->data; + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],DataTemp); + fillData.copy(DataTemp,VelxData); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],DataTemp); + fillData.copy(DataTemp,VelyData); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],DataTemp); + fillData.copy(DataTemp,VelzData); + } + + if (vis_db->getWithDefault( "save_distance", false )){ + + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VariableType::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(SignDistVar); + + ASSERT(visData[0].vars[5]->name=="SignDist"); + Array& SignData = visData[0].vars[5]->data; + fillData.copy(Averages.SDs,SignData); + } + + if (vis_db->getWithDefault( "write_silo", true )){ + IO::writeData( timestep, visData, Dm->Comm ); + } + + if (vis_db->getWithDefault( "save_8bit_raw", true )){ + //TODO + //char CurrentIDFilename[40]; + //sprintf(CurrentIDFilename,"id_t%d.raw",timestep); + //Averages.AggregateLabels(CurrentIDFilename); + } + +} + void ScaLBL_GreyscaleColorModel::WriteDebug(){ // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index d7043257..1ba92c8a 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -90,5 +90,6 @@ private: double MorphInit(const double beta, const double morph_delta); double SeedPhaseField(const double seed_water_in_oil); double MorphOpenConnected(double target_volume_change); + double WriteVisFiles(); }; From d02eff3017bda53b4053897f10e935299e2aa0ed Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Mon, 14 Dec 2020 01:08:59 -0500 Subject: [PATCH 263/270] done: add restart and write visfiles for greyscale Color --- common/ScaLBL.h | 2 ++ cpu/GreyscaleColor.cpp | 26 +++++++++++++++++++++++ gpu/GreyscaleColor.cu | 38 ++++++++++++++++++++++++++++++++++ models/GreyscaleColorModel.cpp | 28 +++++++++++++++---------- models/GreyscaleColorModel.h | 2 +- 5 files changed, 84 insertions(+), 12 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index a5efe816..3b129d06 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -186,6 +186,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta, double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np); +extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np); + // MRT MODEL extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz); diff --git a/cpu/GreyscaleColor.cpp b/cpu/GreyscaleColor.cpp index cf01cd84..057c8a7d 100644 --- a/cpu/GreyscaleColor.cpp +++ b/cpu/GreyscaleColor.cpp @@ -1338,6 +1338,32 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl } } +extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){ + int idx; + double nA,nB; + + for (idx=start; idx>>(Den, Aq, Bq, start, finish, Np); + cudaError_t err = cudaGetLastError(); + if (cudaSuccess != err){ + printf("CUDA error in ScaLBL_PhaseField_InitFromRestart: %s \n",cudaGetErrorString(err)); + } +} ////Model-2&3 //extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, // double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, diff --git a/models/GreyscaleColorModel.cpp b/models/GreyscaleColorModel.cpp index 9ebad61c..f1b37dad 100644 --- a/models/GreyscaleColorModel.cpp +++ b/models/GreyscaleColorModel.cpp @@ -673,13 +673,17 @@ void ScaLBL_GreyscaleColorModel::Create(){ } void ScaLBL_GreyscaleColorModel::Initialize(){ - - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); - //ScaLBL_D3Q19_GreyscaleColor_Init(fq, Porosity_dvc, Np); /* * This function initializes model */ + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + //ScaLBL_D3Q19_GreyscaleColor_Init(fq, Porosity_dvc, Np); + + if (rank==0) printf ("Initializing phase field \n"); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (Restart == true){ if (rank==0){ printf("Reading restart file! \n"); @@ -738,11 +742,11 @@ void ScaLBL_GreyscaleColorModel::Initialize(){ ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - } - if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + if (rank==0) printf ("Initializing phase field from Restart\n"); + ScaLBL_PhaseField_InitFromRestart(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_PhaseField_InitFromRestart(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } // establish reservoirs for external bC if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4 ){ @@ -794,6 +798,8 @@ void ScaLBL_GreyscaleColorModel::Run(){ double delta_volume_target = 0.0; //TODO -------- For temporary use - should be included in the analysis framework later ------------- + int visualization_interval = 50000; + int restart_interval = 100000; if (analysis_db->keyExists( "visualization_interval" )){ visualization_interval = analysis_db->getScalar( "visualization_interval" ); } @@ -1482,11 +1488,11 @@ void ScaLBL_GreyscaleColorModel::WriteVisFiles(){ PhaseVar->type = IO::VariableType::VolumeVariable; PhaseVar->dim = 1; PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(PaseVar); + visData[0].vars.push_back(PhaseVar); ASSERT(visData[0].vars[0]->name=="Phase"); Array& PhaseData = visData[0].vars[0]->data; - ScaLBL_Comm->RegularLayout(Map,Phase,DataTemp); + ScaLBL_CopyToHost(DataTemp.data(), Phi, sizeof(double)*Nx*Ny*Nz); fillData.copy(DataTemp,PhaseData); } @@ -1546,7 +1552,7 @@ void ScaLBL_GreyscaleColorModel::WriteVisFiles(){ ASSERT(visData[0].vars[5]->name=="SignDist"); Array& SignData = visData[0].vars[5]->data; - fillData.copy(Averages.SDs,SignData); + fillData.copy(Averages->SDs,SignData); } if (vis_db->getWithDefault( "write_silo", true )){ diff --git a/models/GreyscaleColorModel.h b/models/GreyscaleColorModel.h index 1ba92c8a..8c8b4eee 100644 --- a/models/GreyscaleColorModel.h +++ b/models/GreyscaleColorModel.h @@ -90,6 +90,6 @@ private: double MorphInit(const double beta, const double morph_delta); double SeedPhaseField(const double seed_water_in_oil); double MorphOpenConnected(double target_volume_change); - double WriteVisFiles(); + void WriteVisFiles(); }; From 6252822da25aadbaf9605c6d0cafbb0e99e26adf Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 26 Dec 2020 14:00:17 -0500 Subject: [PATCH 264/270] adding analysis capabilities for electrochemistry --- analysis/ElectroChemistry.cpp | 29 +++++++++++++++++++ analysis/ElectroChemistry.h | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 analysis/ElectroChemistry.cpp create mode 100644 analysis/ElectroChemistry.h diff --git a/analysis/ElectroChemistry.cpp b/analysis/ElectroChemistry.cpp new file mode 100644 index 00000000..571efc95 --- /dev/null +++ b/analysis/ElectroChemistry.cpp @@ -0,0 +1,29 @@ +#include "analysis/ElectroChemistry.h" + +void ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr Dm): + Dm(dm){ + Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; + Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; + + ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0); + ElectricalPotential.resize(Nx,Ny,Nz); ElectricalPotential.fill(0); + Pressure.resize(Nx,Ny,Nz); Pressure.fill(0); + Rho.resize(Nx,Ny,Nz); Rho.fill(0); + Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field + Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0); + Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0); + SDs.resize(Nx,Ny,Nz); SDs.fill(0); + + DoubleArray Rho; // density field + DoubleArray ChemicalPotential; // density field + DoubleArray ElectricalPotential; // density field + DoubleArray Pressure; // pressure field + DoubleArray Vel_x; // velocity field + DoubleArray Vel_y; + DoubleArray Vel_z; + DoubleArray SDs; +} + +void ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){ + +} diff --git a/analysis/ElectroChemistry.h b/analysis/ElectroChemistry.h new file mode 100644 index 00000000..d290cccc --- /dev/null +++ b/analysis/ElectroChemistry.h @@ -0,0 +1,52 @@ +/* + * Sub-phase averaging tools + */ + +#ifndef ElectroChem_INC +#define ElectroChem_INC + +#include +#include "common/Domain.h" +#include "common/Communication.h" +#include "analysis/analysis.h" +#include "analysis/distance.h" +#include "analysis/Minkowski.h" +#include "common/Utilities.h" +#include "common/MPI_Helpers.h" +#include "IO/MeshDatabase.h" +#include "IO/Reader.h" +#include "IO/Writer.h" + +class ElectroChemistryAnalyzer{ +public: + std::shared_ptr Dm; + double Volume; + // input variables + double rho_n, rho_w; + double nu_n, nu_w; + double gamma_wn, beta; + double Fx, Fy, Fz; + + //........................................................................... + int Nx,Ny,Nz; + DoubleArray Rho; // density field + DoubleArray ChemicalPotential; // density field + DoubleArray ElectricalPotential; // density field + DoubleArray Pressure; // pressure field + DoubleArray Vel_x; // velocity field + DoubleArray Vel_y; + DoubleArray Vel_z; + DoubleArray SDs; + + ElectroChemistryAnalyzer(std::shared_ptr Dm); + ~ElectroChemistryAnalyzer(); + + void SetParams(); + void Basic(); + void Write(int time); + +private: + FILE *TIMELOG; +} +#endif + From 1122b0480dffb2741d934c042bb7964c003fdd8a Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 26 Dec 2020 20:26:27 -0500 Subject: [PATCH 265/270] electrochem analyzer compiles --- analysis/ElectroChemistry.cpp | 34 ++++++++++++++++++++++++++++++++-- analysis/ElectroChemistry.h | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/analysis/ElectroChemistry.cpp b/analysis/ElectroChemistry.cpp index 571efc95..84d618b1 100644 --- a/analysis/ElectroChemistry.cpp +++ b/analysis/ElectroChemistry.cpp @@ -1,6 +1,6 @@ #include "analysis/ElectroChemistry.h" -void ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr Dm): +ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr dm): Dm(dm){ Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; @@ -22,8 +22,38 @@ void ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr DoubleArray Vel_y; DoubleArray Vel_z; DoubleArray SDs; + + if (Dm->rank()==0){ + bool WriteHeader=false; + TIMELOG = fopen("electrokinetic.csv","r"); + if (TIMELOG != NULL) + fclose(TIMELOG); + else + WriteHeader=true; + + TIMELOG = fopen("electrokinetic.csv","a+"); + if (WriteHeader) + { + // If timelog is empty, write a short header to list the averages + //fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n"); + fprintf(TIMELOG,"sw krw krn vw vn pw pn\n"); + } + } + } -void ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){ +ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){ + +} + +void ElectroChemistryAnalyzer::SetParams(){ + +} + +void ElectroChemistryAnalyzer::Basic(){ + +} + +void ElectroChemistryAnalyzer::Write(int time){ } diff --git a/analysis/ElectroChemistry.h b/analysis/ElectroChemistry.h index d290cccc..ad3a3578 100644 --- a/analysis/ElectroChemistry.h +++ b/analysis/ElectroChemistry.h @@ -47,6 +47,6 @@ public: private: FILE *TIMELOG; -} +}; #endif From 879f8637bfcacbdf5b647b48ab9060c94a24d546 Mon Sep 17 00:00:00 2001 From: James McClure Date: Sat, 26 Dec 2020 20:26:55 -0500 Subject: [PATCH 266/270] fix getVelocity --- models/StokesModel.cpp | 17 ++++++----------- models/StokesModel.h | 1 - 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index 964baaae..a365c46e 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -375,25 +375,20 @@ void ScaLBL_StokesModel::getVelocity(int timestep){ ScaLBL_D3Q19_Momentum(fq, Velocity, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - DoubleArray PhaseField(Nx,Ny,Nz); - - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField); - Velocity_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); + Velocity_LB_to_Phys(Velocity_x); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); sprintf(OutputFilename,"Velocity_X_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField); - Velocity_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); + Velocity_LB_to_Phys(Velocity_y); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); sprintf(OutputFilename,"Velocity_Y_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField); - Velocity_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); + Velocity_LB_to_Phys(Velocity_z); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); sprintf(OutputFilename,"Velocity_Z_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); } void ScaLBL_StokesModel::getVelocity_debug(int timestep){ diff --git a/models/StokesModel.h b/models/StokesModel.h index b7ad345e..6375d4ff 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -69,7 +69,6 @@ public: double *Pressure; //Minkowski Morphology; - DoubleArray Velocity_x; DoubleArray Velocity_y; DoubleArray Velocity_z; From 9826ef5624f65f05ae853399d129d654786506d7 Mon Sep 17 00:00:00 2001 From: James McClure Date: Tue, 29 Dec 2020 14:04:43 -0500 Subject: [PATCH 267/270] adding silo vis capabilities to electrochem --- analysis/ElectroChemistry.cpp | 119 ++++++++++++++++-- analysis/ElectroChemistry.h | 13 +- models/IonModel.cpp | 16 +-- models/IonModel.h | 7 +- models/PoissonSolver.cpp | 32 ++--- models/PoissonSolver.h | 8 +- models/StokesModel.cpp | 17 ++- models/StokesModel.h | 6 +- tests/TestIonModel.cpp | 2 +- tests/TestNernstPlanck.cpp | 6 +- tests/TestPNP_Stokes.cpp | 8 +- tests/TestPoissonSolver.cpp | 4 +- ...m_electrokinetic_SingleFluid_simulator.cpp | 17 +-- 13 files changed, 176 insertions(+), 79 deletions(-) diff --git a/analysis/ElectroChemistry.cpp b/analysis/ElectroChemistry.cpp index 84d618b1..b052d459 100644 --- a/analysis/ElectroChemistry.cpp +++ b/analysis/ElectroChemistry.cpp @@ -1,7 +1,11 @@ #include "analysis/ElectroChemistry.h" ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr dm): - Dm(dm){ + Dm(dm), + fillData(dm->Comm,dm->rank_info,{dm->Nx-2,dm->Ny-2,dm->Nz-2},{1,1,1},0,1) +{ + + MPI_Comm_dup(dm->Comm,&comm); Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; @@ -14,15 +18,6 @@ ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr dm): Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0); SDs.resize(Nx,Ny,Nz); SDs.fill(0); - DoubleArray Rho; // density field - DoubleArray ChemicalPotential; // density field - DoubleArray ElectricalPotential; // density field - DoubleArray Pressure; // pressure field - DoubleArray Vel_x; // velocity field - DoubleArray Vel_y; - DoubleArray Vel_z; - DoubleArray SDs; - if (Dm->rank()==0){ bool WriteHeader=false; TIMELOG = fopen("electrokinetic.csv","r"); @@ -36,7 +31,7 @@ ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr dm): { // If timelog is empty, write a short header to list the averages //fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n"); - fprintf(TIMELOG,"sw krw krn vw vn pw pn\n"); + fprintf(TIMELOG,"TBD TBD\n"); } } @@ -50,10 +45,108 @@ void ElectroChemistryAnalyzer::SetParams(){ } -void ElectroChemistryAnalyzer::Basic(){ +void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes){ + + Poisson.getElectricPotential(ElectricalPotential); + for (int ion=0; ion input_db, int timestep){ + auto vis_db = input_db->getDatabase( "Visualization" ); + char VisName[40]; + + IO::initialize("","silo","false"); + // Create the MeshDataStruct + visData.resize(1); + + visData[0].meshName = "domain"; + visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); + auto ElectricPotential = std::make_shared(); + auto IonConcentration = std::make_shared(); + auto VxVar = std::make_shared(); + auto VyVar = std::make_shared(); + auto VzVar = std::make_shared(); + + if (vis_db->getWithDefault( "save_electric_potential", true )){ + ElectricPotential->name = "ElectricPotential"; + ElectricPotential->type = IO::VariableType::VolumeVariable; + ElectricPotential->dim = 1; + ElectricPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(ElectricPotential); + } + + if (vis_db->getWithDefault( "save_concentration", true )){ + for (int ion=0; ionname = VisName; + IonConcentration->type = IO::VariableType::VolumeVariable; + IonConcentration->dim = 1; + IonConcentration->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(IonConcentration); + } + + } + if (vis_db->getWithDefault( "save_velocity", false )){ + VxVar->name = "Velocity_x"; + VxVar->type = IO::VariableType::VolumeVariable; + VxVar->dim = 1; + VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VxVar); + VyVar->name = "Velocity_y"; + VyVar->type = IO::VariableType::VolumeVariable; + VyVar->dim = 1; + VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VyVar); + VzVar->name = "Velocity_z"; + VzVar->type = IO::VariableType::VolumeVariable; + VzVar->dim = 1; + VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(VzVar); + } + + if (vis_db->getWithDefault( "save_electric_potential", true )){ + ASSERT(visData[0].vars[0]->name=="ElectricPotential"); + Poisson.getElectricPotential(ElectricalPotential); + Array& ElectricPotentialData = visData[0].vars[0]->data; + fillData.copy(ElectricalPotential,ElectricPotentialData); + } + + if (vis_db->getWithDefault( "save_concentration", true )){ + for (int ion=0; ionname = VisName; + ASSERT(visData[0].vars[1]->name==VisName); + Array& IonConcentrationData = visData[0].vars[1]->data; + Ion.getIonConcentration(Rho,ion); + fillData.copy(Rho,IonConcentrationData); + } + } + + if (vis_db->getWithDefault( "save_velocity", false )){ + ASSERT(visData[0].vars[2]->name=="Velocity_x"); + ASSERT(visData[0].vars[3]->name=="Velocity_y"); + ASSERT(visData[0].vars[4]->name=="Velocity_z"); + Stokes.getVelocity(Vel_x,Vel_y,Vel_z); + Array& VelxData = visData[0].vars[2]->data; + Array& VelyData = visData[0].vars[3]->data; + Array& VelzData = visData[0].vars[4]->data; + fillData.copy(Vel_x,VelxData); + fillData.copy(Vel_y,VelyData); + fillData.copy(Vel_z,VelzData); + } + + if (vis_db->getWithDefault( "write_silo", true )) + IO::writeData( timestep, visData, comm ); + +/* if (vis_db->getWithDefault( "save_8bit_raw", true )){ + char CurrentIDFilename[40]; + sprintf(CurrentIDFilename,"id_t%d.raw",timestep); + Averages.AggregateLabels(CurrentIDFilename); + } +*/ } diff --git a/analysis/ElectroChemistry.h b/analysis/ElectroChemistry.h index ad3a3578..fa404fb0 100644 --- a/analysis/ElectroChemistry.h +++ b/analysis/ElectroChemistry.h @@ -1,5 +1,5 @@ /* - * Sub-phase averaging tools + * averaging tools for electrochemistry */ #ifndef ElectroChem_INC @@ -16,9 +16,14 @@ #include "IO/MeshDatabase.h" #include "IO/Reader.h" #include "IO/Writer.h" +#include "models/IonModel.h" +#include "models/PoissonSolver.h" +#include "models/StokesModel.h" class ElectroChemistryAnalyzer{ public: + MPI_Comm comm; + int tag; std::shared_ptr Dm; double Volume; // input variables @@ -42,10 +47,12 @@ public: ~ElectroChemistryAnalyzer(); void SetParams(); - void Basic(); - void Write(int time); + void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes); + void WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr input_db, int timestep); private: + std::vector visData; + fillHalo fillData; FILE *TIMELOG; }; #endif diff --git a/models/IonModel.cpp b/models/IonModel.cpp index 852bc194..6fbb627e 100644 --- a/models/IonModel.cpp +++ b/models/IonModel.cpp @@ -900,17 +900,13 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){ //if (rank==0) printf("********************************************************\n"); } -void ScaLBL_IonModel::getIonConcentration(int timestep){ - //This function wirte out the data in a normal layout (by aggregating all decomposed domains) - DoubleArray PhaseField(Nx,Ny,Nz); - for (int ic=0; icRegularLayout(Map,&Ci[ic*Np],PhaseField); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - IonConcentration_LB_to_Phys(PhaseField); +void ScaLBL_IonModel::getIonConcentration(DoubleArray &IonConcentration, const int ic){ + //This function wirte out the data in a normal layout (by aggregating all decomposed domains) + + ScaLBL_Comm->RegularLayout(Map,&Ci[ic*Np],IonConcentration); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + IonConcentration_LB_to_Phys(IonConcentration); - sprintf(OutputFilename,"Ion%02i_Time_%i.raw",ic+1,timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); - } } void ScaLBL_IonModel::getIonConcentration_debug(int timestep){ diff --git a/models/IonModel.h b/models/IonModel.h index 59382002..4b370978 100644 --- a/models/IonModel.h +++ b/models/IonModel.h @@ -1,6 +1,10 @@ /* * Ion transporte LB Model */ + +#ifndef ScaLBL_IonModel_INC +#define ScaLBL_IonModel_INC + #include #include #include @@ -30,7 +34,7 @@ public: void Create(); void Initialize(); void Run(double *Velocity, double *ElectricField); - void getIonConcentration(int timestep); + void getIonConcentration(DoubleArray &IonConcentration, const int ic); void getIonConcentration_debug(int timestep); void DummyFluidVelocity(); void DummyElectricField(); @@ -95,3 +99,4 @@ private: void AssignIonConcentration_FromFile(double *Ci,const vector &File_ion); void IonConcentration_LB_to_Phys(DoubleArray &Den_reg); }; +#endif diff --git a/models/PoissonSolver.cpp b/models/PoissonSolver.cpp index c50d8816..b0dde2c7 100644 --- a/models/PoissonSolver.cpp +++ b/models/PoissonSolver.cpp @@ -587,38 +587,26 @@ void ScaLBL_Poisson::getElectricPotential_debug(int timestep){ fclose(OUTFILE); } -void ScaLBL_Poisson::getElectricPotential(int timestep){ +void ScaLBL_Poisson::getElectricPotential(DoubleArray &ReturnValues){ //This function wirte out the data in a normal layout (by aggregating all decomposed domains) - DoubleArray PhaseField(Nx,Ny,Nz); //ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField); - ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - sprintf(OutputFilename,"Electric_Potential_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); + ScaLBL_CopyToHost(ReturnValues.data(),Psi,sizeof(double)*Nx*Ny*Nz); } -void ScaLBL_Poisson::getElectricField(int timestep){ +void ScaLBL_Poisson::getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z){ - DoubleArray PhaseField(Nx,Ny,Nz); - - ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],PhaseField); - ElectricField_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],Values_x); + ElectricField_LB_to_Phys(Values_x); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"ElectricField_X_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); - ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],PhaseField); - ElectricField_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],Values_y); + ElectricField_LB_to_Phys(Values_y); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"ElectricField_Y_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); - ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],PhaseField); - ElectricField_LB_to_Phys(PhaseField); + ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],Values_z); + ElectricField_LB_to_Phys(Values_z); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"ElectricField_Z_Time_%i.raw",timestep); - Mask->AggregateLabels(OutputFilename,PhaseField); + } void ScaLBL_Poisson::getElectricField_debug(int timestep){ diff --git a/models/PoissonSolver.h b/models/PoissonSolver.h index 241e871a..74abd775 100644 --- a/models/PoissonSolver.h +++ b/models/PoissonSolver.h @@ -16,6 +16,9 @@ #include "analysis/Minkowski.h" #include "ProfilerApp.h" +#ifndef ScaLBL_POISSON_INC +#define ScaLBL_POISSON_INC + class ScaLBL_Poisson{ public: ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM); @@ -29,9 +32,9 @@ public: void Create(); void Initialize(); void Run(double *ChargeDensity); - void getElectricPotential(int timestep); + void getElectricPotential(DoubleArray &ReturnValues); void getElectricPotential_debug(int timestep); - void getElectricField(int timestep); + void getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z); void getElectricField_debug(int timestep); void DummyChargeDensity();//for debugging @@ -96,3 +99,4 @@ private: void getConvergenceLog(int timestep,double error); }; +#endif diff --git a/models/StokesModel.cpp b/models/StokesModel.cpp index a365c46e..086e3633 100644 --- a/models/StokesModel.cpp +++ b/models/StokesModel.cpp @@ -370,25 +370,22 @@ void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){ } } -void ScaLBL_StokesModel::getVelocity(int timestep){ +void ScaLBL_StokesModel::getVelocity(DoubleArray &Vel_x, DoubleArray &Vel_y, DoubleArray &Vel_z){ //get velocity in physical unit [m/sec] ScaLBL_D3Q19_Momentum(fq, Velocity, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x); - Velocity_LB_to_Phys(Velocity_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Vel_x); + Velocity_LB_to_Phys(Vel_x); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"Velocity_X_Time_%i.raw",timestep); - ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y); - Velocity_LB_to_Phys(Velocity_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Vel_y); + Velocity_LB_to_Phys(Vel_y); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"Velocity_Y_Time_%i.raw",timestep); - ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z); - Velocity_LB_to_Phys(Velocity_z); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Vel_z); + Velocity_LB_to_Phys(Vel_z); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - sprintf(OutputFilename,"Velocity_Z_Time_%i.raw",timestep); } void ScaLBL_StokesModel::getVelocity_debug(int timestep){ diff --git a/models/StokesModel.h b/models/StokesModel.h index 6375d4ff..8da373bd 100644 --- a/models/StokesModel.h +++ b/models/StokesModel.h @@ -1,6 +1,9 @@ /* * Multi-relaxation time LBM Model */ +#ifndef ScaLBL_StokesModel_INC +#define ScaLBL_StokesModel_INC + #include #include #include @@ -31,7 +34,7 @@ public: void Run(); void Run_Lite(double *ChargeDensity, double *ElectricField); void VelocityField(); - void getVelocity(int timestep); + void getVelocity(DoubleArray &Velx, DoubleArray &Vel_y, DoubleArray &Vel_z); void getVelocity_debug(int timestep); double CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField); @@ -86,3 +89,4 @@ private: void Velocity_LB_to_Phys(DoubleArray &Vel_reg); vector computeElectricForceAvg(double *ChargeDensity, double *ElectricField); }; +#endif \ No newline at end of file diff --git a/tests/TestIonModel.cpp b/tests/TestIonModel.cpp index 0b57ff1c..2a0a02a9 100644 --- a/tests/TestIonModel.cpp +++ b/tests/TestIonModel.cpp @@ -76,7 +76,7 @@ int main(int argc, char **argv) error = IonModel.CalIonDenConvergence(ci_avg_previous); } } - IonModel.getIonConcentration(timestep); + IonModel.getIonConcentration_debug(timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); diff --git a/tests/TestNernstPlanck.cpp b/tests/TestNernstPlanck.cpp index 96d9c388..def67d5b 100644 --- a/tests/TestNernstPlanck.cpp +++ b/tests/TestNernstPlanck.cpp @@ -87,9 +87,9 @@ int main(int argc, char **argv) } } - PoissonSolver.getElectricPotential(timestep); - PoissonSolver.getElectricField(timestep); - IonModel.getIonConcentration(timestep); + PoissonSolver.getElectricPotential_debug(timestep); + PoissonSolver.getElectricField_debug(timestep); + IonModel.getIonConcentration_debug(timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); diff --git a/tests/TestPNP_Stokes.cpp b/tests/TestPNP_Stokes.cpp index 0e0c3e81..bf05f73c 100644 --- a/tests/TestPNP_Stokes.cpp +++ b/tests/TestPNP_Stokes.cpp @@ -107,10 +107,10 @@ int main(int argc, char **argv) } } - PoissonSolver.getElectricPotential(timestep); - PoissonSolver.getElectricField(timestep); - IonModel.getIonConcentration(timestep); - StokesModel.getVelocity(timestep); + PoissonSolver.getElectricPotential_debug(timestep); + PoissonSolver.getElectricField_debug(timestep); + IonModel.getIonConcentration_debug(timestep); + StokesModel.getVelocity_debug(timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); diff --git a/tests/TestPoissonSolver.cpp b/tests/TestPoissonSolver.cpp index 8753a7b0..32353f65 100644 --- a/tests/TestPoissonSolver.cpp +++ b/tests/TestPoissonSolver.cpp @@ -59,8 +59,8 @@ int main(int argc, char **argv) PoissonSolver.DummyChargeDensity(); PoissonSolver.Run(PoissonSolver.ChargeDensityDummy); - PoissonSolver.getElectricPotential(1); - PoissonSolver.getElectricField(1); + PoissonSolver.getElectricPotential_debug(1); + PoissonSolver.getElectricField_debug(1); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); diff --git a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp index 0c1054a2..2b3726a4 100644 --- a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp +++ b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp @@ -12,6 +12,7 @@ #include "models/PoissonSolver.h" #include "models/MultiPhysController.h" #include "common/Utilities.h" +#include "analysis/ElectroChemistry.h" using namespace std; @@ -53,7 +54,7 @@ int main(int argc, char **argv) ScaLBL_IonModel IonModel(rank,nprocs,comm); ScaLBL_Poisson PoissonSolver(rank,nprocs,comm); ScaLBL_Multiphys_Controller Study(rank,nprocs,comm);//multiphysics controller coordinating multi-model coupling - + // Load controller information Study.ReadParams(filename); @@ -68,7 +69,10 @@ int main(int argc, char **argv) IonModel.SetDomain(); IonModel.ReadInput(); - IonModel.Create(); + IonModel.Create(); + + // Create analysis object + ElectroChemistryAnalyzer Analysis(IonModel.Dm); // Get internal iteration number StokesModel.timestepMax = Study.getStokesNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv); @@ -96,18 +100,17 @@ int main(int argc, char **argv) timestep++;//AA operations if (timestep%Study.visualization_interval==0){ - PoissonSolver.getElectricPotential(timestep); + Analysis.WriteVis(IonModel,PoissonSolver,StokesModel,Study.db,timestep); + /* PoissonSolver.getElectricPotential(timestep); PoissonSolver.getElectricField(timestep); IonModel.getIonConcentration(timestep); StokesModel.getVelocity(timestep); + */ } } if (rank==0) printf("Save simulation raw data at maximum timestep\n"); - PoissonSolver.getElectricPotential(timestep); - PoissonSolver.getElectricField(timestep); - IonModel.getIonConcentration(timestep); - StokesModel.getVelocity(timestep); + Analysis.WriteVis(IonModel,PoissonSolver,StokesModel,Study.db,timestep); if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n"); if (rank==0) printf("*************************************************************\n"); From 6f0a10c48d0d54c1d90d6b9c74cbac8b592af409 Mon Sep 17 00:00:00 2001 From: James McClure Date: Thu, 31 Dec 2020 11:19:12 -0500 Subject: [PATCH 268/270] adding basic analysis capabilities --- analysis/ElectroChemistry.cpp | 78 +++++++++++++++++++++++++++++++++-- analysis/ElectroChemistry.h | 2 +- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/analysis/ElectroChemistry.cpp b/analysis/ElectroChemistry.cpp index b052d459..4a7bcf29 100644 --- a/analysis/ElectroChemistry.cpp +++ b/analysis/ElectroChemistry.cpp @@ -38,21 +38,94 @@ ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr dm): } ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){ - + if (Dm->rank()==0){ + fclose(TIMELOG); + } } void ElectroChemistryAnalyzer::SetParams(){ } -void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes){ +void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep){ + int i,j,k; Poisson.getElectricPotential(ElectricalPotential); + + /* local sub-domain averages */ + double rho_avg_local[Ion.number_ion_species]; + double rho_mu_avg_local[Ion.number_ion_species]; + double rho_mu_fluctuation_local[Ion.number_ion_species]; + double rho_psi_avg_local[Ion.number_ion_species]; + double rho_psi_fluctuation_local[Ion.number_ion_species]; + /* global averages */ + double rho_avg_global[Ion.number_ion_species]; + double rho_mu_avg_global[Ion.number_ion_species]; + double rho_mu_fluctuation_global[Ion.number_ion_species]; + double rho_psi_avg_global[Ion.number_ion_species]; + double rho_psi_fluctuation_global[Ion.number_ion_species]; + for (int ion=0; ionComm, rho_avg_local[ion]); + rho_mu_avg_global[ion]=sumReduce( Dm->Comm, rho_mu_avg_local[ion]); + rho_psi_avg_global[ion]=sumReduce( Dm->Comm, rho_psi_avg_local[ion]); + + rho_mu_avg_global[ion] /= rho_avg_global[ion]; + rho_psi_avg_global[ion] /= rho_avg_global[ion]; } + for (int ion=0; ionComm, rho_mu_fluctuation_local[ion]); + rho_psi_fluctuation_global[ion]=sumReduce( Dm->Comm, rho_psi_fluctuation_local[ion]); + } + if (Dm->rank()==0){ + fprintf(TIMELOG,"%i ",timestep); + for (int ion=0; ion input_db, int timestep){ @@ -89,7 +162,6 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P IonConcentration->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); visData[0].vars.push_back(IonConcentration); } - } if (vis_db->getWithDefault( "save_velocity", false )){ VxVar->name = "Velocity_x"; diff --git a/analysis/ElectroChemistry.h b/analysis/ElectroChemistry.h index fa404fb0..8d613ef7 100644 --- a/analysis/ElectroChemistry.h +++ b/analysis/ElectroChemistry.h @@ -47,7 +47,7 @@ public: ~ElectroChemistryAnalyzer(); void SetParams(); - void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes); + void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep); void WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr input_db, int timestep); private: From 0aa352cf24cc84e97c4ba36a1401cc9d0502647f Mon Sep 17 00:00:00 2001 From: Rex Zhe Li Date: Sat, 2 Jan 2021 19:21:15 -0500 Subject: [PATCH 269/270] update writing IonConcentration for visualization --- analysis/ElectroChemistry.cpp | 38 +++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/analysis/ElectroChemistry.cpp b/analysis/ElectroChemistry.cpp index 4a7bcf29..30487f03 100644 --- a/analysis/ElectroChemistry.cpp +++ b/analysis/ElectroChemistry.cpp @@ -140,7 +140,10 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P visData[0].meshName = "domain"; visData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); auto ElectricPotential = std::make_shared(); - auto IonConcentration = std::make_shared(); + std::vector> IonConcentration; + for (int ion=0; ion()); + } auto VxVar = std::make_shared(); auto VyVar = std::make_shared(); auto VzVar = std::make_shared(); @@ -155,14 +158,15 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P if (vis_db->getWithDefault( "save_concentration", true )){ for (int ion=0; ionname = VisName; - IonConcentration->type = IO::VariableType::VolumeVariable; - IonConcentration->dim = 1; - IonConcentration->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); - visData[0].vars.push_back(IonConcentration); + sprintf(VisName,"IonConcentration_%i",ion+1); + IonConcentration[ion]->name = VisName; + IonConcentration[ion]->type = IO::VariableType::VolumeVariable; + IonConcentration[ion]->dim = 1; + IonConcentration[ion]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); + visData[0].vars.push_back(IonConcentration[ion]); } } + if (vis_db->getWithDefault( "save_velocity", false )){ VxVar->name = "Velocity_x"; VxVar->type = IO::VariableType::VolumeVariable; @@ -190,23 +194,23 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P if (vis_db->getWithDefault( "save_concentration", true )){ for (int ion=0; ionname = VisName; - ASSERT(visData[0].vars[1]->name==VisName); - Array& IonConcentrationData = visData[0].vars[1]->data; + sprintf(VisName,"IonConcentration_%i",ion+1); + IonConcentration[ion]->name = VisName; + ASSERT(visData[0].vars[1+ion]->name==VisName); + Array& IonConcentrationData = visData[0].vars[1+ion]->data; Ion.getIonConcentration(Rho,ion); fillData.copy(Rho,IonConcentrationData); } } if (vis_db->getWithDefault( "save_velocity", false )){ - ASSERT(visData[0].vars[2]->name=="Velocity_x"); - ASSERT(visData[0].vars[3]->name=="Velocity_y"); - ASSERT(visData[0].vars[4]->name=="Velocity_z"); + ASSERT(visData[0].vars[1+Ion.number_ion_species+0]->name=="Velocity_x"); + ASSERT(visData[0].vars[1+Ion.number_ion_species+1]->name=="Velocity_y"); + ASSERT(visData[0].vars[1+Ion.number_ion_species+2]->name=="Velocity_z"); Stokes.getVelocity(Vel_x,Vel_y,Vel_z); - Array& VelxData = visData[0].vars[2]->data; - Array& VelyData = visData[0].vars[3]->data; - Array& VelzData = visData[0].vars[4]->data; + Array& VelxData = visData[0].vars[1+Ion.number_ion_species+0]->data; + Array& VelyData = visData[0].vars[1+Ion.number_ion_species+1]->data; + Array& VelzData = visData[0].vars[1+Ion.number_ion_species+2]->data; fillData.copy(Vel_x,VelxData); fillData.copy(Vel_y,VelyData); fillData.copy(Vel_z,VelzData); From 5df60909c20b40110f946e9375134322cefbe893 Mon Sep 17 00:00:00 2001 From: James McClure Date: Mon, 4 Jan 2021 00:07:11 -0500 Subject: [PATCH 270/270] add analysis interval to simulator --- tests/lbpm_electrokinetic_SingleFluid_simulator.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp index 2b3726a4..689745b9 100644 --- a/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp +++ b/tests/lbpm_electrokinetic_SingleFluid_simulator.cpp @@ -99,13 +99,16 @@ int main(int argc, char **argv) timestep++;//AA operations + if (timestep%Study.analysis_interval==0){ + Analysis.Basic(IonModel,PoissonSolver,StokesModel,timestep); + } if (timestep%Study.visualization_interval==0){ Analysis.WriteVis(IonModel,PoissonSolver,StokesModel,Study.db,timestep); - /* PoissonSolver.getElectricPotential(timestep); + /* PoissonSolver.getElectricPotential(timestep); PoissonSolver.getElectricField(timestep); IonModel.getIonConcentration(timestep); StokesModel.getVelocity(timestep); - */ + */ } }