OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:JamesEMcClure/LBPM-WIA

Browse Source
This commit is contained in:
James E McClure 2018-05-25 07:53:34 -04:00
commit b7cf5c5997
91 changed files with 6338 additions and 10178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -158,6 +158,7 @@ IF ( NOT ONLY_BUILD_DOCS )
ADD_PACKAGE_SUBDIRECTORY( analysis )
ADD_PACKAGE_SUBDIRECTORY( IO )
ADD_PACKAGE_SUBDIRECTORY( threadpool )
ADD_PACKAGE_SUBDIRECTORY( models )
IF ( USE_CUDA )
ADD_PACKAGE_SUBDIRECTORY( gpu )
ELSE()

3
IO/Reader.cpp
View File

@ -44,7 +44,6 @@ std::vector<std::string> IO::readTimesteps( const std::string& filename )
FILE *fid= fopen(filename.c_str(),"rb");
if ( fid==NULL )
ERROR("Error opening file");
auto pos = std::min(filename.find_last_of(47),filename.find_last_of(90));
std::vector<std::string> timesteps;
char buf[1000];
while (fgets(buf,sizeof(buf),fid) != NULL) {
@ -160,7 +159,6 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
#ifdef USE_SILO
const DatabaseEntry& database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str());
auto fid = silo::open( filename, silo::READ );
if ( meshDatabase.meshClass=="PointList" ) {
Array<double> coords = silo::readPointMesh<double>( fid, database.name );
@ -262,7 +260,6 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const st
const auto& database = meshDatabase.domains[domain];
auto variableDatabase = meshDatabase.getVariableDatabase( variable );
std::string filename = path + "/" + timestep + "/" + database.file;
int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str());
auto fid = silo::open( filename, silo::READ );
var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
if ( meshDatabase.meshClass=="PointList" ) {

2
IO/silo.hpp
View File

@ -322,9 +322,7 @@ void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords,
}
auto zones = mesh->zones;
int N_zones = zones->nzones;
int ndim_zones = zones->ndims;
ASSERT( zones->nshapes==1 );
int shape_type = zones->shapetype[0];
int shapesize = zones->shapesize[0];
tri.resize(N_zones,shapesize);
for (int i=0; i<N_zones; i++) {

252
analysis/TwoPhase.cpp
View File

@ -54,7 +54,7 @@
#define PI 3.14159265359
// Constructor
TwoPhase::TwoPhase(Domain &dm):
TwoPhase::TwoPhase(std::shared_ptr <Domain> dm):
n_nw_pts(0), n_ns_pts(0), n_ws_pts(0), n_nws_pts(0), n_local_sol_pts(0), n_local_nws_pts(0),
n_nw_tris(0), n_ns_tris(0), n_ws_tris(0), n_nws_seg(0), n_local_sol_tris(0),
nc(0), kstart(0), kfinish(0), fluid_isovalue(0), solid_isovalue(0), Volume(0),
@ -69,8 +69,8 @@ TwoPhase::TwoPhase(Domain &dm):
trRwn(0), trRwn_global(0), nwp_volume_global(0), wp_volume_global(0),
As_global(0), wwndnw_global(0), wwnsdnwn_global(0), Jwnwwndnw_global(0), dEs(0), dAwn(0), dAns(0)
{
Nx=dm.Nx; Ny=dm.Ny; Nz=dm.Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz*1.0;
Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
TempID = new char[Nx*Ny*Nz];
@ -135,7 +135,7 @@ TwoPhase::TwoPhase(Domain &dm):
Gns_global.resize(6);
Gws_global.resize(6);
//.........................................
if (Dm.rank==0){
if (Dm->rank()==0){
TIMELOG = fopen("timelog.tcat","a+");
if (fseek(TIMELOG,0,SEEK_SET) == fseek(TIMELOG,0,SEEK_CUR))
{
@ -165,7 +165,7 @@ TwoPhase::TwoPhase(Domain &dm):
}
else{
char LocalRankString[8];
sprintf(LocalRankString,"%05d",Dm.rank);
sprintf(LocalRankString,"%05d",Dm->rank());
char LocalRankFilename[40];
sprintf(LocalRankFilename,"%s%s","timelog.tcat.",LocalRankString);
TIMELOG = fopen(LocalRankFilename,"a+");
@ -196,8 +196,8 @@ TwoPhase::~TwoPhase()
void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData)
{
double factor,temp,value;
/* factor=0.5/Beta;
/*double factor,temp,value;
factor=0.5/Beta;
// Initialize to -1,1 (segmentation)
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
@ -232,7 +232,7 @@ void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, Double
}
}
Eikonal(DistData,TempID,Dm,50);
CalcDist(DistData,TempID,Dm);
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
@ -259,7 +259,7 @@ void TwoPhase::ComputeDelPhi()
int i,j,k;
double fx,fy,fz;
Dm.CommunicateMeshHalo(Phase);
Dm->CommunicateMeshHalo(Phase);
for (k=1; k<Nz-1; k++){
for (j=1; j<Ny-1; j++){
for (i=1; i<Nx-1; i++){
@ -307,8 +307,8 @@ void TwoPhase::SetupCubes(Domain &Dm)
int i,j,k;
kstart = 1;
kfinish = Nz-1;
if (Dm.BoundaryCondition !=0 && Dm.kproc==0) kstart = 4;
if (Dm.BoundaryCondition !=0 && Dm.kproc==Dm.nprocz-1) kfinish = Nz-4;
if (Dm->BoundaryCondition !=0 && Dm->kproc==0) kstart = 4;
if (Dm->BoundaryCondition !=0 && Dm->kproc==Dm->nprocz-1) kfinish = Nz-4;
nc=0;
for (k=kstart; k<kfinish; k++){
for (j=1; j<Ny-1; j++){
@ -327,17 +327,17 @@ void TwoPhase::SetupCubes(Domain &Dm)
void TwoPhase::UpdateSolid()
{
Dm.CommunicateMeshHalo(SDs);
Dm->CommunicateMeshHalo(SDs);
//...........................................................................
// Gradient of the Signed Distance function
//...........................................................................
pmmc_MeshGradient(SDs,SDs_x,SDs_y,SDs_z,Nx,Ny,Nz);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_x);
Dm->CommunicateMeshHalo(SDs_x);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_y);
Dm->CommunicateMeshHalo(SDs_y);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_z);
Dm->CommunicateMeshHalo(SDs_z);
//...........................................................................
}
@ -346,27 +346,27 @@ void TwoPhase::UpdateMeshValues()
{
int i,j,k,n;
//...........................................................................
Dm.CommunicateMeshHalo(SDn);
Dm->CommunicateMeshHalo(SDn);
//...........................................................................
// Compute the gradients of the phase indicator and signed distance fields
pmmc_MeshGradient(SDn,SDn_x,SDn_y,SDn_z,Nx,Ny,Nz);
//...........................................................................
// Gradient of the phase indicator field
//...........................................................................
Dm.CommunicateMeshHalo(SDn_x);
Dm->CommunicateMeshHalo(SDn_x);
//...........................................................................
Dm.CommunicateMeshHalo(SDn_y);
Dm->CommunicateMeshHalo(SDn_y);
//...........................................................................
Dm.CommunicateMeshHalo(SDn_z);
Dm->CommunicateMeshHalo(SDn_z);
//...........................................................................
Dm.CommunicateMeshHalo(SDs);
Dm->CommunicateMeshHalo(SDs);
pmmc_MeshGradient(SDs,SDs_x,SDs_y,SDs_z,Nx,Ny,Nz);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_x);
Dm->CommunicateMeshHalo(SDs_x);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_y);
Dm->CommunicateMeshHalo(SDs_y);
//...........................................................................
Dm.CommunicateMeshHalo(SDs_z);
Dm->CommunicateMeshHalo(SDs_z);
//...........................................................................
// Compute the mesh curvature of the phase indicator field
pmmc_MeshCurvature(SDn, MeanCurvature, GaussCurvature, Nx, Ny, Nz);
@ -375,26 +375,26 @@ void TwoPhase::UpdateMeshValues()
// Map Phase_tplus and Phase_tminus
for (int n=0; n<Nx*Ny*Nz; n++) dPdt(n) = 0.125*(Phase_tplus(n) - Phase_tminus(n));
//...........................................................................
Dm.CommunicateMeshHalo(Press);
Dm->CommunicateMeshHalo(Press);
//...........................................................................
Dm.CommunicateMeshHalo(Vel_x);
Dm->CommunicateMeshHalo(Vel_x);
//...........................................................................
Dm.CommunicateMeshHalo(Vel_y);
Dm->CommunicateMeshHalo(Vel_y);
//...........................................................................
Dm.CommunicateMeshHalo(Vel_z);
Dm->CommunicateMeshHalo(Vel_z);
//...........................................................................
Dm.CommunicateMeshHalo(MeanCurvature);
Dm->CommunicateMeshHalo(MeanCurvature);
//...........................................................................
Dm.CommunicateMeshHalo(GaussCurvature);
Dm->CommunicateMeshHalo(GaussCurvature);
//...........................................................................
Dm.CommunicateMeshHalo(DelPhi);
Dm->CommunicateMeshHalo(DelPhi);
//...........................................................................
// Initializing the blob ID
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] == 0){
if (Dm->id[n] == 0){
// Solid phase
PhaseID(i,j,k) = 0;
}
@ -418,8 +418,8 @@ void TwoPhase::ComputeLocal()
// If external boundary conditions are set, do not average over the inlet
kmin=1; kmax=Nz-1;
if (Dm.BoundaryCondition > 0 && Dm.kproc == 0) kmin=4;
if (Dm.BoundaryCondition > 0 && Dm.kproc == Dm.nprocz-1) kmax=Nz-4;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
for (k=kmin; k<kmax; k++){
for (j=1; j<Ny-1; j++){
@ -431,7 +431,7 @@ void TwoPhase::ComputeLocal()
// Compute volume averages
for (int p=0;p<8;p++){
n = i+cube[p][0] + (j+cube[p][1])*Nx + (k+cube[p][2])*Nx*Ny;
if ( Dm.id[n] != 0 ){
if ( Dm->id[n] != 0 ){
// 1-D index for this cube corner
// compute the norm of the gradient of the phase indicator field
// Compute the non-wetting phase volume contribution
@ -551,7 +551,7 @@ void TwoPhase::AssignComponentLabels()
int LabelNWP=1;
int LabelWP=2;
// NOTE: labeling the wetting phase components is tricky! One sandstone media had over 800,000 components
// NumberComponents_WP = ComputeGlobalPhaseComponent(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,PhaseID,LabelWP,Label_WP);
// NumberComponents_WP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelWP,Label_WP);
// treat all wetting phase is connected
NumberComponents_WP=1;
for (int k=0; k<Nz; k++){
@ -566,8 +566,8 @@ void TwoPhase::AssignComponentLabels()
}
// Fewer non-wetting phase features are present
//NumberComponents_NWP = ComputeGlobalPhaseComponent(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,PhaseID,LabelNWP,Label_NWP);
NumberComponents_NWP = ComputeGlobalBlobIDs(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,SDs,SDn,solid_isovalue,fluid_isovalue,Label_NWP,Dm.Comm);
//NumberComponents_NWP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelNWP,Label_NWP);
NumberComponents_NWP = ComputeGlobalBlobIDs(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,SDs,SDn,solid_isovalue,fluid_isovalue,Label_NWP,Dm->Comm);
}
void TwoPhase::ComponentAverages()
@ -585,15 +585,15 @@ void TwoPhase::ComponentAverages()
ComponentAverages_WP.fill(0.0);
ComponentAverages_NWP.fill(0.0);
if (Dm.rank==0){
if (Dm->rank()==0){
printf("Number of wetting phase components is %i \n",NumberComponents_WP);
printf("Number of non-wetting phase components is %i \n",NumberComponents_NWP);
}
// If external boundary conditions are set, do not average over the inlet
kmin=1; kmax=Nz-1;
if (Dm.BoundaryCondition > 0 && Dm.kproc == 0) kmin=4;
if (Dm.BoundaryCondition > 0 && Dm.kproc == Dm.nprocz-1) kmax=Nz-4;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
for (k=kmin; k<kmax; k++){
for (j=1; j<Ny-1; j++){
@ -624,7 +624,7 @@ void TwoPhase::ComponentAverages()
// Compute volume averages
for (int p=0;p<8;p++){
n = i+cube[p][0] + (j+cube[p][1])*Nx + (k+cube[p][2])*Nx*Ny;
if ( Dm.id[n] != 0 ){
if ( Dm->id[n] != 0 ){
// 1-D index for this cube corner
// compute the norm of the gradient of the phase indicator field
// Compute the non-wetting phase volume contribution
@ -636,9 +636,9 @@ void TwoPhase::ComponentAverages()
ComponentAverages_NWP(VY,LabelNWP) += 0.125*Vel_y(n);
ComponentAverages_NWP(VZ,LabelNWP) += 0.125*Vel_z(n);
// center of mass
ComponentAverages_NWP(CMX,LabelNWP) += 0.125*(i+cube[p][0]+Dm.iproc*Nx);
ComponentAverages_NWP(CMY,LabelNWP) += 0.125*(j+cube[p][1]+Dm.jproc*Ny);
ComponentAverages_NWP(CMZ,LabelNWP) += 0.125*(k+cube[p][2]+Dm.kproc*Nz);
ComponentAverages_NWP(CMX,LabelNWP) += 0.125*(i+cube[p][0]+Dm->iproc()*Nx);
ComponentAverages_NWP(CMY,LabelNWP) += 0.125*(j+cube[p][1]+Dm->jproc()*Ny);
ComponentAverages_NWP(CMZ,LabelNWP) += 0.125*(k+cube[p][2]+Dm->kproc()*Nz);
// twice the kinetic energy
ComponentAverages_NWP(VSQ,LabelNWP) += 0.125*(Vel_x(n)*Vel_x(n)+Vel_y(n)*Vel_y(n)+Vel_z(n)*Vel_z(n));
@ -656,9 +656,9 @@ void TwoPhase::ComponentAverages()
ComponentAverages_WP(VY,LabelWP)+= 0.125*Vel_y(n);
ComponentAverages_WP(VZ,LabelWP) += 0.125*Vel_z(n);
// Center of mass
ComponentAverages_WP(CMX,LabelWP) += 0.125*(i+cube[p][0]+Dm.iproc*Nx);
ComponentAverages_WP(CMY,LabelWP) += 0.125*(j+cube[p][1]+Dm.jproc*Ny);
ComponentAverages_WP(CMZ,LabelWP) += 0.125*(k+cube[p][2]+Dm.kproc*Nz);
ComponentAverages_WP(CMX,LabelWP) += 0.125*(i+cube[p][0]+Dm->iproc()*Nx);
ComponentAverages_WP(CMY,LabelWP) += 0.125*(j+cube[p][1]+Dm->jproc()*Ny);
ComponentAverages_WP(CMZ,LabelWP) += 0.125*(k+cube[p][2]+Dm->kproc()*Nz);
// twice the kinetic energy
ComponentAverages_WP(VSQ,LabelWP) += 0.125*(Vel_x(n)*Vel_x(n)+Vel_y(n)*Vel_y(n)+Vel_z(n)*Vel_z(n));
@ -802,24 +802,24 @@ void TwoPhase::ComponentAverages()
}
}
MPI_Barrier(Dm.Comm);
if (Dm.rank==0){
MPI_Barrier(Dm->Comm);
if (Dm->rank()==0){
printf("Component averages computed locally -- reducing result... \n");
}
// Globally reduce the non-wetting phase averages
RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP);
/* for (int b=0; b<NumberComponents_NWP; b++){
MPI_Barrier(Dm.Comm);
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
}
*/
MPI_Barrier(Dm.Comm);
MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm.Comm);
// MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm.Comm);
MPI_Barrier(Dm->Comm);
MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm->Comm);
// MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
if (Dm.rank==0){
if (Dm->rank()==0){
printf("rescaling... \n");
}
@ -907,15 +907,15 @@ void TwoPhase::ComponentAverages()
}
}
if (Dm.rank==0){
if (Dm->rank()==0){
printf("reduce WP averages... \n");
}
// reduce the wetting phase averages
for (int b=0; b<NumberComponents_WP; b++){
MPI_Barrier(Dm.Comm);
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm.Comm);
MPI_Barrier(Dm->Comm);
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_WP(idx,b)=RecvBuffer(idx);
}
@ -1052,15 +1052,15 @@ void TwoPhase::WriteSurfaces(int logcount)
C = P;
}
// Remap the points
A.x += 1.0*Dm.iproc*(Nx-2);
A.y += 1.0*Dm.jproc*(Nx-2);
A.z += 1.0*Dm.kproc*(Nx-2);
B.x += 1.0*Dm.iproc*(Nx-2);
B.y += 1.0*Dm.jproc*(Nx-2);
B.z += 1.0*Dm.kproc*(Nx-2);
C.x += 1.0*Dm.iproc*(Nx-2);
C.y += 1.0*Dm.jproc*(Nx-2);
C.z += 1.0*Dm.kproc*(Nx-2);
A.x += 1.0*Dm->iproc()*(Nx-2);
A.y += 1.0*Dm->jproc()*(Nx-2);
A.z += 1.0*Dm->kproc()*(Nx-2);
B.x += 1.0*Dm->iproc()*(Nx-2);
B.y += 1.0*Dm->jproc()*(Nx-2);
B.z += 1.0*Dm->kproc()*(Nx-2);
C.x += 1.0*Dm->iproc()*(Nx-2);
C.y += 1.0*Dm->jproc()*(Nx-2);
C.z += 1.0*Dm->kproc()*(Nx-2);
wn_mesh->A.push_back(A);
wn_mesh->B.push_back(B);
wn_mesh->C.push_back(C);
@ -1070,15 +1070,15 @@ void TwoPhase::WriteSurfaces(int logcount)
B = ws_pts(ws_tris(1,r));
C = ws_pts(ws_tris(2,r));
// Remap the points
A.x += 1.0*Dm.iproc*(Nx-2);
A.y += 1.0*Dm.jproc*(Nx-2);
A.z += 1.0*Dm.kproc*(Nx-2);
B.x += 1.0*Dm.iproc*(Nx-2);
B.y += 1.0*Dm.jproc*(Nx-2);
B.z += 1.0*Dm.kproc*(Nx-2);
C.x += 1.0*Dm.iproc*(Nx-2);
C.y += 1.0*Dm.jproc*(Nx-2);
C.z += 1.0*Dm.kproc*(Nx-2);
A.x += 1.0*Dm->iproc()*(Nx-2);
A.y += 1.0*Dm->jproc()*(Nx-2);
A.z += 1.0*Dm->kproc()*(Nx-2);
B.x += 1.0*Dm->iproc()*(Nx-2);
B.y += 1.0*Dm->jproc()*(Nx-2);
B.z += 1.0*Dm->kproc()*(Nx-2);
C.x += 1.0*Dm->iproc()*(Nx-2);
C.y += 1.0*Dm->jproc()*(Nx-2);
C.z += 1.0*Dm->kproc()*(Nx-2);
ws_mesh->A.push_back(A);
ws_mesh->B.push_back(B);
ws_mesh->C.push_back(C);
@ -1088,15 +1088,15 @@ void TwoPhase::WriteSurfaces(int logcount)
B = ns_pts(ns_tris(1,r));
C = ns_pts(ns_tris(2,r));
// Remap the points
A.x += 1.0*Dm.iproc*(Nx-2);
A.y += 1.0*Dm.jproc*(Nx-2);
A.z += 1.0*Dm.kproc*(Nx-2);
B.x += 1.0*Dm.iproc*(Nx-2);
B.y += 1.0*Dm.jproc*(Nx-2);
B.z += 1.0*Dm.kproc*(Nx-2);
C.x += 1.0*Dm.iproc*(Nx-2);
C.y += 1.0*Dm.jproc*(Nx-2);
C.z += 1.0*Dm.kproc*(Nx-2);
A.x += 1.0*Dm->iproc()*(Nx-2);
A.y += 1.0*Dm->jproc()*(Nx-2);
A.z += 1.0*Dm->kproc()*(Nx-2);
B.x += 1.0*Dm->iproc()*(Nx-2);
B.y += 1.0*Dm->jproc()*(Nx-2);
B.z += 1.0*Dm->kproc()*(Nx-2);
C.x += 1.0*Dm->iproc()*(Nx-2);
C.y += 1.0*Dm->jproc()*(Nx-2);
C.z += 1.0*Dm->kproc()*(Nx-2);
ns_mesh->A.push_back(A);
ns_mesh->B.push_back(B);
ns_mesh->C.push_back(C);
@ -1112,7 +1112,7 @@ void TwoPhase::WriteSurfaces(int logcount)
meshData[1].mesh = ws_mesh;
meshData[2].meshName = "ns-tris";
meshData[2].mesh = ns_mesh;
IO::writeData( logcount, meshData, Dm.Comm );
IO::writeData( logcount, meshData, Dm->Comm );
}
@ -1121,43 +1121,43 @@ void TwoPhase::Reduce()
int i;
double iVol_global=1.0/Volume;
//...........................................................................
MPI_Barrier(Dm.Comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
// Phase averages
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Barrier(Dm.Comm);
MPI_Barrier(Dm->Comm);
// Normalize the phase averages
// (density of both components = 1.0)
@ -1224,8 +1224,8 @@ void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT)
void TwoPhase::PrintAll(int timestep)
{
if (Dm.rank==0){
fprintf(TIMELOG,"%i %.5g ",timestep-5,dEs); // change in surface energy
if (Dm->rank()==0){
fprintf(TIMELOG,"%i %.5g ",timestep,dEs); // change in surface energy
fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw_global,pan_global); // saturation and pressure
fprintf(TIMELOG,"%.5g %.5g %.5g ",awn_global,ans_global,aws_global); // interfacial areas
fprintf(TIMELOG,"%.5g %.5g ",Jwn_global, Kwn_global); // curvature of wn interface
@ -1250,7 +1250,7 @@ void TwoPhase::PrintAll(int timestep)
else{
sat_w = 1.0 - nwp_volume/(nwp_volume+wp_volume);
fprintf(TIMELOG,"%i ",timestep-5); // change in surface energy
fprintf(TIMELOG,"%i ",timestep); // change in surface energy
fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw,pan); // saturation and pressure
fprintf(TIMELOG,"%.5g %.5g %.5g ",awn,ans,aws); // interfacial areas
fprintf(TIMELOG,"%.5g %.5g ",Jwn, Kwn); // curvature of wn interface
@ -1277,7 +1277,7 @@ void TwoPhase::PrintAll(int timestep)
void TwoPhase::PrintComponents(int timestep)
{
if (Dm.rank==0){
if (Dm->rank()==0){
printf("PRINT %i COMPONENT AVEREAGES: time = %i \n",(int)ComponentAverages_NWP.size(1),timestep);
for (int b=0; b<NumberComponents_NWP; b++){
//if (ComponentAverages_NWP(TRIMVOL,b) > 0.0){

4
analysis/TwoPhase.h
View File

@ -64,7 +64,7 @@ class TwoPhase{
public:
//...........................................................................
Domain& Dm;
std::shared_ptr <Domain> Dm;
int NumberComponents_WP,NumberComponents_NWP;
//...........................................................................
// Averaging variables
@ -145,7 +145,7 @@ public:
DoubleArray ComponentAverages_WP;
DoubleArray ComponentAverages_NWP;
//...........................................................................
TwoPhase(Domain &dm);
TwoPhase(std::shared_ptr <Domain> Dm);
~TwoPhase();
void Initialize();
// void SetupCubes(Domain &Dm);

4
analysis/analysis.cpp
View File

@ -329,7 +329,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
}
const BlobIDArray LocalIDs = IDs;
// Copy the ids and get the neighbors through the halos
fillHalo<BlobIDType> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
fillHalo<BlobIDType> fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1,{true,true,true});
fillData.fill(IDs);
// Create a list of all neighbor ranks (excluding self)
std::vector<int> neighbors;
@ -420,7 +420,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
}
}
// Fill the ghosts
fillHalo<BlobIDType> fillData2(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
fillHalo<BlobIDType> fillData2(comm,rank_info,{nx,ny,nz},{1,1,1},0,1,{true,true,true});
fillData2.fill(IDs);
// Reorder based on size (and compress the id space
int N_blobs_global = ReorderBlobIDs2(IDs,N_blobs_tot,ngx,ngy,ngz,comm);

189
analysis/distance.cpp Normal file
View File

@ -0,0 +1,189 @@
#include "analysis/distance.h"
/******************************************************************
* A fast distance calculation *
******************************************************************/
template<class TYPE>
void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
const std::array<bool,3>& periodic, const std::array<double,3>& dx )
{
ASSERT( Distance.size() == ID.size() );
std::array<int,3> n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 };
fillHalo<int> fillData( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,false,false}, periodic );
Array<int> id(ID.size());
Array<Vec> vecDist(Distance.size());
for (size_t i=0; i<ID.length(); i++)
id(i) = ID(i) == 0 ? -1:1;
fillData.fill( id );
CalcVecDist( vecDist, id, Dm, periodic, dx );
for (size_t i=0; i<Distance.length(); i++)
Distance(i) = id(i)*vecDist(i).norm();
}
/******************************************************************
* Vector-based distance calculation *
* Initialize cells adjacent to boundaries *
******************************************************************/
static void calcVecInitialize( Array<Vec> &d, const Array<int> &ID, double dx, double dy, double dz )
{
d.fill( Vec( 1e50, 1e50, 1e50 ) );
const double dx0 = 0.5*dx;
const double dy0 = 0.5*dy;
const double dz0 = 0.5*dz;
//const double dxy0 = 0.25*sqrt( dx*dx + dy*dy );
//const double dxz0 = 0.25*sqrt( dx*dx + dz*dz );
//const double dyz0 = 0.25*sqrt( dy*dy + dz*dz );
//const double dxyz0 = sqrt( dx*dx + dy*dy + dz*dz );
int Nx = d.size(0);
int Ny = d.size(1);
int Nz = d.size(2);
for (int k=1; k<Nz-1; k++) {
for (int j=1; j<Ny-1; j++) {
for (int i=1; i<Nx-1; i++) {
int id = ID(i,j,k);
bool x[2] = { id != ID(i-1,j,k), id != ID(i+1,j,k) };
bool y[2] = { id != ID(i,j-1,k), id != ID(i,j+1,k) };
bool z[2] = { id != ID(i,j,k-1), id != ID(i,j,k+1) };
if ( x[0] ) d(i,j,k) = Vec( dx0, 0, 0 );
if ( x[1] ) d(i,j,k) = Vec( -dx0, 0, 0 );
if ( y[0] ) d(i,j,k) = Vec( 0, dy0, 0 );
if ( y[1] ) d(i,j,k) = Vec( 0, -dy0, 0 );
if ( z[0] ) d(i,j,k) = Vec( 0, 0, dz0 );
if ( z[1] ) d(i,j,k) = Vec( 0, 0, -dz0 );
/*if ( x[0] && y[0] ) d(i,j,k) = Vec( dxy0, dxy0, 0 );
if ( x[0] && y[1] ) d(i,j,k) = Vec( dxy0, -dxy0, 0 );
if ( x[1] && y[0] ) d(i,j,k) = Vec( -dxy0, dxy0, 0 );
if ( x[1] && y[1] ) d(i,j,k) = Vec( -dxy0, -dxy0, 0 );
if ( x[0] && z[0] ) d(i,j,k) = Vec( dxz0, 0, dxz0 );
if ( x[0] && z[1] ) d(i,j,k) = Vec( dxz0, 0, -dxz0 );
if ( x[1] && z[0] ) d(i,j,k) = Vec( -dxz0, 0, dxz0 );
if ( x[1] && z[1] ) d(i,j,k) = Vec( -dxz0, 0, -dxz0 );
if ( y[0] && z[0] ) d(i,j,k) = Vec( 0, dyz0, dyz0 );
if ( y[0] && z[1] ) d(i,j,k) = Vec( 0, dyz0, -dyz0 );
if ( y[1] && z[0] ) d(i,j,k) = Vec( 0, -dyz0, dyz0 );
if ( y[1] && z[1] ) d(i,j,k) = Vec( 0, -dyz0, -dyz0 );*/
}
}
}
}
/******************************************************************
* Vector-based distance calculation *
* Update interior cells *
******************************************************************/
static double calcVecUpdateInterior( Array<Vec> &d, double dx, double dy, double dz )
{
double err = 0;
int Nx = d.size(0);
int Ny = d.size(1);
int Nz = d.size(2);
// Propagate (+,+,+)
for (int k=1; k<Nz; k++) {
for (int j=1; j<Ny; j++) {
for (int i=1; i<Nx; i++) {
auto vx = d(i-1,j,k);
auto vy = d(i,j-1,k);
auto vz = d(i,j,k-1);
vx.x += dx;
vy.y += dy;
vz.z += dz;
auto v = std::min( std::min(vx,vy), vz );
double d1 = v.norm2();
double d2 = d(i,j,k).norm2();
if ( d1 < d2 ) {
d(i,j,k) = v;
err = std::max( err, sqrt(d2)-sqrt(d1) );
}
}
}
}
// Propagate (-,-,-)
for (int k=Nz-2; k>=0; k--) {
for (int j=Ny-2; j>=0; j--) {
for (int i=Nx-2; i>=0; i--) {
auto vx = d(i+1,j,k);
auto vy = d(i,j+1,k);
auto vz = d(i,j,k+1);
vx.x -= dx;
vy.y -= dy;
vz.z -= dz;
auto v = std::min( std::min(vx,vy), vz );
double d1 = v.norm2();
double d2 = d(i,j,k).norm2();
if ( d1 < d2 ) {
d(i,j,k) = v;
err = std::max( err, sqrt(d2)-sqrt(d1) );
}
}
}
}
return err;
}
/******************************************************************
* Vector-based distance calculation *
******************************************************************/
void CalcVecDist( Array<Vec> &d, const Array<int> &ID0, const Domain &Dm,
const std::array<bool,3>& periodic, const std::array<double,3>& dx )
{
std::array<int,3> N = { Dm.Nx, Dm.Ny, Dm.Nz };
std::array<int,3> n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 };
// Create ID with ghosts
Array<int> ID(N[0],N[1],N[2]);
fillHalo<int> fillDataID( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,true,true}, periodic );
fillDataID.copy( ID0, ID );
// Fill ghosts with nearest neighbor
for (int k=1; k<N[2]-1; k++) {
for (int j=1; j<N[1]-1; j++) {
ID(0,j,k) = ID(1,j,k);
ID(N[0]-1,j,k) = ID(N[0]-2,j,k);
}
}
for (int k=1; k<N[2]-1; k++) {
for (int i=0; i<N[0]; i++) {
ID(i,0,k) = ID(i,1,k);
ID(i,N[1]-1,k) = ID(i,N[1]-2,k);
}
}
for (int i=0; i<N[0]; i++) {
for (int j=0; j<N[1]; j++) {
ID(i,j,0) = ID(i,j,1);
ID(i,j,N[1]-1) = ID(i,j,N[2]-2);
}
}
// Communicate ghosts
fillDataID.fill( ID );
// Create communicator for distance
fillHalo<Vec> fillData( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,false,false}, periodic );
// Calculate the local distances
calcVecInitialize( d, ID, dx[0], dx[1], dx[2] );
double err = 1e100;
double tol = 0.5 * std::min( std::min(dx[0],dx[1]), dx[2] );
for (int it=0; it<=50 && err>tol; it++) {
err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] );
}
// Calculate the global distances
int N_it = Dm.nprocx() + Dm.nprocy() + Dm.nprocz() + 100;
for ( int it=0; it<N_it; it++ ) {
// Update ghosts
fillData.fill( d );
// Update distance
double err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] );
// Check if we are finished
err = maxReduce( Dm.Comm, err );
if ( err < tol )
break;
}
}
// Explicit instantiations
template void CalcDist<float>( Array<float>&, const Array<char>&, const Domain&, const std::array<bool,3>&, const std::array<double,3>& );
template void CalcDist<double>( Array<double>&, const Array<char>&, const Domain&, const std::array<bool,3>&, const std::array<double,3>& );

42
analysis/distance.h Normal file
View File

@ -0,0 +1,42 @@
#ifndef Distance_H_INC
#define Distance_H_INC
#include "common/Domain.h"
struct Vec {
double x;
double y;
double z;
inline Vec(): x(0), y(0), z(0) {}
inline Vec( double x_, double y_, double z_ ): x(x_), y(y_), z(z_) {}
inline double norm() const { return sqrt(x*x+y*y+z*z); }
inline double norm2() const { return x*x+y*y+z*z; }
};
inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; }
/*!
* @brief Calculate the distance using a simple method
* @details This routine calculates the vector distance to the nearest domain surface.
* @param[out] Distance Distance function
* @param[in] ID Segmentation id
* @param[in] Dm Domain information
* @param[in] periodic Directions that are periodic
*/
template<class TYPE>
void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );
/*!
* @brief Calculate the distance using a simple method
* @details This routine calculates the vector distance to the nearest domain surface.
* @param[out] Distance Distance function
* @param[in] ID Domain id
* @param[in] Dm Domain information
* @param[in] periodic Directions that are periodic
*/
void CalcVecDist( Array<Vec> &Distance, const Array<int> &ID, const Domain &Dm,
const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );
#endif

58
analysis/eikonal.h
View File

@ -1,58 +0,0 @@
#ifndef Eikonal_H_INC
#define Eikonal_H_INC
/*!
* @brief Calculate the distance solving the Eikonal equation
* @details This routine converts the data in the Distance array to a signed distance
* by solving the equation df/dt = sign*(1-|grad f|), where Distance provides
* the values of f on the mesh associated with domain Dm
* It has been tested with segmented data initialized to values [-1,1]
* and will converge toward the signed distance to the surface bounding the associated phases
*
* Reference:
* Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229
*
* @return Returns the number of cubes in the blob
* @param[in/out] Distance Distance function
* @param[in] ID Segmentation id
* @param[in] DM Domain information
* @param[in] timesteps Maximum number of timesteps to process
* @return Returns the global variation
*/
inline double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps);
float Eikonal3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm, const int timesteps);
/*!
* @brief Calculate the distance using a simple method
* @details This routine calculates the distance using a very simple method working off the segmentation id.
*
* @return Returns the number of cubes in the blob
* @param[in/out] Distance Distance function
* @param[in] ID Segmentation id
* @param[in] DM Domain information
* @return Returns the global variation
*/
void CalcDist3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm );
/*!
* @brief Calculate the distance using a multi-level method
* @details This routine calculates the distance using a multi-grid method
*
* @return Returns the number of cubes in the blob
* @param[in/out] Distance Distance function
* @param[in] ID Segmentation id
* @param[in] DM Domain information
* @return Returns the global variation
*/
void CalcDistMultiLevel( Array<float> &Distance, const Array<char> &ID, const Domain &Dm );
#include "analysis/eikonal.hpp"
#endif

519
analysis/eikonal.hpp
View File

@ -1,519 +0,0 @@
#ifndef Eikonal_HPP_INC
#define Eikonal_HPP_INC
#include "analysis/eikonal.h"
#include "analysis/imfilter.h"
inline float minmod(float &a, float &b)
{
float value = a;
if ( a*b < 0.0)
value=0.0;
else if (fabs(a) > fabs(b))
value = b;
return value;
}
inline double minmod(double &a, double &b){
double value;
value = a;
if ( a*b < 0.0) value=0.0;
else if (fabs(a) > fabs(b)) value = b;
return value;
}
/******************************************************************
* Solve the eikonal equation *
******************************************************************/
inline double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
/*
* This routine converts the data in the Distance array to a signed distance
* by solving the equation df/dt = sign(1-|grad f|), where Distance provides
* the values of f on the mesh associated with domain Dm
* It has been tested with segmented data initialized to values [-1,1]
* and will converge toward the signed distance to the surface bounding the associated phases
*
* Reference:
* Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229
*/
int i,j,k;
double dt=0.1;
double Dx,Dy,Dz;
double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
double sign,norm;
double LocalVar,GlobalVar,LocalMax,GlobalMax;
int xdim,ydim,zdim;
xdim=Dm.Nx-2;
ydim=Dm.Ny-2;
zdim=Dm.Nz-2;
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
// Arrays to store the second derivatives
DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
int count = 0;
while (count < timesteps){
// Communicate the halo of values
fillData.fill(Distance);
// Compute second order derivatives
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
}
}
}
fillData.fill(Dxx);
fillData.fill(Dyy);
fillData.fill(Dzz);
LocalMax=LocalVar=0.0;
// Execute the next timestep
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
sign = 1;
if (ID[n] == 0) sign = -1;
// local second derivative terms
Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
/* //............Compute upwind derivatives ...................
Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
*/
Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
// Compute upwind derivatives for Godunov Hamiltonian
if (sign < 0.0){
if (Dxp + Dxm > 0.f) Dx = Dxp*Dxp;
else Dx = Dxm*Dxm;
if (Dyp + Dym > 0.f) Dy = Dyp*Dyp;
else Dy = Dym*Dym;
if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp;
else Dz = Dzm*Dzm;
}
else{
if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp;
else Dx = Dxm*Dxm;
if (Dyp + Dym < 0.f) Dy = Dyp*Dyp;
else Dy = Dym*Dym;
if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp;
else Dz = Dzm*Dzm;
}
//Dx = max(Dxp*Dxp,Dxm*Dxm);
//Dy = max(Dyp*Dyp,Dym*Dym);
//Dz = max(Dzp*Dzp,Dzm*Dzm);
norm=sqrt(Dx + Dy + Dz);
if (norm > 1.0) norm=1.0;
Distance(i,j,k) += dt*sign*(1.0 - norm);
LocalVar += dt*sign*(1.0 - norm);
if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
LocalMax = fabs(dt*sign*(1.0 - norm));
}
}
}
MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz;
count++;
if (count%50 == 0 && Dm.rank==0 ){
printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
fflush(stdout);
}
if (fabs(GlobalMax) < 1e-5){
if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n");
count=timesteps;
}
}
return GlobalVar;
}
inline float Eikonal3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm, const int timesteps)
{
PROFILE_START("Eikonal3D");
/*
* This routine converts the data in the Distance array to a signed distance
* by solving the equation df/dt = sign*(1-|grad f|), where Distance provides
* the values of f on the mesh associated with domain Dm
* It has been tested with segmented data initialized to values [-1,1]
* and will converge toward the signed distance to the surface bounding the associated phases
*
* Reference:
* Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229
*/
int i,j,k;
float dt=0.1;
float Dx,Dy,Dz;
float Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
float Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
float sign,norm;
float LocalVar,GlobalVar,LocalMax,GlobalMax;
int xdim,ydim,zdim;
xdim=Dm.Nx-2;
ydim=Dm.Ny-2;
zdim=Dm.Nz-2;
fillHalo<float> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
// Arrays to store the second derivatives
Array<float> Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
Array<float> Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
Array<float> Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
int count = 0;
while (count < timesteps){
// Communicate the halo of values
fillData.fill(Distance);
// Compute second order derivatives
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
}
}
}
fillData.fill(Dxx);
fillData.fill(Dyy);
fillData.fill(Dzz);
LocalMax=LocalVar=0.0;
// Execute the next timestep
// f(n+1) = f(n) + dt*sign(1-|grad f|)
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
sign = -1;
if (ID(i,j,k) == 1) sign = 1;
// local second derivative terms
Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
/* //............Compute upwind derivatives ...................
Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
*/
Dxp = Distance(i+1,j,k);
Dyp = Distance(i,j+1,k);
Dzp = Distance(i,j,k+1);
Dxm = Distance(i-1,j,k);
Dym = Distance(i,j-1,k);
Dzm = Distance(i,j,k-1);
// Compute upwind derivatives for Godunov Hamiltonian
if (sign < 0.0){
if (Dxp > Dxm) Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp;
else Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm;
if (Dyp > Dym) Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp;
else Dy = Distance(i,j,k) - Dym + 0.5*Dyym;
if (Dzp > Dzm) Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp;
else Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm;
}
else{
if (Dxp < Dxm) Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp;
else Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm;
if (Dyp < Dym) Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp;
else Dy = Distance(i,j,k) - Dym + 0.5*Dyym;
if (Dzp < Dzm) Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp;
else Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm;
}
norm=sqrt(Dx*Dx+Dy*Dy+Dz*Dz);
if (norm > 1.0) norm=1.0;
Distance(i,j,k) += dt*sign*(1.0 - norm);
LocalVar += dt*sign*(1.0 - norm);
if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
LocalMax = fabs(dt*sign*(1.0 - norm));
}
}
}
MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_FLOAT,MPI_SUM,Dm.Comm);
MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_FLOAT,MPI_MAX,Dm.Comm);
GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz;
count++;
if (count%50 == 0 && Dm.rank==0 )
printf(" Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
if (fabs(GlobalMax) < 1e-5){
if (Dm.rank==0) printf(" Exiting with max tolerance of 1e-5 \n");
count=timesteps;
}
}
PROFILE_STOP("Eikonal3D");
return GlobalVar;
}
/******************************************************************
* A fast distance calculation *
******************************************************************/
inline bool CalcDist3DIteration( Array<float> &Distance, const Domain &Dm )
{
const float sq2 = sqrt(2.0f);
const float sq3 = sqrt(3.0f);
float dist0[27];
dist0[0] = sq3; dist0[1] = sq2; dist0[2] = sq3;
dist0[3] = sq2; dist0[4] = 1; dist0[5] = sq2;
dist0[6] = sq3; dist0[7] = sq2; dist0[8] = sq3;
dist0[9] = sq2; dist0[10] = 1; dist0[11] = sq2;
dist0[12] = 1; dist0[13] = 0; dist0[14] = 1;
dist0[15] = sq2; dist0[16] = 1; dist0[17] = sq2;
dist0[18] = sq3; dist0[19] = sq2; dist0[20] = sq3;
dist0[21] = sq2; dist0[22] = 1; dist0[23] = sq2;
dist0[24] = sq3; dist0[25] = sq2; dist0[26] = sq3;
bool changed = false;
for (size_t k=1; k<Distance.size(2)-1; k++) {
for (size_t j=1; j<Distance.size(1)-1; j++) {
for (size_t i=1; i<Distance.size(0)-1; i++) {
float dist[27];
dist[0] = Distance(i-1,j-1,k-1); dist[1] = Distance(i,j-1,k-1); dist[2] = Distance(i+1,j-1,k-1);
dist[3] = Distance(i-1,j,k-1); dist[4] = Distance(i,j,k-1); dist[5] = Distance(i+1,j,k-1);
dist[6] = Distance(i-1,j+1,k-1); dist[7] = Distance(i,j+1,k-1); dist[8] = Distance(i+1,j+1,k-1);
dist[9] = Distance(i-1,j-1,k); dist[10] = Distance(i,j-1,k); dist[11] = Distance(i+1,j-1,k);
dist[12] = Distance(i-1,j,k); dist[13] = Distance(i,j,k); dist[14] = Distance(i+1,j,k);
dist[15] = Distance(i-1,j+1,k); dist[16] = Distance(i,j+1,k); dist[17] = Distance(i+1,j+1,k);
dist[18] = Distance(i-1,j-1,k+1); dist[19] = Distance(i,j-1,k+1); dist[20] = Distance(i+1,j-1,k+1);
dist[21] = Distance(i-1,j,k+1); dist[22] = Distance(i,j,k+1); dist[23] = Distance(i+1,j,k+1);
dist[24] = Distance(i-1,j+1,k+1); dist[25] = Distance(i,j+1,k+1); dist[26] = Distance(i+1,j+1,k+1);
float tmp = 1e100;
for (int k=0; k<27; k++)
tmp = std::min(tmp,dist[k]+dist0[k]);
if ( tmp < Distance(i,j,k) ) {
Distance(i,j,k) = tmp;
changed = true;
}
}
}
}
return changed;
}
inline void CalcDist3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm )
{
PROFILE_START("Calc Distance");
// Initialize the distance to be 0 fore the cells adjacent to the interface
Distance.fill(1e100);
for (size_t k=1; k<ID.size(2)-1; k++) {
for (size_t j=1; j<ID.size(1)-1; j++) {
for (size_t i=1; i<ID.size(0)-1; i++) {
char id = ID(i,j,k);
if ( id!=ID(i-1,j,k) || id!=ID(i+1,j,k) || id!=ID(i,j-1,k) || id!=ID(i,j+1,k) || id!=ID(i,j,k-1) || id!=ID(i,j,k+1) )
Distance(i,j,k) = 0.5;
}
}
}
// Compute the distance everywhere
fillHalo<float> fillData(Dm.Comm, Dm.rank_info,Dm.Nx,Dm.Ny,Dm.Nz,1,1,1,0,1);
while ( true ) {
// Communicate the halo of values
fillData.fill(Distance);
// The distance of the cell is the minimum of the distance of the neighbors plus the distance to that node
bool changed = CalcDist3DIteration( Distance, Dm );
changed = sumReduce(Dm.Comm,changed);
if ( !changed )
break;
}
// Update the sign of the distance
for (size_t i=0; i<ID.length(); i++)
Distance(i) *= ID(i)>0 ? 1:-1;
PROFILE_STOP("Calc Distance");
}
/******************************************************************
* A fast distance calculation *
******************************************************************/
inline void CalcDistMultiLevelHelper( Array<float> &Distance, const Domain &Dm )
{
size_t ratio = 4;
std::function<float(const Array<float>&)> coarsen = [ratio]( const Array<float>& data )
{
float tmp = 1e100;
int nx = data.size(0);
int ny = data.size(1);
int nz = data.size(2);
for (int k=0; k<nz; k++) {
float z = k-0.5*(nz-1);
for (int j=0; j<ny; j++) {
float y = j-0.5*(ny-1);
for (int i=0; i<nx; i++) {
float x = i-0.5*(nx-1);
tmp = std::min(data(i,j,k)+sqrt(x*x+y*y+z*z),tmp);
}
}
}
return tmp/ratio;
};
int Nx = Dm.Nx-2;
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
ASSERT(int(Distance.size(0))==Nx+2&&int(Distance.size(1))==Ny+2&&int(Distance.size(2))==Nz+2);
fillHalo<float> fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
if ( Nx%ratio==0 && Nx>8 && Ny%ratio==0 && Ny>8 && Nz%ratio==0 && Nz>8 ) {
// Use recursive version
int Nr = std::max(std::max(ratio,ratio),ratio);
// Run Nr iterations, communicate, run Nr iterations
for (int i=0; i<Nr; i++)
CalcDist3DIteration( Distance, Dm );
/*fillData.fill(Distance);
for (int i=0; i<Nr; i++)
CalcDist3DIteration( Distance, Dm );*/
// Coarsen
Array<float> dist(Nx,Ny,Nz);
fillData.copy(Distance,dist);
Domain Dm2(Nx/ratio,Ny/ratio,Nz/ratio,Dm.rank,Dm.nprocx,Dm.nprocy,Dm.nprocz,Dm.Lx,Dm.Ly,Dm.Lz,0);
Dm2.CommInit(Dm.Comm);
fillHalo<float> fillData2(Dm2.Comm,Dm2.rank_info,Nx/ratio,Ny/ratio,Nz/ratio,1,1,1,0,1);
auto dist2 = dist.coarsen( {ratio,ratio,ratio}, coarsen );
Array<float> Distance2(Nx/ratio+2,Ny/ratio+2,Nz/ratio+2);
fillData2.copy(dist2,Distance2);
// Solve
CalcDistMultiLevelHelper( Distance2, Dm2 );
// Interpolate the coarse grid to the fine grid
fillData2.copy(Distance2,dist2);
for (int k=0; k<Nz; k++) {
int k2 = k/ratio;
float z = (k-k2*ratio)-0.5*(ratio-1);
for (int j=0; j<Ny; j++) {
int j2 = j/ratio;
float y = (j-j2*ratio)-0.5*(ratio-1);
for (int i=0; i<Nx; i++) {
int i2 = i/ratio;
float x = (i-i2*ratio)-0.5*(ratio-1);
dist(i,j,k) = std::min(dist(i,j,k),ratio*dist2(i2,j2,k2)+sqrt(x*x+y*y+z*z));
}
}
}
fillData.copy(dist,Distance);
// Run Nr iterations, communicate, run Nr iterations
for (int i=0; i<Nr; i++)
CalcDist3DIteration( Distance, Dm );
fillData.fill(Distance);
for (int i=0; i<Nr; i++)
CalcDist3DIteration( Distance, Dm );
} else {
// Use coarse-grid version
while ( true ) {
// Communicate the halo of values
fillData.fill(Distance);
// The distance of the cell is the minimum of the distance of the neighbors plus the distance to that node
bool changed = CalcDist3DIteration( Distance, Dm );
changed = sumReduce(Dm.Comm,changed);
if ( !changed )
break;
}
}
}
inline void CalcDistMultiLevel( Array<float> &Distance, const Array<char> &ID, const Domain &Dm )
{
PROFILE_START("Calc Distance Multilevel");
int Nx = Dm.Nx-2;
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
ASSERT(int(Distance.size(0))==Nx+2&&int(Distance.size(1))==Ny+2&&int(Distance.size(2))==Nz+2);
fillHalo<float> fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
// Initialize the distance to be 0 fore the cells adjacent to the interface
Distance.fill(1e100);
for (size_t k=1; k<ID.size(2)-1; k++) {
for (size_t j=1; j<ID.size(1)-1; j++) {
for (size_t i=1; i<ID.size(0)-1; i++) {
char id = ID(i,j,k);
if ( id!=ID(i-1,j,k) || id!=ID(i+1,j,k) || id!=ID(i,j-1,k) || id!=ID(i,j+1,k) || id!=ID(i,j,k-1) || id!=ID(i,j,k+1) )
Distance(i,j,k) = 0.5;
}
}
}
// Solve the for the distance using a recursive method
CalcDistMultiLevelHelper( Distance, Dm );
// Update the sign of the distance
for (size_t i=0; i<ID.length(); i++)
Distance(i) *= ID(i)>0 ? 1:-1;
fillData.fill(Distance);
// Run a quick filter to smooth the data
float sigma = 0.6;
Array<float> H = imfilter::create_filter<float>( { 1 }, "gaussian", &sigma );
std::vector<imfilter::BC> BC(3,imfilter::BC::replicate);
Distance = imfilter::imfilter_separable<float>( Distance, {H,H,H}, BC );
PROFILE_STOP("Calc Distance Multilevel");
}
#endif

52
analysis/pmmc.h
View File

@ -745,12 +745,20 @@ inline Point VertexInterp(const Point &p1, const Point &p2, double valp1, double
inline void SolidMarchingCubes(DoubleArray &A, const double &v, DoubleArray &B, const double &isovalue,
int i,int j,int k, int m, int n, int o, DTMutableList<Point>
&cellvertices, int &lengthvertices, IntArray &Triangles, int &nTris,
DoubleArray &values){
DoubleArray &values)
{
NULL_USE( isovalue );
NULL_USE( m );
NULL_USE( n );
NULL_USE( o );
// THIS SUBROUTINE COMPUTES THE VERTICES FOR THE SOLID PHASE IN
// A PARTICULAR GRID CELL, THEN ARRANGES THEM INTO TRIANGLES
// ALSO ORGANIZES THE LIST OF VALUES TO CORRESPOND WITH VERTEX LIST
NULL_USE( v );
//int N = 0;
Point P,Q;
Point PlaceHolder;
@ -1079,7 +1087,12 @@ inline void SolidMarchingCubes(DoubleArray &A, const double &v, DoubleArray &B,
inline void SOL_SURF(DoubleArray &A, const double &v, DoubleArray &B, const double &isovalue,
int i,int j,int k, int m, int n, int o, DTMutableList<Point>
&cellvertices, int &lengthvertices, IntArray &Tlist, int &nTris,
DoubleArray &values){
DoubleArray &values)
{
NULL_USE( isovalue );
NULL_USE( m );
NULL_USE( n );
NULL_USE( o );
// THIS SUBROUTINE COMPUTES THE VERTICES FOR THE SOLID PHASE IN
// A PARTICULAR GRID CELL, THEN ARRANGES THEM INTO TRIANGLES
@ -2540,6 +2553,11 @@ inline void EDGE(DoubleArray &A, double &v, DoubleArray &solid, int &i, int &j,
DTMutableList<Point> &nw_pts, int &n_nw_pts, IntArray &nw_tris, int &n_nw_tris,
DTMutableList<Point> &local_nws_pts, int &n_local_nws_pts)
{
NULL_USE( m );
NULL_USE( n );
NULL_USE( o );
// FIND THE POINTS ON THE nw SURFACE THAT ARE ON THE EDGE (COMMON LINE WITH SOLID PHASE)
// function A is the fluid data padded (so that it has values inside the solid phase)
@ -3319,6 +3337,8 @@ inline void pmmc_MeshCurvature(DoubleArray &f, DoubleArray &MeanCurvature, Doubl
//--------------------------------------------------------------------------------------------------------
inline int pmmc_CubeListFromMesh(IntArray &cubeList, int ncubes, int Nx, int Ny, int Nz)
{
NULL_USE( ncubes );
int i,j,k,nc;
nc=0;
//...........................................................................
@ -3974,6 +3994,8 @@ inline double pmmc_CommonCurveSpeed(DoubleArray &CubeValues, DoubleArray &dPdt,
DoubleArray &Sx, DoubleArray &Sy, DoubleArray &Sz,
DTMutableList<Point> &Points, int i, int j, int k, int npts)
{
NULL_USE( CubeValues );
int p;
double s,lwns,norm;
double zeta;
@ -4125,7 +4147,12 @@ inline void pmmc_CurveCurvature(DoubleArray &f, DoubleArray &s,
DoubleArray &f_x, DoubleArray &f_y, DoubleArray &f_z,
DoubleArray &s_x, DoubleArray &s_y, DoubleArray &s_z,
DoubleArray &KN, DoubleArray &KG,
double &KNavg, double &KGavg, DTMutableList<Point> &Points, int npts, int ic, int jc, int kc){
double &KNavg, double &KGavg, DTMutableList<Point> &Points, int npts, int ic, int jc, int kc)
{
NULL_USE( f );
NULL_USE( s );
NULL_USE( KN );
NULL_USE( KG );
int p,i,j,k;
double length;
@ -4324,6 +4351,10 @@ inline double pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArr
DoubleArray &SurfaceVector, DoubleArray &AvgSpeed, DoubleArray &AvgVel,
int i, int j, int k, int npts, int ntris)
{
NULL_USE( CubeValues );
NULL_USE( SurfaceVector );
NULL_USE( npts );
Point A,B,C,P;
double x,y,z;
double s,s1,s2,s3,area;
@ -4378,7 +4409,15 @@ inline double pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArr
}
//--------------------------------------------------------------------------------------------------------
inline double geomavg_EulerCharacteristic(DTMutableList<Point> &Points, IntArray &Triangles,
int &npts, int &ntris, int &i, int &j, int &k){
int &npts, int &ntris, int &i, int &j, int &k)
{
NULL_USE( Points );
NULL_USE( Triangles );
NULL_USE( npts );
NULL_USE( ntris );
NULL_USE( i );
NULL_USE( j );
NULL_USE( k );
/* REFERENCE
* Huang, Liu, Lee, Yang, Tsang
@ -4506,7 +4545,10 @@ inline double mink_phase_epc6(IntArray &PhaseID, DoubleArray &CubeValues, int Ph
}
inline double mink_EulerCharacteristic(DTMutableList<Point> &Points, IntArray &Triangles,
DoubleArray &CubeValues, int &npts, int &ntris, int &i, int &j, int &k){
DoubleArray &CubeValues, int &npts, int &ntris, int &i, int &j, int &k)
{
NULL_USE( CubeValues );
// Compute the Euler characteristic for triangles in a cube
// Exclude edges and vertices shared with between multiple cubes

103
analysis/runAnalysis.cpp
View File

@ -237,14 +237,14 @@ private:
* MPI comm wrapper for use with analysis *
******************************************************************/
runAnalysis::commWrapper::commWrapper( int tag_, MPI_Comm comm_, runAnalysis* analysis_ ):
tag(tag_),
comm(comm_),
tag(tag_),
analysis(analysis_)
{
}
runAnalysis::commWrapper::commWrapper( commWrapper &&rhs ):
tag(rhs.tag),
comm(rhs.comm),
tag(rhs.tag),
analysis(rhs.analysis)
{
rhs.tag = -1;
@ -281,30 +281,35 @@ runAnalysis::commWrapper runAnalysis::getComm( )
/******************************************************************
* Constructor/Destructors *
******************************************************************/
runAnalysis::runAnalysis( int restart_interval, int analysis_interval,
int blobid_interval, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm,
int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
IntArray Map, const std::string& LocalRestartFile ):
runAnalysis::runAnalysis( std::shared_ptr<Database> db,
const RankInfoStruct& rank_info, std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> Dm,
int Np, bool pBC, double beta, IntArray Map ):
d_Np( Np ),
d_restart_interval( restart_interval ),
d_analysis_interval( analysis_interval ),
d_blobid_interval( blobid_interval ),
d_beta( beta ),
d_ScaLBL_Comm( ScaLBL_Comm ),
d_rank_info( rank_info ),
d_Map( Map ),
d_fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1),
d_restartFile( LocalRestartFile )
d_ScaLBL_Comm( ScaLBL_Comm),
d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1)
{
d_N[0] = Nx;
d_N[1] = Ny;
d_N[2] = Nz;
NULL_USE( pBC );
INSIST( db, "Input database is empty" );
char rankString[20];
sprintf(rankString,"%05d",Dm->rank());
d_N[0] = Dm->Nx;
d_N[1] = Dm->Ny;
d_N[2] = Dm->Nz;
d_restart_interval = db->getScalar<int>( "restart_interval" );
d_analysis_interval = db->getScalar<int>( "analysis_interval" );
d_blobid_interval = db->getScalar<int>( "blobid_interval" );
d_visualization_interval = db->getScalar<int>( "visualization_interval" );
auto restart_file = db->getScalar<std::string>( "restart_file" );
d_restartFile = restart_file + "." + rankString;
d_rank = MPI_WORLD_RANK();
writeIDMap(ID_map_struct(),0,id_map_filename);
// Create the MeshDataStruct
d_meshData.resize(1);
d_meshData[0].meshName = "domain";
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz );
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
auto PhaseVar = std::make_shared<IO::Variable>();
auto PressVar = std::make_shared<IO::Variable>();
auto SignDistVar = std::make_shared<IO::Variable>();
@ -312,22 +317,22 @@ runAnalysis::runAnalysis( int restart_interval, int analysis_interval,
PhaseVar->name = "phase";
PhaseVar->type = IO::VariableType::VolumeVariable;
PhaseVar->dim = 1;
PhaseVar->data.resize(Nx-2,Ny-2,Nz-2);
PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
d_meshData[0].vars.push_back(PhaseVar);
PressVar->name = "Pressure";
PressVar->type = IO::VariableType::VolumeVariable;
PressVar->dim = 1;
PressVar->data.resize(Nx-2,Ny-2,Nz-2);
PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
d_meshData[0].vars.push_back(PressVar);
SignDistVar->name = "SignDist";
SignDistVar->type = IO::VariableType::VolumeVariable;
SignDistVar->dim = 1;
SignDistVar->data.resize(Nx-2,Ny-2,Nz-2);
SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
d_meshData[0].vars.push_back(SignDistVar);
BlobIDVar->name = "BlobID";
BlobIDVar->type = IO::VariableType::VolumeVariable;
BlobIDVar->dim = 1;
BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2);
BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
d_meshData[0].vars.push_back(BlobIDVar);
// Initialize the comms
MPI_Comm_dup(MPI_COMM_WORLD,&d_comm);
@ -335,6 +340,10 @@ runAnalysis::runAnalysis( int restart_interval, int analysis_interval,
d_comms[i] = MPI_COMM_NULL;
d_comm_used[i] = false;
}
// Initialize the threads
int N_threads = db->getWithDefault<int>( "N_threads", 4 );
auto method = db->getWithDefault<std::string>( "load_balance", "default" );
createThreads( method, N_threads );
}
runAnalysis::~runAnalysis( )
{
@ -423,6 +432,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep )
AnalysisType type = AnalysisType::AnalyzeNone;
if ( timestep%d_analysis_interval + 8 == d_analysis_interval ) {
// Copy the phase indicator field for the earlier timestep
printf("Copy phase indicator,timestep=%i\n",timestep);
type |= AnalysisType::CopyPhaseIndicator;
}
if ( timestep%d_blobid_interval == 0 ) {
@ -437,20 +447,22 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep )
type |= AnalysisType::IdentifyBlobs;
}
#endif */
if ( timestep%d_analysis_interval + 4 == 0 ) {
if ( timestep%d_analysis_interval + 4 == d_analysis_interval ) {
// Copy the averages to the CPU (and identify blobs)
printf("Copy sim state, timestep=%i \n",timestep);
type |= AnalysisType::CopySimState;
type |= AnalysisType::IdentifyBlobs;
}
if ( timestep%d_analysis_interval == 0 ) {
// Run the analysis
printf("Compute averages, timestep=%i \n",timestep);
type |= AnalysisType::ComputeAverages;
}
if (timestep%d_restart_interval == 0) {
// Write the restart file
type |= AnalysisType::CreateRestart;
}
if (timestep%d_restart_interval == 0) {
if (timestep%d_visualization_interval == 0) {
// Write the visualization data
type |= AnalysisType::WriteVis;
type |= AnalysisType::CopySimState;
@ -465,7 +477,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep )
* Run the analysis *
******************************************************************/
void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den )
double *Pressure, double *Velocity, double *fq, double *Den)
{
int N = d_N[0]*d_N[1]*d_N[2];
@ -487,12 +499,35 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
PROFILE_START("Copy data to host",1);
std::shared_ptr<DoubleArray> phase;
if ( matches(type,AnalysisType::CopyPhaseIndicator) ||
matches(type,AnalysisType::ComputeAverages) ||
matches(type,AnalysisType::CopySimState) ||
matches(type,AnalysisType::IdentifyBlobs) )
matches(type,AnalysisType::ComputeAverages) ||
matches(type,AnalysisType::CopySimState) ||
matches(type,AnalysisType::IdentifyBlobs) )
{
phase = std::shared_ptr<DoubleArray>(new DoubleArray(d_N[0],d_N[1],d_N[2]));
ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double));
phase = std::shared_ptr<DoubleArray>(new DoubleArray(d_N[0],d_N[1],d_N[2]));
//ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double));
// try 2 d_ScaLBL_Comm.RegularLayout(d_Map,Phi,Averages.Phase);
// memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double));
int Nx = d_N[0];
int Ny = d_N[1];
int Nz = d_N[2];
double *TmpDat;
double value;
TmpDat = new double [d_Np];
ScaLBL_CopyToHost(&TmpDat[0],&Phi[0], d_Np*sizeof(double));
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
int n=k*Nx*Ny+j*Nx+i;
int idx=d_Map(i,j,k);
if (!(idx<0)){
double value=TmpDat[idx];
//regdata(i,j,k)=value;
phase->data()[n]=value;
}
}
}
}
delete [] TmpDat;
}
if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double));
@ -513,10 +548,10 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
PROFILE_STOP("Copy-Wait",1);
PROFILE_START("Copy-State",1);
memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double));
d_ScaLBL_Comm.RegularLayout(d_Map,Pressure,Averages.Press);
d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[0],Averages.Vel_x);
d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y);
d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z);
d_ScaLBL_Comm->RegularLayout(d_Map,Pressure,Averages.Press);
d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[0],Averages.Vel_x);
d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y);
d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z);
PROFILE_STOP("Copy-State",1);
}
std::shared_ptr<double> cDen, cfq;
@ -551,7 +586,9 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
}
// Spawn threads to do the analysis work
if ( matches(type,AnalysisType::ComputeAverages) ) {
//if (timestep%d_restart_interval==0){
if ( matches(type,AnalysisType::ComputeAverages) ) {
//if ( timestep%d_analysis_interval == 0 ) {
auto work = new AnalysisWorkItem(type,timestep,Averages,d_last_index,d_last_id_map,d_beta);
work->add_dependency(d_wait_blobID);
work->add_dependency(d_wait_analysis);

12
analysis/runAnalysis.h
View File

@ -23,16 +23,14 @@ class runAnalysis
public:
//! Constructor
runAnalysis( int restart_interval, int analysis_interval, int blobid_interval,
const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm,
int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
IntArray Map, const std::string& LocalRestartFile );
runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool pBC, double beta, IntArray Map );
//! Destructor
~runAnalysis();
//! Run the next analysis
void run( int timestep, TwoPhase& Averages, const double *Phi,
void run( int timestep, TwoPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
//! Finish all active analysis
@ -85,10 +83,9 @@ private:
int d_N[3];
int d_Np;
int d_rank;
int d_restart_interval, d_analysis_interval, d_blobid_interval;
int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
double d_beta;
ThreadPool d_tpool;
ScaLBL_Communicator d_ScaLBL_Comm;
RankInfoStruct d_rank_info;
IntArray d_Map;
BlobIDstruct d_last_ids;
@ -100,6 +97,7 @@ private:
MPI_Comm d_comm;
MPI_Comm d_comms[1024];
volatile bool d_comm_used[1024];
std::shared_ptr<ScaLBL_Communicator> d_ScaLBL_Comm;
// Ids of work items to use for dependencies
ThreadPool::thread_id_t d_wait_blobID;

22
analysis/uCT.cpp
View File

@ -1,6 +1,6 @@
#include "analysis/uCT.h"
#include "analysis/analysis.h"
#include "analysis/eikonal.h"
#include "analysis/distance.h"
#include "analysis/filters.h"
#include "analysis/imfilter.h"
@ -149,7 +149,7 @@ void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
fillFloat.fill( Mean );
segment( Mean, ID, 0.01 );
// Compute the distance using the segmented volume
Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx );
CalcDist( Dist, ID, Dm );
fillFloat.fill(Dist);
smooth( VOL, Dist, 2.0, MultiScaleSmooth, fillFloat );
// Compute non-local mean
@ -210,9 +210,7 @@ void refine( const Array<float>& Dist_coarse,
//removeDisconnected( ID, Dm );
// Compute the distance using the segmented volume
if ( level > 0 ) {
//Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx );
//CalcDist3D( Dist, ID, Dm );
CalcDistMultiLevel( Dist, ID, Dm );
CalcDist( Dist, ID, Dm );
fillFloat.fill(Dist);
}
}
@ -232,7 +230,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
// Calculate the distance
CalcDistMultiLevel( Dist, ID, Dm );
CalcDist( Dist, ID, Dm );
fillFloat.fill(Dist);
// Compute the range to shrink the volume based on the L2 norm of the distance
Array<float> Dist0(Nx,Ny,Nz);
@ -256,8 +254,8 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
}
//Array<float> Dist1 = Dist;
//Array<float> Dist2 = Dist;
CalcDistMultiLevel( Dist1, ID1, Dm );
CalcDistMultiLevel( Dist2, ID2, Dm );
CalcDist( Dist1, ID1, Dm );
CalcDist( Dist2, ID2, Dm );
fillFloat.fill(Dist1);
fillFloat.fill(Dist2);
// Keep those regions that are within dx2 of the new volumes
@ -272,8 +270,8 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
}
}
// Find regions of uncertainty that are entirely contained within another region
fillHalo<double> fillDouble(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
fillHalo<BlobIDType> fillInt(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
fillHalo<double> fillDouble(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1);
fillHalo<BlobIDType> fillInt(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1);
BlobIDArray GlobalBlobID;
DoubleArray SignDist(ID.size());
for (size_t i=0; i<ID.length(); i++)
@ -343,7 +341,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
// Perform the final segmentation and update the distance
fillFloat.fill(Mean);
segment( Mean, ID, 0.01 );
CalcDistMultiLevel( Dist, ID, Dm );
CalcDist( Dist, ID, Dm );
fillFloat.fill(Dist);
}
@ -356,7 +354,7 @@ void filter_src( const Domain& Dm, Array<float>& src )
int Nx = Dm.Nx-2;
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
fillHalo<float> fillFloat(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
fillHalo<float> fillFloat(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1);
// Perform a hot-spot filter on the data
std::vector<imfilter::BC> BC = { imfilter::BC::replicate, imfilter::BC::replicate, imfilter::BC::replicate };
std::function<float(const Array<float>&)> filter_3D = []( const Array<float>& data )

25
common/Communication.h
View File

@ -5,6 +5,8 @@
#include "common/Utilities.h"
#include "common/Array.h"
#include <array>
// ********** COMMUNICTION **************************************
/*
//..............................................................
@ -43,18 +45,17 @@ public:
/*!
* @brief Default constructor
* @param[in] info Rank and neighbor rank info
* @param[in] nx Number of local cells in the x direction
* @param[in] ny Number of local cells in the y direction
* @param[in] nz Number of local cells in the z direction
* @param[in] ngx Number of ghost cells in the x direction
* @param[in] ngy Number of ghost cells in the y direction
* @param[in] ngz Number of ghost cells in the z direction
* @param[in] n Number of local cells
* @param[in] ng Number of ghost cells
* @param[in] tag Initial tag to use for the communication (we will require tag:tag+26)
* @param[in] depth Maximum depth to support
* @param[in] fill Fill {faces,edges,corners}
* @param[in] periodic Periodic dimensions
*/
fillHalo( MPI_Comm comm, const RankInfoStruct& info, int nx, int ny, int nz,
int ngx, int ngy, int ngz, int tag, int depth,
bool fill_face=true, bool fill_edge=true, bool fill_corner=true );
fillHalo( MPI_Comm comm, const RankInfoStruct& info,
std::array<int,3> n, std::array<int,3> ng, int tag, int depth,
std::array<bool,3> fill = {true,true,true},
std::array<bool,3> periodic = {true,true,true} );
//! Destructor
~fillHalo( );
@ -75,15 +76,17 @@ public:
private:
MPI_Comm comm;
RankInfoStruct info;
int nx, ny, nz, ngx, ngy, ngz, depth;
std::array<int,3> n, ng;
int depth;
bool fill_pattern[3][3][3];
int tag[3][3][3];
int N_send_recv[3][3][3];
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
MPI_Comm comm;
size_t N_type;
MPI_Datatype datatype;
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor

157
common/Communication.hpp
View File

@ -11,16 +11,30 @@
* Structure to store the rank info *
********************************************************/
template<class TYPE>
fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, int ny0, int nz0,
int ngx0, int ngy0, int ngz0, int tag0, int depth0,
bool fill_face, bool fill_edge, bool fill_corner ):
info(info0), nx(nx0), ny(ny0), nz(nz0), ngx(ngx0), ngy(ngy0), ngz(ngz0), depth(depth0)
fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,
std::array<int,3> n_, std::array<int,3> ng_, int tag0, int depth_,
std::array<bool,3> fill, std::array<bool,3> periodic ):
comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_)
{
comm = comm0;
datatype = getMPItype<TYPE>();
if ( std::is_same<TYPE,double>() ) {
N_type = 1;
datatype = MPI_DOUBLE;
} else if ( std::is_same<TYPE,float>() ) {
N_type = 1;
datatype = MPI_FLOAT;
} else if ( sizeof(TYPE)%sizeof(double)==0 ) {
N_type = sizeof(TYPE) / sizeof(double);
datatype = MPI_DOUBLE;
} else if ( sizeof(TYPE)%sizeof(float)==0 ) {
N_type = sizeof(TYPE) / sizeof(float);
datatype = MPI_FLOAT;
} else {
N_type = sizeof(TYPE);
datatype = MPI_BYTE;
}
// Set the fill pattern
memset(fill_pattern,0,sizeof(fill_pattern));
if ( fill_face ) {
if ( fill[0] ) {
fill_pattern[0][1][1] = true;
fill_pattern[2][1][1] = true;
fill_pattern[1][0][1] = true;
@ -28,7 +42,7 @@ fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0,
fill_pattern[1][1][0] = true;
fill_pattern[1][1][2] = true;
}
if ( fill_edge ) {
if ( fill[1] ) {
fill_pattern[0][0][1] = true;
fill_pattern[0][2][1] = true;
fill_pattern[2][0][1] = true;
@ -42,7 +56,7 @@ fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0,
fill_pattern[1][2][0] = true;
fill_pattern[1][2][2] = true;
}
if ( fill_corner ) {
if ( fill[2] ) {
fill_pattern[0][0][0] = true;
fill_pattern[0][0][2] = true;
fill_pattern[0][2][0] = true;
@ -52,13 +66,50 @@ fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0,
fill_pattern[2][2][0] = true;
fill_pattern[2][2][2] = true;
}
// Remove communication for non-perioidic directions
if ( !periodic[0] && info.ix==0 ) {
for (int j=0; j<3; j++) {
for (int k=0; k<3; k++)
fill_pattern[0][j][k] = false;
}
}
if ( !periodic[0] && info.ix==info.nx-1 ) {
for (int j=0; j<3; j++) {
for (int k=0; k<3; k++)
fill_pattern[2][j][k] = false;
}
}
if ( !periodic[1] && info.jy==0 ) {
for (int i=0; i<3; i++) {
for (int k=0; k<3; k++)
fill_pattern[i][0][k] = false;
}
}
if ( !periodic[1] && info.jy==info.ny-1 ) {
for (int i=0; i<3; i++) {
for (int k=0; k<3; k++)
fill_pattern[i][2][k] = false;
}
}
if ( !periodic[2] && info.kz==0 ) {
for (int i=0; i<3; i++) {
for (int j=0; j<3; j++)
fill_pattern[i][j][0] = false;
}
}
if ( !periodic[2] && info.kz==info.nz-1 ) {
for (int i=0; i<3; i++) {
for (int j=0; j<3; j++)
fill_pattern[i][j][2] = false;
}
}
// Determine the number of elements for each send/recv
for (int i=0; i<3; i++) {
int ni = (i-1)==0 ? nx:ngx;
int ni = (i-1)==0 ? n[0]:ng[0];
for (int j=0; j<3; j++) {
int nj = (j-1)==0 ? ny:ngy;
int nj = (j-1)==0 ? n[1]:ng[1];
for (int k=0; k<3; k++) {
int nk = (k-1)==0 ? nz:ngz;
int nk = (k-1)==0 ? n[2]:ng[2];
if ( fill_pattern[i][j][k] )
N_send_recv[i][j][k] = ni*nj*nk;
else
@ -106,9 +157,9 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
{
//PROFILE_START("fillHalo::fill",1);
int depth2 = data.size(3);
ASSERT((int)data.size(0)==nx+2*ngx);
ASSERT((int)data.size(1)==ny+2*ngy);
ASSERT((int)data.size(2)==nz+2*ngz);
ASSERT((int)data.size(0)==n[0]+2*ng[0]);
ASSERT((int)data.size(1)==n[1]+2*ng[1]);
ASSERT((int)data.size(2)==n[2]+2*ng[2]);
ASSERT(depth2<=depth);
ASSERT(data.ndim()==3||data.ndim()==4);
// Start the recieves
@ -117,7 +168,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k], datatype,
MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] );
}
}
@ -129,7 +180,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
if ( !fill_pattern[i][j][k] )
continue;
pack( data, i-1, j-1, k-1, send[i][j][k] );
MPI_Isend( send[i][j][k], depth2*N_send_recv[i][j][k], datatype,
MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] );
}
}
@ -162,12 +213,12 @@ template<class TYPE>
void fillHalo<TYPE>::pack( const Array<TYPE>& data, int i0, int j0, int k0, TYPE *buffer )
{
int depth2 = data.size(3);
int ni = i0==0 ? nx:ngx;
int nj = j0==0 ? ny:ngy;
int nk = k0==0 ? nz:ngz;
int is = i0==0 ? ngx:((i0==-1)?ngx:nx);
int js = j0==0 ? ngy:((j0==-1)?ngy:ny);
int ks = k0==0 ? ngz:((k0==-1)?ngz:nz);
int ni = i0==0 ? n[0]:ng[0];
int nj = j0==0 ? n[1]:ng[1];
int nk = k0==0 ? n[2]:ng[2];
int is = i0==0 ? ng[0]:((i0==-1)?ng[0]:n[0]);
int js = j0==0 ? ng[1]:((j0==-1)?ng[1]:n[1]);
int ks = k0==0 ? ng[2]:((k0==-1)?ng[2]:n[2]);
for (int d=0; d<depth2; d++) {
for (int k=0; k<nk; k++) {
for (int j=0; j<nj; j++) {
@ -182,12 +233,12 @@ template<class TYPE>
void fillHalo<TYPE>::unpack( Array<TYPE>& data, int i0, int j0, int k0, const TYPE *buffer )
{
int depth2 = data.size(3);
int ni = i0==0 ? nx:ngx;
int nj = j0==0 ? ny:ngy;
int nk = k0==0 ? nz:ngz;
int is = i0==0 ? ngx:((i0==-1)?0:nx+ngx);
int js = j0==0 ? ngy:((j0==-1)?0:ny+ngy);
int ks = k0==0 ? ngz:((k0==-1)?0:nz+ngz);
int ni = i0==0 ? n[0]:ng[0];
int nj = j0==0 ? n[1]:ng[1];
int nk = k0==0 ? n[2]:ng[2];
int is = i0==0 ? ng[0]:((i0==-1)?0:n[0]+ng[0]);
int js = j0==0 ? ng[1]:((j0==-1)?0:n[1]+ng[1]);
int ks = k0==0 ? ng[2]:((k0==-1)?0:n[2]+ng[2]);
for (int d=0; d<depth2; d++) {
for (int k=0; k<nk; k++) {
for (int j=0; j<nj; j++) {
@ -208,27 +259,27 @@ template<class TYPE1, class TYPE2>
void fillHalo<TYPE>::copy( const Array<TYPE1>& src, Array<TYPE2>& dst )
{
//PROFILE_START("fillHalo::copy",1);
ASSERT( (int)src.size(0)==nx || (int)src.size(0)==nx+2*ngx );
ASSERT( (int)dst.size(0)==nx || (int)dst.size(0)==nx+2*ngx );
bool src_halo = (int)src.size(0)==nx+2*ngx;
bool dst_halo = (int)dst.size(0)==nx+2*ngx;
ASSERT( (int)src.size(0)==n[0] || (int)src.size(0)==n[0]+2*ng[0] );
ASSERT( (int)dst.size(0)==n[0] || (int)dst.size(0)==n[0]+2*ng[0] );
bool src_halo = (int)src.size(0)==n[0]+2*ng[0];
bool dst_halo = (int)dst.size(0)==n[0]+2*ng[0];
if ( src_halo ) {
ASSERT((int)src.size(0)==nx+2*ngx);
ASSERT((int)src.size(1)==ny+2*ngy);
ASSERT((int)src.size(2)==nz+2*ngz);
ASSERT((int)src.size(0)==n[0]+2*ng[0]);
ASSERT((int)src.size(1)==n[1]+2*ng[1]);
ASSERT((int)src.size(2)==n[2]+2*ng[2]);
} else {
ASSERT((int)src.size(0)==nx);
ASSERT((int)src.size(1)==ny);
ASSERT((int)src.size(2)==nz);
ASSERT((int)src.size(0)==n[0]);
ASSERT((int)src.size(1)==n[1]);
ASSERT((int)src.size(2)==n[2]);
}
if ( dst_halo ) {
ASSERT((int)dst.size(0)==nx+2*ngx);
ASSERT((int)dst.size(1)==ny+2*ngy);
ASSERT((int)dst.size(2)==nz+2*ngz);
ASSERT((int)dst.size(0)==n[0]+2*ng[0]);
ASSERT((int)dst.size(1)==n[1]+2*ng[1]);
ASSERT((int)dst.size(2)==n[2]+2*ng[2]);
} else {
ASSERT((int)dst.size(0)==nx);
ASSERT((int)dst.size(1)==ny);
ASSERT((int)dst.size(2)==nz);
ASSERT((int)dst.size(0)==n[0]);
ASSERT((int)dst.size(1)==n[1]);
ASSERT((int)dst.size(2)==n[2]);
}
if ( src_halo == dst_halo ) {
// Src and dst halos match
@ -236,19 +287,19 @@ void fillHalo<TYPE>::copy( const Array<TYPE1>& src, Array<TYPE2>& dst )
dst(i) = src(i);
} else if ( src_halo && !dst_halo ) {
// Src has halos
for (int k=0; k<nz; k++) {
for (int j=0; j<ny; j++) {
for (int i=0; i<nx; i++) {
dst(i,j,k) = src(i+ngx,j+ngy,k+ngz);
for (int k=0; k<n[2]; k++) {
for (int j=0; j<n[1]; j++) {
for (int i=0; i<n[0]; i++) {
dst(i,j,k) = src(i+ng[0],j+ng[1],k+ng[2]);
}
}
}
} else if ( !src_halo && dst_halo ) {
// Dst has halos
for (int k=0; k<nz; k++) {
for (int j=0; j<ny; j++) {
for (int i=0; i<nx; i++) {
dst(i+ngx,j+ngy,k+ngz) = src(i,j,k);
for (int k=0; k<n[2]; k++) {
for (int j=0; j<n[1]; j++) {
for (int i=0; i<n[0]; i++) {
dst(i+ng[0],j+ng[1],k+ng[2]) = src(i,j,k);
}
}
}

597
common/Database.cpp Normal file
View File

@ -0,0 +1,597 @@
#include "common/Database.h"
#include "common/Utilities.h"
#include <algorithm>
#include <cstring>
#include <iomanip>
#include <sstream>
#include <string>
#include <tuple>
/********************************************************************
* Constructors/destructor *
********************************************************************/
Database::Database() = default;
Database::~Database() = default;
Database::Database( const Database& rhs ) : KeyData( rhs )
{
d_data.clear();
for ( const auto& tmp : rhs.d_data )
putData( tmp.first, tmp.second->clone() );
}
Database& Database::operator=( const Database& rhs )
{
if ( this == &rhs )
return *this;
d_data.clear();
for ( const auto& tmp : rhs.d_data )
putData( tmp.first, tmp.second->clone() );
return *this;
}
Database::Database( Database&& rhs ) { std::swap( d_data, rhs.d_data ); }
Database& Database::operator=( Database&& rhs )
{
if ( this != &rhs )
std::swap( d_data, rhs.d_data );
return *this;
}
/********************************************************************
* Clone the database *
********************************************************************/
std::shared_ptr<KeyData> Database::clone() const { return cloneDatabase(); }
std::shared_ptr<Database> Database::cloneDatabase() const
{
auto db = std::make_shared<Database>();
for ( const auto& tmp : d_data )
db->putData( tmp.first, tmp.second->clone() );
return db;
}
/********************************************************************
* Get the data object *
********************************************************************/
bool Database::keyExists( const std::string& key ) const
{
return d_data.find( key ) != d_data.end();
}
std::shared_ptr<KeyData> Database::getData( const std::string& key )
{
auto it = d_data.find( key );
if ( it == d_data.end() ) {
char msg[1000];
sprintf( msg, "Variable %s was not found in database", key.c_str() );
ERROR( msg );
}
return it->second;
}
std::shared_ptr<const KeyData> Database::getData( const std::string& key ) const
{
return const_cast<Database*>( this )->getData( key );
}
bool Database::isDatabase( const std::string& key ) const
{
auto ptr = getData( key );
auto ptr2 = std::dynamic_pointer_cast<const Database>( ptr );
return ptr2 != nullptr;
}
std::shared_ptr<Database> Database::getDatabase( const std::string& key )
{
std::shared_ptr<KeyData> ptr = getData( key );
std::shared_ptr<Database> ptr2 = std::dynamic_pointer_cast<Database>( ptr );
if ( ptr2 == nullptr ) {
char msg[1000];
sprintf( msg, "Variable %s is not a database", key.c_str() );
ERROR( msg );
}
return ptr2;
}
std::shared_ptr<const Database> Database::getDatabase( const std::string& key ) const
{
return const_cast<Database*>( this )->getDatabase( key );
}
std::vector<std::string> Database::getAllKeys() const
{
std::vector<std::string> keys;
keys.reserve( d_data.size() );
for ( const auto& it : d_data )
keys.push_back( it.first );
return keys;
}
void Database::putDatabase( const std::string& key, std::shared_ptr<Database> db )
{
d_data[key] = std::move( db );
}
void Database::putData( const std::string& key, std::shared_ptr<KeyData> data )
{
d_data[key] = std::move( data );
}
/********************************************************************
* Is the data of the given type *
********************************************************************/
template<>
bool Database::isType<double>( const std::string& key ) const
{
auto type = getData( key )->type();
return type == "double";
}
template<>
bool Database::isType<float>( const std::string& key ) const
{
auto type = getData( key )->type();
return type == "double";
}
template<>
bool Database::isType<int>( const std::string& key ) const
{
bool pass = true;
auto type = getData( key )->type();
if ( type == "double" ) {
auto data = getVector<double>( key );
for ( auto tmp : data )
pass = pass && static_cast<double>( static_cast<int>( tmp ) ) == tmp;
} else {
pass = false;
}
return pass;
}
template<>
bool Database::isType<std::string>( const std::string& key ) const
{
auto type = getData( key )->type();
return type == "string";
}
template<>
bool Database::isType<bool>( const std::string& key ) const
{
auto type = getData( key )->type();
return type == "bool";
}
/********************************************************************
* Get a vector *
********************************************************************/
template<>
std::vector<std::string> Database::getVector<std::string>(
const std::string& key, const Units& ) const
{
std::shared_ptr<const KeyData> ptr = getData( key );
if ( std::dynamic_pointer_cast<const EmptyKeyData>( ptr ) )
return std::vector<std::string>();
const auto* ptr2 = dynamic_cast<const KeyDataString*>( ptr.get() );
if ( ptr2 == nullptr ) {
ERROR( "Key '" + key + "' is not a string" );
}
return ptr2->d_data;
}
template<>
std::vector<bool> Database::getVector<bool>( const std::string& key, const Units& ) const
{
std::shared_ptr<const KeyData> ptr = getData( key );
if ( std::dynamic_pointer_cast<const EmptyKeyData>( ptr ) )
return std::vector<bool>();
const auto* ptr2 = dynamic_cast<const KeyDataBool*>( ptr.get() );
if ( ptr2 == nullptr ) {
ERROR( "Key '" + key + "' is not a bool" );
}
return ptr2->d_data;
}
template<class TYPE>
std::vector<TYPE> Database::getVector( const std::string& key, const Units& unit ) const
{
std::shared_ptr<const KeyData> ptr = getData( key );
if ( std::dynamic_pointer_cast<const EmptyKeyData>( ptr ) )
return std::vector<TYPE>();
std::vector<TYPE> data;
if ( std::dynamic_pointer_cast<const KeyDataDouble>( ptr ) ) {
const auto* ptr2 = dynamic_cast<const KeyDataDouble*>( ptr.get() );
const std::vector<double>& data2 = ptr2->d_data;
double factor = 1;
if ( !unit.isNull() ) {
INSIST( !ptr2->d_unit.isNull(), "Field " + key + " must have units" );
factor = ptr2->d_unit.convert( unit );
INSIST( factor != 0, "Unit conversion failed" );
}
data.resize( data2.size() );
for ( size_t i = 0; i < data2.size(); i++ )
data[i] = static_cast<TYPE>( factor * data2[i] );
} else if ( std::dynamic_pointer_cast<const KeyDataString>( ptr ) ) {
ERROR( "Converting std::string to another type" );
} else if ( std::dynamic_pointer_cast<const KeyDataBool>( ptr ) ) {
ERROR( "Converting std::bool to another type" );
} else {
ERROR( "Unable to convert data format" );
}
return data;
}
/********************************************************************
* Put a vector *
********************************************************************/
template<>
void Database::putVector<std::string>(
const std::string& key, const std::vector<std::string>& data, const Units& )
{
std::shared_ptr<KeyDataString> ptr( new KeyDataString() );
ptr->d_data = data;
d_data[key] = ptr;
}
template<>
void Database::putVector<bool>(
const std::string& key, const std::vector<bool>& data, const Units& )
{
std::shared_ptr<KeyDataBool> ptr( new KeyDataBool() );
ptr->d_data = data;
d_data[key] = ptr;
}
template<class TYPE>
void Database::putVector( const std::string& key, const std::vector<TYPE>& data, const Units& unit )
{
std::shared_ptr<KeyDataDouble> ptr( new KeyDataDouble() );
ptr->d_unit = unit;
ptr->d_data.resize( data.size() );
for ( size_t i = 0; i < data.size(); i++ )
ptr->d_data[i] = static_cast<double>( data[i] );
d_data[key] = ptr;
}
/********************************************************************
* Print the database *
********************************************************************/
void Database::print( std::ostream& os, const std::string& indent ) const
{
for ( const auto& it : d_data ) {
os << indent << it.first;
if ( dynamic_cast<const Database*>( it.second.get() ) ) {
const auto* db = dynamic_cast<const Database*>( it.second.get() );
os << " {\n";
db->print( os, indent + " " );
os << indent << "}\n";
} else {
os << " = ";
it.second->print( os, "" );
}
}
}
std::string Database::print( const std::string& indent ) const
{
std::stringstream ss;
print( ss, indent );
return ss.str();
}
/********************************************************************
* Read input database file *
********************************************************************/
Database::Database( const std::string& filename )
{
// Read the input file into memory
FILE* fid = fopen( filename.c_str(), "rb" );
if ( fid == nullptr )
ERROR( "Error opening file " + filename );
fseek( fid, 0, SEEK_END );
size_t bytes = ftell( fid );
rewind( fid );
auto* buffer = new char[bytes + 4];
size_t result = fread( buffer, 1, bytes, fid );
fclose( fid );
if ( result != bytes )
ERROR( "Error reading file " + filename );
buffer[bytes + 0] = '\n';
buffer[bytes + 1] = '}';
buffer[bytes + 2] = '\n';
buffer[bytes + 3] = 0;
// Create the database entries
loadDatabase( buffer, *this );
// Free temporary memory
delete[] buffer;
}
std::shared_ptr<Database> Database::createFromString( const std::string& data )
{
std::shared_ptr<Database> db( new Database() );
auto* buffer = new char[data.size() + 4];
memcpy( buffer, data.data(), data.size() );
buffer[data.size() + 0] = '\n';
buffer[data.size() + 1] = '}';
buffer[data.size() + 2] = '\n';
buffer[data.size() + 3] = 0;
loadDatabase( buffer, *db );
delete[] buffer;
return db;
}
enum class token_type {
newline,
line_comment,
block_start,
block_stop,
quote,
equal,
bracket,
end_bracket,
end
};
inline size_t length( token_type type )
{
size_t len = 0;
if ( type == token_type::newline || type == token_type::quote || type == token_type::equal ||
type == token_type::bracket || type == token_type::end_bracket ||
type == token_type::end ) {
len = 1;
} else if ( type == token_type::line_comment || type == token_type::block_start ||
type == token_type::block_stop ) {
len = 2;
}
return len;
}
inline std::tuple<size_t, token_type> find_next_token( const char* buffer )
{
size_t i = 0;
while ( true ) {
if ( buffer[i] == '\n' || buffer[i] == '\r' ) {
return std::pair<size_t, token_type>( i + 1, token_type::newline );
} else if ( buffer[i] == 0 ) {
return std::pair<size_t, token_type>( i + 1, token_type::end );
} else if ( buffer[i] == '"' ) {
return std::pair<size_t, token_type>( i + 1, token_type::quote );
} else if ( buffer[i] == '=' ) {
return std::pair<size_t, token_type>( i + 1, token_type::equal );
} else if ( buffer[i] == '{' ) {
return std::pair<size_t, token_type>( i + 1, token_type::bracket );
} else if ( buffer[i] == '}' ) {
return std::pair<size_t, token_type>( i + 1, token_type::end_bracket );
} else if ( buffer[i] == '/' ) {
if ( buffer[i + 1] == '/' ) {
return std::pair<size_t, token_type>( i + 2, token_type::line_comment );
} else if ( buffer[i + 1] == '*' ) {
return std::pair<size_t, token_type>( i + 2, token_type::block_start );
}
} else if ( buffer[i] == '*' ) {
if ( buffer[i + 1] == '/' )
return std::pair<size_t, token_type>( i + 2, token_type::block_stop );
}
i++;
}
return std::pair<size_t, token_type>( 0, token_type::end );
}
inline std::string deblank( const std::string& str )
{
size_t i1 = 0xFFFFFFF, i2 = 0;
for ( size_t i = 0; i < str.size(); i++ ) {
if ( str[i] != ' ' ) {
i1 = std::min( i1, i );
i2 = std::max( i2, i );
}
}
return i1 <= i2 ? str.substr( i1, i2 - i1 + 1 ) : std::string();
}
size_t skip_comment( const char* buffer )
{
auto tmp = find_next_token( buffer );
const token_type end_comment = ( std::get<1>( tmp ) == token_type::line_comment ) ?
token_type::newline :
token_type::block_stop;
size_t pos = 0;
while ( std::get<1>( tmp ) != end_comment ) {
if ( std::get<1>( tmp ) == token_type::end )
ERROR( "Encountered end of file before block comment end" );
pos += std::get<0>( tmp );
tmp = find_next_token( &buffer[pos] );
}
pos += std::get<0>( tmp );
return pos;
}
inline std::string lower( const std::string& str )
{
std::string tmp( str );
std::transform( tmp.begin(), tmp.end(), tmp.begin(), ::tolower );
return tmp;
}
static std::tuple<size_t, std::shared_ptr<KeyData>> read_value(
const char* buffer, const std::string& key )
{
// Get the value as a std::string
size_t pos = 0;
token_type type = token_type::end;
std::tie( pos, type ) = find_next_token( &buffer[pos] );
size_t len = pos - length( type );
while ( type != token_type::newline ) {
if ( type == token_type::quote ) {
size_t i = 0;
std::tie( i, type ) = find_next_token( &buffer[pos] );
pos += i;
while ( type != token_type::quote ) {
ASSERT( type != token_type::end );
std::tie( i, type ) = find_next_token( &buffer[pos] );
pos += i;
}
} else if ( type == token_type::line_comment || type == token_type::block_start ) {
len = pos - length( type );
pos += skip_comment( &buffer[pos - length( type )] ) - length( type );
break;
}
size_t i = 0;
std::tie( i, type ) = find_next_token( &buffer[pos] );
pos += i;
len = pos - length( type );
}
const std::string value = deblank( std::string( buffer, len ) );
// Split the value to an array of values
std::vector<std::string> values;
size_t i0 = 0, i = 0, count = 0;
for ( ; i < value.size(); i++ ) {
if ( value[i] == '"' ) {
count++;
} else if ( value[i] == ',' && count % 2 == 0 ) {
values.push_back( deblank( value.substr( i0, i - i0 ) ) );
i0 = i + 1;
}
}
values.push_back( deblank( value.substr( i0 ) ) );
// Convert the string value to the database value
std::shared_ptr<KeyData> data;
if ( value.empty() ) {
data.reset( new EmptyKeyData() );
} else if ( value.find( '"' ) != std::string::npos ) {
auto* data2 = new KeyDataString();
data.reset( data2 );
data2->d_data.resize( values.size() );
for ( size_t i = 0; i < values.size(); i++ ) {
ASSERT( values[i].size() >= 2 );
ASSERT( values[i][0] == '"' && values[i][values[i].size() - 1] == '"' );
data2->d_data[i] = values[i].substr( 1, values[i].size() - 2 );
}
} else if ( lower( value ) == "true" || lower( value ) == "false" ) {
auto* data2 = new KeyDataBool();
data.reset( data2 );
data2->d_data.resize( values.size() );
for ( size_t i = 0; i < values.size(); i++ ) {
ASSERT( values[i].size() >= 2 );
if ( lower( values[i] ) != "true" && lower( values[i] ) != "false" )
ERROR( "Error converting " + key + " to logical array" );
data2->d_data[i] = lower( values[i] ) == "true";
}
} else { // if ( value.find('.')!=std::string::npos || value.find('e')!=std::string::npos ) {
auto* data2 = new KeyDataDouble();
data.reset( data2 );
data2->d_data.resize( values.size(), 0 );
for ( size_t i = 0; i < values.size(); i++ ) {
Units unit;
std::tie( data2->d_data[i], unit ) = KeyDataDouble::read( values[i] );
if ( !unit.isNull() )
data2->d_unit = unit;
}
//} else {
// ERROR("Unable to determine data type: "+value);
}
return std::tuple<size_t, std::shared_ptr<KeyData>>( pos, data );
}
size_t Database::loadDatabase( const char* buffer, Database& db )
{
size_t pos = 0;
while ( true ) {
size_t i;
token_type type;
std::tie( i, type ) = find_next_token( &buffer[pos] );
const std::string key =
deblank( std::string( &buffer[pos], std::max<int>( i - length( type ), 1 ) - 1 ) );
if ( type == token_type::line_comment || type == token_type::block_start ) {
// Comment
INSIST( key.empty(), "Key should be empty: " + key );
pos += skip_comment( &buffer[pos] );
} else if ( type == token_type::newline ) {
INSIST( key.empty(), "Key should be empty: " + key );
pos += i;
} else if ( type == token_type::equal ) {
// Reading key/value pair
ASSERT( !key.empty() );
pos += i;
std::shared_ptr<KeyData> data;
std::tie( i, data ) = read_value( &buffer[pos], key );
ASSERT( data.get() != nullptr );
db.d_data[key] = data;
pos += i;
} else if ( type == token_type::bracket ) {
// Read database
ASSERT( !key.empty() );
pos += i;
std::shared_ptr<Database> database( new Database() );
pos += loadDatabase( &buffer[pos], *database );
db.d_data[key] = database;
} else if ( type == token_type::end_bracket ) {
// Finished with the database
pos += i;
break;
} else {
ERROR( "Error loading data" );
}
}
return pos;
}
/********************************************************************
* Data type helper functions *
********************************************************************/
void KeyDataDouble::print( std::ostream& os, const std::string& indent ) const
{
os << indent;
for ( size_t i = 0; i < d_data.size(); i++ ) {
if ( i > 0 )
os << ", ";
if ( d_data[i] != d_data[i] ) {
os << "nan";
} else if ( d_data[i] == std::numeric_limits<double>::infinity() ) {
os << "inf";
} else if ( d_data[i] == -std::numeric_limits<double>::infinity() ) {
os << "-inf";
} else {
os << std::setprecision( 12 ) << d_data[i];
}
}
if ( !d_unit.isNull() )
os << " " << d_unit.str();
os << std::endl;
}
std::tuple<double, Units> KeyDataDouble::read( const std::string& str )
{
std::string tmp = deblank( str );
size_t index = tmp.find( " " );
if ( index != std::string::npos ) {
return std::make_tuple(
readValue( tmp.substr( 0, index ) ), Units( tmp.substr( index + 1 ) ) );
} else {
return std::make_tuple( readValue( tmp ), Units() );
}
}
double KeyDataDouble::readValue( const std::string& str )
{
const std::string tmp = lower( str );
double data = 0;
if ( tmp == "inf" || tmp == "infinity" ) {
data = std::numeric_limits<double>::infinity();
} else if ( tmp == "-inf" || tmp == "-infinity" ) {
data = -std::numeric_limits<double>::infinity();
} else if ( tmp == "nan" ) {
data = std::numeric_limits<double>::quiet_NaN();
} else if ( tmp.find( '/' ) != std::string::npos ) {
ERROR( "Error reading value" );
} else {
char* pos = nullptr;
data = strtod( tmp.c_str(), &pos );
if ( static_cast<size_t>( pos - tmp.c_str() ) == tmp.size() + 1 )
ERROR( "Error reading value" );
}
return data;
}
/********************************************************************
* Instantiations *
********************************************************************/
template std::vector<char> Database::getVector<char>( const std::string&, const Units& ) const;
template std::vector<int> Database::getVector<int>( const std::string&, const Units& ) const;
template std::vector<size_t> Database::getVector<size_t>( const std::string&, const Units& ) const;
template std::vector<float> Database::getVector<float>( const std::string&, const Units& ) const;
template std::vector<double> Database::getVector<double>( const std::string&, const Units& ) const;
template void Database::putVector<char>(
const std::string&, const std::vector<char>&, const Units& );
template void Database::putVector<int>( const std::string&, const std::vector<int>&, const Units& );
template void Database::putVector<size_t>(
const std::string&, const std::vector<size_t>&, const Units& );
template void Database::putVector<float>(
const std::string&, const std::vector<float>&, const Units& );
template void Database::putVector<double>(
const std::string&, const std::vector<double>&, const Units& );
template bool Database::isType<int>( const std::string& ) const;
template bool Database::isType<float>( const std::string& ) const;
template bool Database::isType<double>( const std::string& ) const;
template bool Database::isType<std::string>( const std::string& ) const;

296
common/Database.h Normal file
View File

@ -0,0 +1,296 @@
#ifndef included_Database
#define included_Database
#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "common/Units.h"
inline bool exists( const std::string& filename )
{
std::ifstream domain( filename );
return domain.good();
}
//! Base class to hold data of a given type
class KeyData
{
protected:
//! Empty constructor
KeyData() {}
public:
//! Destructor
virtual ~KeyData() {}
//! Copy the data
virtual std::shared_ptr<KeyData> clone() const = 0;
//! Print the data to a stream
virtual void print( std::ostream& os, const std::string& indent = "" ) const = 0;
//! Return the native data type
virtual std::string type() const = 0;
protected:
KeyData( const KeyData& ) {}
KeyData& operator=( const KeyData& );
};
//! Class to a database
class Database : public KeyData
{
public:
//! Empty constructor
Database();
/**
* Open an database file.
* @param filename Name of input file to open
*/
explicit Database( const std::string& filename );
/**
* Create database from string
* @param data String containing the database data
*/
static std::shared_ptr<Database> createFromString( const std::string& data );
//! Copy constructor
Database( const Database& );
//! Assignment operator
Database& operator=( const Database& );
//! Move constructor
Database( Database&& rhs );
//! Move assignment operator
Database& operator=( Database&& rhs );
//! Destructor
virtual ~Database();
//! Copy the data
virtual std::shared_ptr<KeyData> clone() const override;
//! Copy the data
std::shared_ptr<Database> cloneDatabase() const;
/**
* Return true if the specified key exists in the database and false
* otherwise.
* @param[in] key Key name to lookup.
*/
bool keyExists( const std::string& key ) const;
/**
* Return all keys in the database.
*/
std::vector<std::string> getAllKeys() const;
//! Return the number of entries in the database
size_t size() const { return d_data.size(); }
/**
* Get the scalar entry from the database with the specified key
* name. If the specified key does not exist in the database or
* is not a scalar of the given type, then an error message is printed and
* the program exits.
*
* @param[in] key Key name in database.
* @param[in] unit Desired units
*/
template<class TYPE>
inline TYPE getScalar( const std::string& key, const Units& unit = Units() ) const;
/// @copydoc Database::getScalar(const std::string&,const Units&) const
template<class TYPE>
inline TYPE getScalar( const std::string& key, const std::string& unit ) const;
/**
* Get the scalar entry from the database with the specified key
* name. If the specified key does not exist in the database the
* the default value will be printed
*
* @param[in] key Key name in database
* @param[in] value Default value
* @param[in] unit Desired units
*/
template<class TYPE>
inline TYPE getWithDefault(
const std::string& key, const TYPE& value, const Units& unit = Units() ) const;
/// @copydoc Database::getWithDefault(const std::string&,const TYPE&,const Units&) const
template<class TYPE>
inline TYPE getWithDefault(
const std::string& key, const TYPE& value, const std::string& unit ) const;
/**
* Put the scalar entry into the database with the specified key name.
* @param key Key name in database.
* @param value Value to store
* @param unit Desired units
*/
template<class TYPE>
inline void putScalar( const std::string& key, const TYPE& value, const Units& unit = Units() );
/**
* Put the scalar entry into the database with the specified key name.
* @param key Key name in database.
* @param value Value to store
* @param unit Desired units
*/
template<class TYPE>
inline void putScalar( const std::string& key, const TYPE& value, const std::string& unit );
/**
* Get the vector entries from the database with the specified key
* name. If the specified key does not exist in the database or
* is not of the given type, then an error message is printed and
* the program exits.
*
* @param key Key name in database.
* @param unit Desired units
*/
template<class TYPE>
std::vector<TYPE> getVector( const std::string& key, const Units& unit = Units() ) const;
/// @copydoc Database::getVector(const std::string&,const Units&) const
template<class TYPE>
inline std::vector<TYPE> getVector( const std::string& key, const std::string& unit ) const;
/**
* Put the vector entries into the database with the specified key
* name. If the specified key does not exist in the database or
* is not of the given type, then an error message is printed and
* the program exits.
*
* @param key Key name in database.
* @param data Data to store
* @param unit Desired units
*/
template<class TYPE>
void putVector(
const std::string& key, const std::vector<TYPE>& data, const Units& unit = Units() );
/// @copydoc Database::putVector(const std::string&,const std::vector<TYPE>&,const Units&)
template<class TYPE>
inline void putVector(
const std::string& key, const std::vector<TYPE>& data, const std::string& unit );
/**
* Get the data for a key in the database. If the specified key
* does not exist in the database an error message is printed and
* the program exits.
*
* @param key Key name in database.
*/
std::shared_ptr<KeyData> getData( const std::string& key );
/**
* Get the data for a key in the database. If the specified key
* does not exist in the database an error message is printed and
* the program exits.
*
* @param key Key name in database.
*/
std::shared_ptr<const KeyData> getData( const std::string& key ) const;
/**
* Put the data for a key in the database.
*
* @param key Key name in database.
* @param data Data to store
*/
void putData( const std::string& key, std::shared_ptr<KeyData> data );
// Check if the key is a database object
bool isDatabase( const std::string& key ) const;
// Check if the entry can be stored as the given type
template<class TYPE>
bool isType( const std::string& key ) const;
/**
* Get the database for a key in the database. If the specified key
* does not exist in the database an error message is printed and
* the program exits.
*
* @param key Key name in database.
*/
std::shared_ptr<Database> getDatabase( const std::string& key );
/**
* Get the database for a key in the database. If the specified key
* does not exist in the database an error message is printed and
* the program exits.
*
* @param key Key name in database.
*/
std::shared_ptr<const Database> getDatabase( const std::string& key ) const;
/**
* Get the database for a key in the database. If the specified key
* does not exist in the database an error message is printed and
* the program exits.
*
* @param key Key name in database.
* @param db Database to store
*/
void putDatabase( const std::string& key, std::shared_ptr<Database> db );
/**
* Print the data to a stream
* @param os Output stream
* @param indent Indenting to use before each line
*/
virtual void print( std::ostream& os, const std::string& indent = "" ) const override;
//! Print the type
virtual std::string type() const override { return "database"; }
/**
* Print the data to a string
* @return Output string
*/
std::string print( const std::string& indent = "" ) const;
protected:
std::map<std::string, std::shared_ptr<KeyData>> d_data;
// Function to load a database from a buffer
static size_t loadDatabase( const char* buffer, Database& db );
};
#include "common/Database.hpp"
#endif

173
common/Database.hpp Normal file
View File

@ -0,0 +1,173 @@
#ifndef included_Database_hpp
#define included_Database_hpp
#include "common/Database.h"
#include "common/Utilities.h"
#include <tuple>
/********************************************************************
* Basic classes for primative data types *
********************************************************************/
class EmptyKeyData : public KeyData
{
public:
EmptyKeyData() {}
virtual ~EmptyKeyData() {}
virtual std::shared_ptr<KeyData> clone() const override
{
return std::make_shared<EmptyKeyData>();
}
virtual void print( std::ostream& os, const std::string& = "" ) const override
{
os << std::endl;
}
virtual std::string type() const override { return ""; }
};
class KeyDataDouble : public KeyData
{
public:
KeyDataDouble() {}
explicit KeyDataDouble( const std::vector<double>& data, const Units& unit )
: d_data( data ), d_unit( unit )
{
}
virtual ~KeyDataDouble() {}
virtual std::shared_ptr<KeyData> clone() const override
{
return std::make_shared<KeyDataDouble>( d_data, d_unit );
}
virtual void print( std::ostream& os, const std::string& indent = "" ) const override;
virtual std::string type() const override { return "double"; }
static std::tuple<double, Units> read( const std::string& );
static double readValue( const std::string& );
public:
std::vector<double> d_data;
Units d_unit;
};
class KeyDataBool : public KeyData
{
public:
KeyDataBool() {}
explicit KeyDataBool( const std::vector<bool>& data ) : d_data( data ) {}
virtual ~KeyDataBool() {}
virtual std::shared_ptr<KeyData> clone() const override
{
return std::make_shared<KeyDataBool>( d_data );
}
virtual void print( std::ostream& os, const std::string& indent = "" ) const override
{
os << indent;
for ( size_t i = 0; i < d_data.size(); i++ ) {
if ( i > 0 ) {
os << ", ";
}
if ( d_data[i] ) {
os << "true";
} else {
os << "false";
}
}
os << std::endl;
}
virtual std::string type() const override { return "bool"; }
std::vector<bool> d_data;
};
class KeyDataString : public KeyData
{
public:
KeyDataString() {}
explicit KeyDataString( const std::vector<std::string>& data ) : d_data( data ) {}
virtual ~KeyDataString() {}
virtual std::shared_ptr<KeyData> clone() const override
{
return std::make_shared<KeyDataString>( d_data );
}
virtual void print( std::ostream& os, const std::string& indent = "" ) const override
{
os << indent;
for ( size_t i = 0; i < d_data.size(); i++ ) {
if ( i > 0 ) {
os << ", ";
}
os << '"' << d_data[i] << '"';
}
os << std::endl;
}
virtual std::string type() const override { return "string"; }
std::vector<std::string> d_data;
};
/********************************************************************
* Get a vector *
********************************************************************/
template<class TYPE>
inline std::vector<TYPE> Database::getVector(
const std::string& key, const std::string& unit ) const
{
return getVector<TYPE>( key, Units( unit ) );
}
template<class TYPE>
inline void Database::putVector(
const std::string& key, const std::vector<TYPE>& data, const std::string& unit )
{
putVector<TYPE>( key, data, Units( unit ) );
}
/********************************************************************
* Get a scalar *
********************************************************************/
template<class TYPE>
inline TYPE Database::getScalar( const std::string& key, const Units& unit ) const
{
const std::vector<TYPE>& data = getVector<TYPE>( key, unit );
if ( data.size() != 1 ) {
char msg[1000];
sprintf( msg, "Variable %s is not a scalar", key.c_str() );
ERROR( msg );
}
return data[0];
}
template<class TYPE>
inline TYPE Database::getWithDefault(
const std::string& key, const TYPE& value, const Units& unit ) const
{
if ( !keyExists( key ) )
return value;
return getScalar<TYPE>( key, unit );
}
template<class TYPE>
inline void Database::putScalar( const std::string& key, const TYPE& data, const Units& unit )
{
putVector<TYPE>( key, std::vector<TYPE>( 1, data ), unit );
}
template<class TYPE>
inline TYPE Database::getScalar( const std::string& key, const std::string& unit ) const
{
return getScalar<TYPE>( key, Units( unit ) );
}
template<class TYPE>
inline TYPE Database::getWithDefault(
const std::string& key, const TYPE& value, const std::string& unit ) const
{
return getWithDefault<TYPE>( key, value, Units( unit ) );
}
template<class TYPE>
inline void Database::putScalar( const std::string& key, const TYPE& data, const std::string& unit )
{
putScalar<TYPE>( key, data, Units( unit ) );
}
template<class TYPE>
inline void putVector(
const std::string& key, const std::vector<TYPE>& data, const std::string& unit )
{
putVector<TYPE>( key, data, Units( unit ) );
}
#endif

1257
common/Domain.cpp
View File

File diff suppressed because it is too large Load Diff

715
common/Domain.h
View File

@ -1,567 +1,240 @@
#ifndef Domain_INC
#define Domain_INC
// Created by James McClure
// Copyright 2008-2013
//#define _GLIBCXX_USE_CXX11_ABI 0
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <math.h>
#include <time.h>
#include <exception> // std::exception
#include <exception>
#include <stdexcept>
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
using namespace std;
#include "common/Database.h"
//! Read the domain information file
void read_domain( int rank, int nprocs, MPI_Comm comm,
int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz,
int& nspheres, double& Lx, double& Ly, double& Lz );
class Domain;
template<class TYPE> class PatchData;
//! Class to hold information about a box
class Box {
public:
int ifirst[3];
int ilast[3];
};
enum class DataLocation { CPU, DEVICE };
//! Class to hold information about a patch
class Patch {
public:
//! Empty constructor
Patch() = delete;
//! Copy constructor
Patch( const Patch& ) = delete;
//! Assignment operator
Patch& operator=( const Patch& ) = delete;
//! Return the box for the patch
inline const Box& getBox() const { return d_box; }
//! Create patch data
template<class TYPE>
std::shared_ptr<PatchData<TYPE>> createPatchData( DataLocation location ) const;
private:
Box d_box;
int d_owner;
Domain *d_domain;
};
//! Class to hold domain info
struct Domain{
// Default constructor
Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
double lx, double ly, double lz, int BC);
class Domain{
public:
//! Default constructor
Domain( std::shared_ptr<Database> db, MPI_Comm Communicator);
// Destructor
~Domain();
//! Obsolete constructor
Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
double lx, double ly, double lz, int BC);
//! Empty constructor
Domain() = delete;
//! Copy constructor
Domain( const Domain& ) = delete;
//! Assignment operator
Domain& operator=( const Domain& ) = delete;
//! Destructor
~Domain();
// Basic domain information
int Nx,Ny,Nz,N;
int iproc,jproc,kproc;
int nprocx,nprocy,nprocz;
double Lx,Ly,Lz,Volume;
int rank;
int BoundaryCondition;
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
MPI_Comm Comm; // MPI Communicator for this domain
//! Get the database
inline std::shared_ptr<const Database> getDatabase() const { return d_db; }
//**********************************
// MPI ranks for all 18 neighbors
//**********************************
int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
//......................................................................................
// Get the actual D3Q19 communication counts (based on location of solid phase)
// Discrete velocity set symmetry implies the sendcount = recvcount
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
//......................................................................................
int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
//......................................................................................
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
//......................................................................................
int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z;
int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ;
int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ;
//......................................................................................
double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z;
double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ;
double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ;
double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z;
double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ;
double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ;
//! Get the domain box
inline const Box& getBox() const { return d_box; }
// Solid indicator function
char *id;
//! Get local patch
inline const Patch& getLocalPatch() const { return *d_localPatch; }
void InitializeRanks();
void CommInit(MPI_Comm comm);
void CommunicateMeshHalo(DoubleArray &Mesh);
void AssignComponentLabels(double *phase);
//! Get all patches
inline const std::vector<Patch>& getAllPatch() const { return d_patches; }
void TestCommInit(MPI_Comm comm);
//void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np);
private:
inline int getRankForBlock( int i, int j, int k )
{
int i2 = (i+nprocx)%nprocx;
int j2 = (j+nprocy)%nprocy;
int k2 = (k+nprocz)%nprocz;
return i2 + j2*nprocx + k2*nprocx*nprocy;
}
void initialize( std::shared_ptr<Database> db );
std::shared_ptr<Database> d_db;
Box d_box;
Patch *d_localPatch;
std::vector<Patch> d_patches;
public: // Public variables (need to create accessors instead)
double Lx,Ly,Lz,Volume;
int Nx,Ny,Nz,N;
RankInfoStruct rank_info;
MPI_Comm Comm; // MPI Communicator for this domain
int BoundaryCondition;
MPI_Group Group; // Group of processors associated with this domain
//**********************************
// MPI ranks for all 18 neighbors
//**********************************
inline int iproc() const { return rank_info.ix; }
inline int jproc() const { return rank_info.jy; }
inline int kproc() const { return rank_info.kz; }
inline int nprocx() const { return rank_info.nx; }
inline int nprocy() const { return rank_info.ny; }
inline int nprocz() const { return rank_info.nz; }
inline int rank() const { return rank_info.rank[1][1][1]; }
inline int rank_X() const { return rank_info.rank[2][1][1]; }
inline int rank_x() const { return rank_info.rank[0][1][1]; }
inline int rank_Y() const { return rank_info.rank[1][2][1]; }
inline int rank_y() const { return rank_info.rank[1][0][1]; }
inline int rank_Z() const { return rank_info.rank[1][1][2]; }
inline int rank_z() const { return rank_info.rank[1][1][0]; }
inline int rank_XY() const { return rank_info.rank[2][2][1]; }
inline int rank_xy() const { return rank_info.rank[0][0][1]; }
inline int rank_Xy() const { return rank_info.rank[2][0][1]; }
inline int rank_xY() const { return rank_info.rank[0][2][1]; }
inline int rank_XZ() const { return rank_info.rank[2][1][2]; }
inline int rank_xz() const { return rank_info.rank[0][1][0]; }
inline int rank_Xz() const { return rank_info.rank[2][1][0]; }
inline int rank_xZ() const { return rank_info.rank[0][1][2]; }
inline int rank_YZ() const { return rank_info.rank[1][2][2]; }
inline int rank_yz() const { return rank_info.rank[1][0][0]; }
inline int rank_Yz() const { return rank_info.rank[1][2][0]; }
inline int rank_yZ() const { return rank_info.rank[1][0][2]; }
//**********************************
//......................................................................................
// Get the actual D3Q19 communication counts (based on location of solid phase)
// Discrete velocity set symmetry implies the sendcount = recvcount
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
//......................................................................................
int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
//......................................................................................
// Solid indicator function
char *id;
void ReadIDs();
void CommunicateMeshHalo(DoubleArray &Mesh);
void CommInit();
int PoreCount();
private:
//......................................................................................
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ;
//......................................................................................
int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z;
int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ;
int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ;
//......................................................................................
double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z;
double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ;
double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ;
double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z;
double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ;
double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ;
};
// Inline function to read line without a return argument
static inline void fgetl( char * str, int num, FILE * stream )
{
char* ptr = fgets( str, num, stream );
if ( 0 ) {char *temp = (char *)&ptr; temp++;}
}
// Class to hold data on a patch
template<class TYPE>
class PatchData {
public:
//! Get the raw data pointer
TYPE* data() { return d_data; }
//! Get the raw data pointer
const TYPE* data() const { return d_data; }
inline double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
/*
* This routine converts the data in the Distance array to a signed distance
* by solving the equation df/dt = sign(1-|grad f|), where Distance provides
* the values of f on the mesh associated with domain Dm
* It has been tested with segmented data initialized to values [-1,1]
* and will converge toward the signed distance to the surface bounding the associated phases
*/
//! Get the patch
const Patch& getPatch() const { return *d_patch; }
int Q=26;
int q,i,j,k;
double dt=0.1;
int in,jn,kn;
double Dqx,Dqy,Dqz,Dx,Dy,Dz,W;
double nx,ny,nz,Cqx,Cqy,Cqz,sign,norm;
double TotalVariation=0.0;
//! Start communication
void beginCommunication();
const static int D3Q27[26][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1},{1,1,1},{-1,-1,-1},{1,1,-1},{-1,-1,1},
{-1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1}};
//! End communication
void endCommunication();
double weights[26];
// Compute the weights from the finite differences
for (q=0; q<Q; q++){
weights[q] = sqrt(1.0*(D3Q27[q][0]*D3Q27[q][0]) + 1.0*(D3Q27[q][1]*D3Q27[q][1]) + 1.0*(D3Q27[q][2]*D3Q27[q][2]));
}
//! Access ghost values
TYPE operator()( int, int, int ) const;
int xdim,ydim,zdim;
xdim=Dm.Nx-2;
ydim=Dm.Ny-2;
zdim=Dm.Nz-2;
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
//! Copy data from another PatchData
void copy( const PatchData& rhs );
int count = 0;
while (count < timesteps){
private:
DataLocation d_location;
const Patch *d_patch;
TYPE *d_data;
TYPE *d_gcw;
};
// Communicate the halo of values
fillData.fill(Distance);
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np);
TotalVariation=0.0;
// Execute the next timestep
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
//sign = Distance(i,j,k) / fabs(Distance(i,j,k));
sign = -1;
if (ID[n] == 1) sign = 1;
/*
if (!(i+1<Nx)) nx=0.5*Distance(i,j,k);
else nx=0.5*Distance(i+1,j,k);;
if (!(j+1<Ny)) ny=0.5*Distance(i,j,k);
else ny=0.5*Distance(i,j+1,k);
if (!(k+1<Nz)) nz=0.5*Distance(i,j,k);
else nz=0.5*Distance(i,j,k+1);
if (i<1) nx-=0.5*Distance(i,j,k);
else nx-=0.5*Distance(i-1,j,k);
if (j<1) ny-=0.5*Distance(i,j,k);
else ny-=0.5*Distance(i,j-1,k);
if (k<1) nz-=0.5*Distance(i,j,k);
else nz-=0.5*Distance(i,j,k-1);
*/
//............Compute the Gradient...................................
nx = 0.5*(Distance(i+1,j,k) - Distance(i-1,j,k));
ny = 0.5*(Distance(i,j+1,k) - Distance(i,j-1,k));
nz = 0.5*(Distance(i,j,k+1) - Distance(i,j,k-1));
W = 0.0; Dx = Dy = Dz = 0.0;
// also ignore places where the gradient is zero since this will not
// result in any local change to Distance
if (nx*nx+ny*ny+nz*nz > 0.0 ){
for (q=0; q<26; q++){
Cqx = 1.0*D3Q27[q][0];
Cqy = 1.0*D3Q27[q][1];
Cqz = 1.0*D3Q27[q][2];
// get the associated neighbor
in = i + D3Q27[q][0];
jn = j + D3Q27[q][1];
kn = k + D3Q27[q][2];
// make sure the neighbor is in the domain (periodic BC)
/* if (in < 0 ) in +=Nx;
* don't need this in parallel since MPI handles the halos
if (jn < 0 ) jn +=Ny;
if (kn < 0 ) kn +=Nz;
if (!(in < Nx) ) in -=Nx;
if (!(jn < Ny) ) jn -=Ny;
if (!(kn < Nz) ) kn -=Nz;
// symmetric boundary
if (in < 0 ) in = i;
if (jn < 0 ) jn = j;
if (kn < 0 ) kn = k;
if (!(in < Nx) ) in = i;
if (!(jn < Ny) ) jn = k;
if (!(kn < Nz) ) kn = k;
*/
// Compute the gradient using upwind finite differences
Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx;
Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy;
Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz;
// Only include upwind derivatives
if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){
Dx += Dqx;
Dy += Dqy;
Dz += Dqz;
W += weights[q];
}
}
// Normalize by the weight to get the approximation to the gradient
if (fabs(W) > 0.0){
Dx /= W;
Dy /= W;
Dz /= W;
}
norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz);
}
else{
norm = 0.0;
}
Distance(i,j,k) += dt*sign*(1.0 - norm);
TotalVariation += dt*sign*(1.0 - norm);
// Disallow any change in phase
// if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k);
}
}
}
TotalVariation /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2);
count++;
}
return TotalVariation;
}
inline void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad)
{
// Read in the full sphere pack
//...... READ IN THE SPHERES...................................
cout << "Reading the packing file..." << endl;
FILE *fid = fopen("pack.out","rb");
INSIST(fid!=NULL,"Error opening pack.out");
//.........Trash the header lines..........
char line[100];
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
//........read the spheres..................
// We will read until a blank like or end-of-file is reached
int count = 0;
while ( !feof(fid) && fgets(line,100,fid)!=NULL ) {
char* line2 = line;
List_cx[count] = strtod(line2,&line2);
List_cy[count] = strtod(line2,&line2);
List_cz[count] = strtod(line2,&line2);
List_rad[count] = strtod(line2,&line2);
count++;
}
cout << "Number of spheres extracted is: " << count << endl;
INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" );
// .............................................................
}
inline void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
{
// Use sphere lists to determine which nodes are in porespace
// Write out binary file for nodes
char value;
int N = Nx*Ny*Nz; // Domain size, including the halo
double hx,hy,hz;
double x,y,z;
double cx,cy,cz,r;
int imin,imax,jmin,jmax,kmin,kmax;
int p,i,j,k,n;
//............................................
double min_x,min_y,min_z;
// double max_x,max_y,max_z;
//............................................
// Lattice spacing for the entire domain
// It should generally be true that hx=hy=hz
// Otherwise, you will end up with ellipsoids
hx = Lx/(Nx*nprocx-1);
hy = Ly/(Ny*nprocy-1);
hz = Lz/(Nz*nprocz-1);
//............................................
// Get maximum and minimum for this domain
// Halo is included !
min_x = double(iproc*Nx-1)*hx;
min_y = double(jproc*Ny-1)*hy;
min_z = double(kproc*Nz-1)*hz;
// max_x = ((iproc+1)*Nx+1)*hx;
// max_y = ((jproc+1)*Ny+1)*hy;
// max_z = ((kproc+1)*Nz+1)*hz;
//............................................
//............................................
// Pre-initialize local ID
for (n=0;n<N;n++){
ID[n]=1;
}
//............................................
//............................................
// .........Loop over the spheres.............
for (p=0;p<nspheres;p++){
// Get the sphere from the list, map to local min
cx = List_cx[p] - min_x;
cy = List_cy[p] - min_y;
cz = List_cz[p] - min_z;
r = List_rad[p];
// Check if
// Range for this sphere in global indexing
imin = int ((cx-r)/hx)-1;
imax = int ((cx+r)/hx)+1;
jmin = int ((cy-r)/hy)-1;
jmax = int ((cy+r)/hy)+1;
kmin = int ((cz-r)/hz)-1;
kmax = int ((cz+r)/hz)+1;
// Obviously we have to do something at the edges
if (imin<0) imin = 0;
if (imin>Nx) imin = Nx;
if (imax<0) imax = 0;
if (imax>Nx) imax = Nx;
if (jmin<0) jmin = 0;
if (jmin>Ny) jmin = Ny;
if (jmax<0) jmax = 0;
if (jmax>Ny) jmax = Ny;
if (kmin<0) kmin = 0;
if (kmin>Nz) kmin = Nz;
if (kmax<0) kmax = 0;
if (kmax>Nz) kmax = Nz;
// Loop over the domain for this sphere (may be null)
for (i=imin;i<imax;i++){
for (j=jmin;j<jmax;j++){
for (k=kmin;k<kmax;k++){
// Initialize ID value to 'fluid (=1)'
x = i*hx;
y = j*hy;
z = k*hz;
value = 1;
// if inside sphere, set to zero
if ( (cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z) < r*r){
value=0;
}
// get the position in the list
n = k*Nx*Ny+j*Nx+i;
if ( ID[n] != 0 ){
ID[n] = value;
}
}
}
}
}
}
inline void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
{
// Use sphere lists to determine which nodes are in porespace
// Write out binary file for nodes
int N = Nx*Ny*Nz; // Domain size, including the halo
double hx,hy,hz;
double x,y,z;
double cx,cy,cz,r;
int imin,imax,jmin,jmax,kmin,kmax;
int p,i,j,k,n;
//............................................
double min_x,min_y,min_z;
double distance;
//............................................
// Lattice spacing for the entire domain
// It should generally be true that hx=hy=hz
// Otherwise, you will end up with ellipsoids
hx = Lx/((Nx-2)*nprocx-1);
hy = Ly/((Ny-2)*nprocy-1);
hz = Lz/((Nz-2)*nprocz-1);
//............................................
// Get maximum and minimum for this domain
// Halo is included !
min_x = double(iproc*(Nx-2)-1)*hx;
min_y = double(jproc*(Ny-2)-1)*hy;
min_z = double(kproc*(Nz-2)-1)*hz;
//............................................
//............................................
// Pre-initialize Distance
for (n=0;n<N;n++){
Distance[n]=100.0;
}
//............................................
//............................................
// .........Loop over the spheres.............
for (p=0;p<nspheres;p++){
// Get the sphere from the list, map to local min
cx = List_cx[p] - min_x;
cy = List_cy[p] - min_y;
cz = List_cz[p] - min_z;
r = List_rad[p];
// Check if
// Range for this sphere in global indexing
imin = int ((cx-2*r)/hx);
imax = int ((cx+2*r)/hx)+2;
jmin = int ((cy-2*r)/hy);
jmax = int ((cy+2*r)/hy)+2;
kmin = int ((cz-2*r)/hz);
kmax = int ((cz+2*r)/hz)+2;
// Obviously we have to do something at the edges
if (imin<0) imin = 0;
if (imin>Nx) imin = Nx;
if (imax<0) imax = 0;
if (imax>Nx) imax = Nx;
if (jmin<0) jmin = 0;
if (jmin>Ny) jmin = Ny;
if (jmax<0) jmax = 0;
if (jmax>Ny) jmax = Ny;
if (kmin<0) kmin = 0;
if (kmin>Nz) kmin = Nz;
if (kmax<0) kmax = 0;
if (kmax>Nz) kmax = Nz;
// Loop over the domain for this sphere (may be null)
for (i=imin;i<imax;i++){
for (j=jmin;j<jmax;j++){
for (k=kmin;k<kmax;k++){
// x,y,z is distance in physical units
x = i*hx;
y = j*hy;
z = k*hz;
// if inside sphere, set to zero
// get the position in the list
n = k*Nx*Ny+j*Nx+i;
// Compute the distance
distance = sqrt((cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z)) - r;
// Assign the minimum distance
if (distance < Distance[n]) Distance[n] = distance;
}
}
}
}
// Map the distance to lattice units
for (n=0; n<N; n++) Distance[n] = Distance[n]/hx;
}
inline void WriteLocalSolidID(char *FILENAME, char *ID, int N)
{
char value;
ofstream File(FILENAME,ios::binary);
for (int n=0; n<N; n++){
value = ID[n];
File.write((char*) &value, sizeof(value));
}
File.close();
}
inline void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N)
{
double value;
ofstream File(FILENAME,ios::binary);
for (int n=0; n<N; n++){
value = Distance[n];
File.write((char*) &value, sizeof(value));
}
File.close();
}
inline void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np)
{
int q,n;
double value;
ofstream File(FILENAME,ios::binary);
for (n=0; n<Np; n++){
// Write the two density values
value = cDen[n];
File.write((char*) &value, sizeof(value));
value = cDen[Np+n];
File.write((char*) &value, sizeof(value));
// Write the even distributions
for (q=0; q<19; q++){
value = cfq[q*Np+n];
File.write((char*) &value, sizeof(value));
}
}
File.close();
}
inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np)
{
int q=0, n=0;
double value=0;
ifstream File(FILENAME,ios::binary);
for (n=0; n<Np; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
cDen[n] = value;
File.read((char*) &value, sizeof(value));
cDen[Np+n] = value;
// Read the even distributions
for (q=0; q<19; q++){
File.read((char*) &value, sizeof(value));
cfq[q*Np+n] = value;
}
}
File.close();
}
inline void ReadBinaryFile(char *FILENAME, double *Data, int N)
{
int n;
double value;
ifstream File(FILENAME,ios::binary);
if (File.good()){
for (n=0; n<N; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
Data[n] = value;
}
}
else {
for (n=0; n<N; n++) Data[n] = 1.2e-34;
}
File.close();
}
void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np);
void ReadBinaryFile(char *FILENAME, double *Data, int N);
#endif

18
common/MPI_Helpers.h
View File

@ -170,6 +170,12 @@ void unpack( std::set<TYPE>& data, const char *buffer );
// Helper functions
inline double sumReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,comm);
return y;
}
inline float sumReduce( MPI_Comm comm, float x )
{
float y = 0;
@ -199,12 +205,24 @@ inline std::vector<int> sumReduce( MPI_Comm comm, const std::vector<int>& x )
MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm);
return y;
}
inline double maxReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_MAX,comm);
return y;
}
inline float maxReduce( MPI_Comm comm, float x )
{
float y = 0;
MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm);
return y;
}
inline int maxReduce( MPI_Comm comm, int x )
{
int y = 0;
MPI_Allreduce(&x,&y,1,MPI_INT,MPI_MAX,comm);
return y;
}
#endif

2921
common/ScaLBL.cpp
View File

File diff suppressed because it is too large Load Diff

162
common/ScaLBL.h
View File

@ -120,149 +120,26 @@ extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, dou
extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area,
int count, int N);
//extern "C" void ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice);
// LIST OF DEPRECATED FUNCTIONS (probably delete)
//extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, double flux,
// int Nx, int Ny, int Nz);
//extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, double flux,
// int Nx, int Ny, int Nz, int outlet);
//extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,
// int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz,
// int Nx, int Ny, int Nz, int outlet);
//extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd,
// double *B_even, double *B_odd, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd,
// double *B_even, double *B_odd, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_AAeven_Compact(char *ID, double *d_dist, int Np);
//extern "C" void ScaLBL_D3Q19_AAodd_Compact(char *ID,int *d_neighborList, double *d_dist, int Np);
//extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N);
//extern "C" void ScaLBL_D3Q7_Init(double *Dist, double *Den, int Np);
//extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den,
// int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_AA_Init(double *f_even, double *f_odd, int Np);
//extern "C" void ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np);
//extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB,
// double Fx, double Fy, double Fz,int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_ColorDistance_Init(char *ID, double *Den, double *Phi, double *Distance,
// double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz);
//extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad,
// double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB,
// double alpha, double beta, double Fx, double Fy, double Fz);
//extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd,
// double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC);
//extern "C" void ScaLBL_D3Q19_AAeven_Color(double *dist, double *Aq, double *Bq, double *Den, double *Vel,
// double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
// double Fx, double Fy, double Fz, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q19_AAodd_Color(int *d_neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Vel,
// double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
// double Fx, double Fy, double Fz, int start, int finish, int Np);
/*extern "C" void ScaLBL_D3Q19_AAeven_ColorMomentum(double *dist, double *Den, double *Vel,
double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_ColorMomentum(int *d_neighborList, double *dist, double *Den, double *Vel,
double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_ColorMass(double *Aq, double *Bq, double *Den, double *Vel,
double *ColorGrad, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_ColorMass(int *d_neighborList, double *Aq, double *Bq, double *Den, double *Vel,
double *ColorGrad, double beta, int start, int finish, int Np);
*/
/*
class ScaLBL_Color{
public:
ScaLBL_Color(Domain &Dm);
~ScaLBL_Color();
int *NeighborList;
int *dvcMap;
// double *f_even,*f_odd;
double *fq, *Aq, *Bq;
double *Den, *Phi;
double *ColorGrad;
double *Vel;
double *Pressure;
ScaLBL_Communicator ScaLBL_Comm;
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular;
void Initialize(Domain &Dm);
void Run(int &timestep);
private:
};
void ScaLBL_Color::ScaLBL_Color(Domain &Dm){
}
void ScaLBL_Color::Initialize(Domain &Dm){
}
void ScaLBL_Color::Run(int &timestep){
}
*/
class ScaLBL_Communicator{
public:
//......................................................................................
ScaLBL_Communicator(Domain &Dm);
ScaLBL_Communicator(std::shared_ptr <Domain> Dm);
//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
~ScaLBL_Communicator();
//......................................................................................
unsigned long int CommunicationCount,SendCount,RecvCount;
int Nx,Ny,Nz,N;
int BoundaryCondition;
int next;
int first_interior,last_interior;
int BoundaryCondition;
//......................................................................................
// Set up for D319 distributions
// - determines how much memory is allocated
@ -277,19 +154,23 @@ public:
double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
//......................................................................................
int LastExterior();
int FirstInterior();
int LastInterior();
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, char *id, int Np);
void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np);
void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np);
void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
void SendD3Q19(double *f_even, double *f_odd);
void RecvD3Q19(double *f_even, double *f_odd);
void SendD3Q19AA(double *f_even, double *f_odd);
void RecvD3Q19AA(double *f_even, double *f_odd);
// void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np);
// void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
// void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np);
// void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
// void SendD3Q19(double *f_even, double *f_odd);
// void RecvD3Q19(double *f_even, double *f_odd);
// void SendD3Q19AA(double *f_even, double *f_odd);
// void RecvD3Q19AA(double *f_even, double *f_odd);
void SendD3Q19AA(double *dist);
void RecvD3Q19AA(double *dist);
void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
// void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
// void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void BiSendD3Q7AA(double *Aq, double *Bq);
void BiRecvD3Q7AA(double *Aq, double *Bq);
void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
@ -297,7 +178,7 @@ public:
void SendHalo(double *data);
void RecvHalo(double *data);
void RecvGrad(double *Phi, double *Gradient);
void RegularLayout(IntArray map, double *data, DoubleArray &regdata);
void RegularLayout(IntArray map, const double *data, DoubleArray &regdata);
// Routines to set boundary conditions
void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
@ -306,8 +187,8 @@ public:
void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
void TestSendD3Q19(double *f_even, double *f_odd);
void TestRecvD3Q19(double *f_even, double *f_odd);
// void TestSendD3Q19(double *f_even, double *f_odd);
// void TestRecvD3Q19(double *f_even, double *f_odd);
// Debugging and unit testing functions
void PrintD3Q19();
@ -319,6 +200,7 @@ private:
bool Lock; // use Lock to make sure only one call at a time to protect data in transit
// only one set of Send requests can be active at any time (per instance)
int i,j,k,n;
int iproc,jproc,kproc;
int nprocx,nprocy,nprocz;
int sendtag,recvtag;
@ -339,12 +221,12 @@ private:
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//......................................................................................
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;

259
common/SpherePack.cpp Normal file
View File

@ -0,0 +1,259 @@
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <math.h>
#include <time.h>
#include <exception>
#include <stdexcept>
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/SpherePack.h"
// Inline function to read line without a return argument
static inline void fgetl( char * str, int num, FILE * stream )
{
char* ptr = fgets( str, num, stream );
if ( 0 ) {char *temp = (char *)&ptr; temp++;}
}
void WriteLocalSolidID(char *FILENAME, char *ID, int N)
{
char value;
ofstream File(FILENAME,ios::binary);
for (int n=0; n<N; n++){
value = ID[n];
File.write((char*) &value, sizeof(value));
}
File.close();
}
void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N)
{
double value;
ofstream File(FILENAME,ios::binary);
for (int n=0; n<N; n++){
value = Distance[n];
File.write((char*) &value, sizeof(value));
}
File.close();
}
void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad)
{
// Read in the full sphere pack
//...... READ IN THE SPHERES...................................
cout << "Reading the packing file..." << endl;
FILE *fid = fopen("pack.out","rb");
INSIST(fid!=NULL,"Error opening pack.out");
//.........Trash the header lines..........
char line[100];
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
fgetl(line, 100, fid);
//........read the spheres..................
// We will read until a blank like or end-of-file is reached
int count = 0;
while ( !feof(fid) && fgets(line,100,fid)!=NULL ) {
char* line2 = line;
List_cx[count] = strtod(line2,&line2);
List_cy[count] = strtod(line2,&line2);
List_cz[count] = strtod(line2,&line2);
List_rad[count] = strtod(line2,&line2);
count++;
}
cout << "Number of spheres extracted is: " << count << endl;
INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" );
// .............................................................
}
void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
{
// Use sphere lists to determine which nodes are in porespace
// Write out binary file for nodes
char value;
int N = Nx*Ny*Nz; // Domain size, including the halo
double hx,hy,hz;
double x,y,z;
double cx,cy,cz,r;
int imin,imax,jmin,jmax,kmin,kmax;
int p,i,j,k,n;
//............................................
double min_x,min_y,min_z;
// double max_x,max_y,max_z;
//............................................
// Lattice spacing for the entire domain
// It should generally be true that hx=hy=hz
// Otherwise, you will end up with ellipsoids
hx = Lx/(Nx*nprocx-1);
hy = Ly/(Ny*nprocy-1);
hz = Lz/(Nz*nprocz-1);
//............................................
// Get maximum and minimum for this domain
// Halo is included !
min_x = double(iproc*Nx-1)*hx;
min_y = double(jproc*Ny-1)*hy;
min_z = double(kproc*Nz-1)*hz;
// max_x = ((iproc+1)*Nx+1)*hx;
// max_y = ((jproc+1)*Ny+1)*hy;
// max_z = ((kproc+1)*Nz+1)*hz;
//............................................
//............................................
// Pre-initialize local ID
for (n=0;n<N;n++){
ID[n]=1;
}
//............................................
//............................................
// .........Loop over the spheres.............
for (p=0;p<nspheres;p++){
// Get the sphere from the list, map to local min
cx = List_cx[p] - min_x;
cy = List_cy[p] - min_y;
cz = List_cz[p] - min_z;
r = List_rad[p];
// Check if
// Range for this sphere in global indexing
imin = int ((cx-r)/hx)-1;
imax = int ((cx+r)/hx)+1;
jmin = int ((cy-r)/hy)-1;
jmax = int ((cy+r)/hy)+1;
kmin = int ((cz-r)/hz)-1;
kmax = int ((cz+r)/hz)+1;
// Obviously we have to do something at the edges
if (imin<0) imin = 0;
if (imin>Nx) imin = Nx;
if (imax<0) imax = 0;
if (imax>Nx) imax = Nx;
if (jmin<0) jmin = 0;
if (jmin>Ny) jmin = Ny;
if (jmax<0) jmax = 0;
if (jmax>Ny) jmax = Ny;
if (kmin<0) kmin = 0;
if (kmin>Nz) kmin = Nz;
if (kmax<0) kmax = 0;
if (kmax>Nz) kmax = Nz;
// Loop over the domain for this sphere (may be null)
for (i=imin;i<imax;i++){
for (j=jmin;j<jmax;j++){
for (k=kmin;k<kmax;k++){
// Initialize ID value to 'fluid (=1)'
x = i*hx;
y = j*hy;
z = k*hz;
value = 1;
// if inside sphere, set to zero
if ( (cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z) < r*r){
value=0;
}
// get the position in the list
n = k*Nx*Ny+j*Nx+i;
if ( ID[n] != 0 ){
ID[n] = value;
}
}
}
}
}
}
void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
{
// Use sphere lists to determine which nodes are in porespace
// Write out binary file for nodes
int N = Nx*Ny*Nz; // Domain size, including the halo
double hx,hy,hz;
double x,y,z;
double cx,cy,cz,r;
int imin,imax,jmin,jmax,kmin,kmax;
int p,i,j,k,n;
//............................................
double min_x,min_y,min_z;
double distance;
//............................................
// Lattice spacing for the entire domain
// It should generally be true that hx=hy=hz
// Otherwise, you will end up with ellipsoids
hx = Lx/((Nx-2)*nprocx-1);
hy = Ly/((Ny-2)*nprocy-1);
hz = Lz/((Nz-2)*nprocz-1);
//............................................
// Get maximum and minimum for this domain
// Halo is included !
min_x = double(iproc*(Nx-2)-1)*hx;
min_y = double(jproc*(Ny-2)-1)*hy;
min_z = double(kproc*(Nz-2)-1)*hz;
//............................................
//............................................
// Pre-initialize Distance
for (n=0;n<N;n++){
Distance[n]=100.0;
}
//............................................
//............................................
// .........Loop over the spheres.............
for (p=0;p<nspheres;p++){
// Get the sphere from the list, map to local min
cx = List_cx[p] - min_x;
cy = List_cy[p] - min_y;
cz = List_cz[p] - min_z;
r = List_rad[p];
// Check if
// Range for this sphere in global indexing
imin = int ((cx-2*r)/hx);
imax = int ((cx+2*r)/hx)+2;
jmin = int ((cy-2*r)/hy);
jmax = int ((cy+2*r)/hy)+2;
kmin = int ((cz-2*r)/hz);
kmax = int ((cz+2*r)/hz)+2;
// Obviously we have to do something at the edges
if (imin<0) imin = 0;
if (imin>Nx) imin = Nx;
if (imax<0) imax = 0;
if (imax>Nx) imax = Nx;
if (jmin<0) jmin = 0;
if (jmin>Ny) jmin = Ny;
if (jmax<0) jmax = 0;
if (jmax>Ny) jmax = Ny;
if (kmin<0) kmin = 0;
if (kmin>Nz) kmin = Nz;
if (kmax<0) kmax = 0;
if (kmax>Nz) kmax = Nz;
// Loop over the domain for this sphere (may be null)
for (i=imin;i<imax;i++){
for (j=jmin;j<jmax;j++){
for (k=kmin;k<kmax;k++){
// x,y,z is distance in physical units
x = i*hx;
y = j*hy;
z = k*hz;
// if inside sphere, set to zero
// get the position in the list
n = k*Nx*Ny+j*Nx+i;
// Compute the distance
distance = sqrt((cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z)) - r;
// Assign the minimum distance
if (distance < Distance[n]) Distance[n] = distance;
}
}
}
}
// Map the distance to lattice units
for (n=0; n<N; n++) Distance[n] = Distance[n]/hx;
}

37
common/SpherePack.h Normal file
View File

@ -0,0 +1,37 @@
#ifndef SpherePack_INC
#define SpherePack_INC
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <math.h>
#include <time.h>
#include <exception>
#include <stdexcept>
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/Database.h"
/*
Simple tools to work with sphere packs
*/
void WriteLocalSolidID(char *FILENAME, char *ID, int N);
void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N);
void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad);
void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz);
void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
double Lx, double Ly, double Lz, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz);
#endif

226
common/Units.cpp Normal file
View File

@ -0,0 +1,226 @@
#include "common/Units.h"
#include "common/Utilities.h"
#include <algorithm>
#include <cmath>
#include <string>
constexpr double Units::d_pow10[22];
constexpr char Units::d_prefixSymbol[];
/********************************************************************
* Constructors *
********************************************************************/
Units::Units() : d_prefix( UnitPrefix::unknown ), d_unit( UnitValue::unknown ) {}
Units::Units( UnitPrefix p, UnitValue u ) : d_prefix( p ), d_unit( u ) {}
Units::Units( const std::string& unit )
: d_prefix( UnitPrefix::unknown ), d_unit( UnitValue::unknown )
{
// Parse the string to get it into a more friendly format
auto tmp = unit;
tmp.erase( std::remove( tmp.begin(), tmp.end(), ' ' ), tmp.end() );
// Check if the character '-' is present indicating a seperation between the prefix and unit
size_t index = tmp.find( '-' );
if ( index != std::string::npos ) {
d_prefix = getUnitPrefix( tmp.substr( 0, index ) );
d_unit = getUnitValue( tmp.substr( index + 1 ) );
} else {
if ( tmp.size() <= 1 ) {
d_prefix = UnitPrefix::none;
d_unit = getUnitValue( tmp );
} else if ( tmp.substr( 0, 2 ) == "da" ) {
d_prefix = UnitPrefix::deca;
d_unit = getUnitValue( tmp.substr( 2 ) );
} else {
d_prefix = getUnitPrefix( tmp.substr( 0, 1 ) );
d_unit = getUnitValue( tmp.substr( 1 ) );
if ( d_prefix == UnitPrefix::unknown || d_unit == UnitValue::unknown ) {
d_prefix = UnitPrefix::none;
d_unit = getUnitValue( tmp );
}
}
}
}
/********************************************************************
* Get prefix *
********************************************************************/
Units::UnitPrefix Units::getUnitPrefix( const std::string& str ) noexcept
{
Units::UnitPrefix value = UnitPrefix::unknown;
if ( str.empty() ) {
value = UnitPrefix::none;
} else if ( str == "yotta" || str == "Y" ) {
value = UnitPrefix::yotta;
} else if ( str == "zetta" || str == "Z" ) {
value = UnitPrefix::zetta;
} else if ( str == "exa" || str == "E" ) {
value = UnitPrefix::exa;
} else if ( str == "peta" || str == "P" ) {
value = UnitPrefix::peta;
} else if ( str == "tera" || str == "T" ) {
value = UnitPrefix::tera;
} else if ( str == "giga" || str == "G" ) {
value = UnitPrefix::giga;
} else if ( str == "mega" || str == "M" ) {
value = UnitPrefix::mega;
} else if ( str == "kilo" || str == "k" ) {
value = UnitPrefix::kilo;
} else if ( str == "hecto" || str == "h" ) {
value = UnitPrefix::hecto;
} else if ( str == "deca" || str == "da" ) {
value = UnitPrefix::deca;
} else if ( str == "deci" || str == "d" ) {
value = UnitPrefix::deci;
} else if ( str == "centi" || str == "c" ) {
value = UnitPrefix::centi;
} else if ( str == "milli" || str == "m" ) {
value = UnitPrefix::milli;
} else if ( str == "micro" || str == "u" ) {
value = UnitPrefix::micro;
} else if ( str == "nano" || str == "n" ) {
value = UnitPrefix::nano;
} else if ( str == "pico" || str == "p" ) {
value = UnitPrefix::pico;
} else if ( str == "femto" || str == "f" ) {
value = UnitPrefix::femto;
} else if ( str == "atto" || str == "a" ) {
value = UnitPrefix::atto;
} else if ( str == "zepto" || str == "z" ) {
value = UnitPrefix::zepto;
} else if ( str == "yocto" || str == "y" ) {
value = UnitPrefix::yocto;
}
return value;
}
/********************************************************************
* Get unit value *
********************************************************************/
Units::UnitValue Units::getUnitValue( const std::string& str ) noexcept
{
Units::UnitValue value = UnitValue::unknown;
if ( str == "meter" || str == "m" ) {
value = UnitValue::meter;
} else if ( str == "gram" || str == "g" ) {
value = UnitValue::gram;
} else if ( str == "second" || str == "s" ) {
value = UnitValue::second;
} else if ( str == "ampere" || str == "A" ) {
value = UnitValue::ampere;
} else if ( str == "kelvin" || str == "K" ) {
value = UnitValue::kelvin;
} else if ( str == "joule" || str == "J" ) {
value = UnitValue::joule;
} else if ( str == "ergs" || str == "erg" ) {
value = UnitValue::erg;
} else if ( str == "degree" || str == "degrees" ) {
value = UnitValue::degree;
} else if ( str == "radian" || str == "radians" ) {
value = UnitValue::radian;
}
return value;
}
/********************************************************************
* Get unit type *
********************************************************************/
Units::UnitType Units::getUnitType( UnitValue u ) noexcept
{
switch ( u ) {
case UnitValue::meter:
return UnitType::length;
case UnitValue::gram:
return UnitType::mass;
case UnitValue::second:
return UnitType::time;
case UnitValue::ampere:
return UnitType::current;
case UnitValue::kelvin:
return UnitType::temperature;
case UnitValue::joule:
case UnitValue::erg:
return UnitType::energy;
case UnitValue::degree:
case UnitValue::radian:
return UnitType::angle;
default:
return UnitType::unknown;
}
}
/********************************************************************
* Convert to another unit system *
********************************************************************/
double Units::convert( const Units& rhs ) const noexcept
{
if ( this->operator==( rhs ) )
return 1;
// Convert the prefix
double cp = convert( d_prefix ) / convert( rhs.d_prefix );
if ( d_unit == rhs.d_unit )
return cp; // Only need to convert prefix
// Convert the unit
if ( getUnitType( d_unit ) != getUnitType( rhs.d_unit ) )
return 0; // Invalid conversion
double cu = 0;
if ( d_unit == UnitValue::joule && rhs.d_unit == UnitValue::erg )
cu = 1e7;
else if ( d_unit == UnitValue::erg && rhs.d_unit == UnitValue::joule )
cu = 1e-7;
else if ( d_unit == UnitValue::degree && rhs.d_unit == UnitValue::radian )
cu = 0.017453292519943;
else if ( d_unit == UnitValue::radian && rhs.d_unit == UnitValue::degree )
cu = 57.295779513082323;
// Return the total conversion
return cp * cu;
}
/********************************************************************
* Write a string for the units *
********************************************************************/
std::string Units::str() const
{
ASSERT( !isNull() );
return std::string( str( d_prefix ).data() ) + str( d_unit );
}
std::array<char, 3> Units::str( UnitPrefix p ) noexcept
{
std::array<char, 3> str;
str[0] = d_prefixSymbol[static_cast<int8_t>( p )];
str[1] = 0;
str[2] = 0;
if ( p == UnitPrefix::deca )
str[1] = 'a';
return str;
}
std::string Units::str( UnitValue u )
{
if ( u == UnitValue::meter ) {
return "m";
} else if ( u == UnitValue::gram ) {
return "g";
} else if ( u == UnitValue::second ) {
return "s";
} else if ( u == UnitValue::ampere ) {
return "A";
} else if ( u == UnitValue::kelvin ) {
return "K";
} else if ( u == UnitValue::joule ) {
return "J";
} else if ( u == UnitValue::erg ) {
return "erg";
} else if ( u == UnitValue::degree ) {
return "degree";
} else if ( u == UnitValue::radian ) {
return "radian";
}
return "unknown";
}

140
common/Units.h Normal file
View File

@ -0,0 +1,140 @@
#ifndef included_Units
#define included_Units
#include <array>
#include <iostream>
#include <map>
#include <memory>
#include <string>
#include <vector>
//! Unit system class
class Units final
{
public:
//! Enum to hold prefix
enum class UnitPrefix : int8_t {
yocto = 0,
zepto = 1,
atto = 2,
femto = 3,
pico = 4,
nano = 5,
micro = 6,
milli = 7,
centi = 8,
deci = 9,
none = 10,
deca = 11,
hecto = 12,
kilo = 13,
mega = 14,
giga = 15,
tera = 16,
peta = 17,
exa = 18,
zetta = 19,
yotta = 20,
unknown = 21
};
//! Enum to unit type
enum class UnitType : uint8_t {
length,
mass,
time,
current,
temperature,
energy,
angle,
unknown
};
//! Enum to hold unit
enum class UnitValue : uint8_t {
meter,
gram,
second,
ampere,
kelvin,
joule,
erg,
degree,
radian,
unknown
};
public:
//! Constructor
Units();
//! Constructor
explicit Units( const std::string& unit );
//! Constructor
explicit Units( UnitPrefix, UnitValue );
//! Get the prefix
inline UnitPrefix getPrefix() const noexcept { return d_prefix; }
//! Get the unit
inline UnitValue getUnit() const noexcept { return d_unit; }
//! Get the unit
inline UnitType getUnitType() const noexcept { return getUnitType( d_unit ); }
//! Get the unit
static UnitType getUnitType( UnitValue ) noexcept;
//! Get the prefix from a string
static UnitPrefix getUnitPrefix( const std::string& ) noexcept;
//! Get the unit value from a string
static UnitValue getUnitValue( const std::string& ) noexcept;
//! Convert to the given unit system
double convert( const Units& ) const noexcept;
//! Convert a prefix to a scalar
static inline double convert( UnitPrefix x ) noexcept
{
return d_pow10[static_cast<int8_t>( x )];
}
//! Get a string representation of the units
std::string str() const;
//! Get a string representation for the prefix
static std::array<char, 3> str( UnitPrefix ) noexcept;
//! Get a string representation for the unit value
static std::string str( UnitValue );
//! Operator ==
inline bool operator==( const Units& rhs ) const noexcept
{
return d_prefix == rhs.d_prefix && d_unit == rhs.d_unit;
}
//! Operator !=
inline bool operator!=( const Units& rhs ) const noexcept
{
return d_prefix != rhs.d_prefix || d_unit != rhs.d_unit;
}
//! Check if unit is null
bool isNull() const { return d_prefix == UnitPrefix::unknown || d_unit == UnitValue::unknown; }
protected:
UnitPrefix d_prefix;
UnitValue d_unit;
private:
constexpr static double d_pow10[22] = { 1e-24, 1e-21, 1e-18, 1e-15, 1e-12, 1e-9, 1e-6, 1e-3,
1e-2, 0.1, 1, 10, 100, 1000, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24, 0 };
constexpr static char d_prefixSymbol[] = "yzafpnumcd\0dhkMGTPEZYu";
};
#endif

6
example/Bubble/Color.in
View File

@ -1,6 +0,0 @@
1.0
1.0e-2 0.95 0.8
0.7
0.0 0.0 0.0
0 0 1.0 1.0
200 1000 1e-5

3
example/Bubble/Domain.in
View File

@ -1,3 +0,0 @@
1 1 1
80 80 80
1.0 1.0 1.0

39
example/Bubble/input.db Normal file
View File

@ -0,0 +1,39 @@
Color {
tauA = 1.0; // relaxation time for fluid A
tauB = 1.0; // relaxation time for fluid B
rhoA = 1.0;
rhoB = 1.0;
alpha = 1e-2;
beta = 0.95;
F = 0, 0, 0
Restart = false
pBC = 0
din = 1.0
dout = 1.0
flux = 1.0e-3
timestepMax = 200
interval = 1000
tol = 1e-5;
das = 0.1
dbs = 0.9
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz)
n_spheres = 1 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 20000 // Frequency to write restart data
visualization_interval = 20000 // Frequency to write visualization data
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}

37
example/Bubble/input.in Normal file
View File

@ -0,0 +1,37 @@
Color {
tau = 1.0;
alpha = 1e-2;
beta = 0.95;
phi_s = 0.8;
wp_saturation = 0.7
F = 0, 0, 0
Restart = false
pBC = 0
din = 1.0
dout = 1.0
timestepMax = 200
interval = 1000
tol = 1e-5;
das = 0.1
dbs = 0.9
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz)
n_spheres = 1 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 20000 // Frequency to write restart data
vis_interval = 20000 // Frequency to write visualization data
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}

37
example/Piston/input.db Normal file
View File

@ -0,0 +1,37 @@
Color {
tau = 1.0;
alpha = 1e-2;
beta = 0.95;
phi_s = 0.8;
wp_saturation = 0.7
F = 0, 0, 0
Restart = false
pBC = 0
din = 1.0
dout = 1.0
timestepMax = 200
interval = 1000
tol = 1e-5;
das = 0.1
dbs = 0.9
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz)
n_spheres = 1 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 20000 // Frequency to write restart data
vis_interval = 20000 // Frequency to write visualization data
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}

20
example/Piston/poiseuille.db Normal file
View File

@ -0,0 +1,20 @@
MRT {
tau = 1.0
F = 0, 0, 1.0e-4
Restart = false
pBC = 0
din = 1.0
dout = 1.0
timestepMax = 2000
tol = 1e-5
flux = 0.0
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz)
n_spheres = 1 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}

39
example/Sph125/input.db Normal file
View File

@ -0,0 +1,39 @@
Color {
tauA = 1.0
tauB = 1.0
rhoA = 1.0
rhoB = 1.0
alpha = 1e-2
beta = 0.95
F = 0, 0, 0
Restart = false
pBC = 0
din = 1.0
dout = 1.0
timestepMax = 200
interval = 1000
tol = 1e-5
das = 0.1
dbs = 0.9
flux = 0.0
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 100, 100, 100 // Size of local domain (Nx,Ny,Nz)
nspheres = 125 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 20000 // Frequency to write restart data
visualization_interval = 20000 // Frequency to write visualization data
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}

37
example/Sph1896/input.db Normal file
View File

@ -0,0 +1,37 @@
Color {
tau = 1.0;
alpha = 1e-2;
beta = 0.95;
phi_s = 0.8;
wp_saturation = 0.7
F = 0, 0, 0
Restart = false
pBC = 0
din = 1.0
dout = 1.0
timestepMax = 200
interval = 1000
tol = 1e-5;
das = 0.1
dbs = 0.9
}
Domain {
nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz)
n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz)
n_spheres = 1 // Number of spheres
L = 1, 1, 1 // Length of domain (x,y,z)
BC = 0 // Boundary condition type
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 20000 // Frequency to write restart data
vis_interval = 20000 // Frequency to write visualization data
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}

17
gpu/Color.cu
View File

@ -136,8 +136,10 @@ __global__ void dvc_ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *D
Den[n] = vA;
Den[Np+n] = vB;
nm = Map[n];
Phi[nm] = (vA-vB)/(vA+vB);
//nm = Map[n];
//Phi[nm] = (vA-vB)/(vA+vB);
Phi[n] = (vA-vB)/(vA+vB);
}
}
__global__ void dvc_ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np)
@ -151,8 +153,9 @@ __global__ void dvc_ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double
Den[Np+n] = vB;
//double valB = Den[Np+n]; // mass that reaches inlet is conserved
nm = Map[n];
Phi[nm] = (vA-vB)/(vA+vB);
//nm = Map[n];
//Phi[nm] = (vA-vB)/(vA+vB);
Phi[n] = (vA-vB)/(vA+vB);
}
}
@ -167,8 +170,10 @@ __global__ void dvc_ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double
Den[n] = vA;
Den[Np+n] = vB;
nm = Map[n];
Phi[nm] = (vA-vB)/(vA+vB);
//nm = Map[n];
//Phi[nm] = (vA-vB)/(vA+vB);
Phi[n] = (vA-vB)/(vA+vB);
}
}
//*************************************************************************

552
models/ColorModel.cpp Normal file
View File

@ -0,0 +1,552 @@
/*
color lattice boltzmann model
*/
#include "models/ColorModel.h"
ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
{
}
ScaLBL_ColorModel::~ScaLBL_ColorModel(){
}
void ScaLBL_ColorModel::ReadParams(string filename){
// read the input database
db = std::make_shared<Database>( filename );
domain_db = db->getDatabase( "Domain" );
color_db = db->getDatabase( "Color" );
analysis_db = db->getDatabase( "Analysis" );
// Color Model parameters
timestepMax = color_db->getScalar<int>( "timestepMax" );
tauA = color_db->getScalar<double>( "tauA" );
tauB = color_db->getScalar<double>( "tauB" );
rhoA = color_db->getScalar<double>( "rhoA" );
rhoB = color_db->getScalar<double>( "rhoB" );
Fx = color_db->getVector<double>( "F" )[0];
Fy = color_db->getVector<double>( "F" )[1];
Fz = color_db->getVector<double>( "F" )[2];
alpha = color_db->getScalar<double>( "alpha" );
beta = color_db->getScalar<double>( "beta" );
Restart = color_db->getScalar<bool>( "Restart" );
din = color_db->getScalar<double>( "din" );
dout = color_db->getScalar<double>( "dout" );
flux = color_db->getScalar<double>( "flux" );
inletA=1.f;
inletB=0.f;
outletA=0.f;
outletB=1.f;
if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details)
// Read domain parameters
auto L = domain_db->getVector<double>( "L" );
auto size = domain_db->getVector<int>( "n" );
auto nproc = domain_db->getVector<int>( "nproc" );
BoundaryCondition = domain_db->getScalar<int>( "BC" );
Nx = size[0];
Ny = size[1];
Nz = size[2];
Lx = L[0];
Ly = L[1];
Lz = L[2];
nprocx = nproc[0];
nprocy = nproc[1];
nprocz = nproc[2];
}
void ScaLBL_ColorModel::SetDomain(){
Dm = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // full domain for analysis
Mask = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // mask domain removes immobile phases
Nx+=2; Ny+=2; Nz += 2;
N = Nx*Ny*Nz;
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
MPI_Barrier(comm);
Dm->CommInit();
MPI_Barrier(comm);
}
void ScaLBL_ColorModel::ReadInput(){
int rank=Dm->rank();
size_t readID;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
Mask->ReadIDs();
sprintf(LocalRankString,"%05d",Dm->rank());
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
// .......... READ THE INPUT FILE .......................................
//...........................................................................
if (rank == 0) cout << "Reading in signed distance function..." << endl;
//.......................................................................
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N);
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
// Read restart file
if (Restart == true){
if (Dm->rank()==0){
size_t readID;
printf("Reading restart file! \n");
ifstream restart("Restart.txt");
if (restart.is_open()){
restart >> timestep;
printf("Restarting from timestep =%i \n",timestep);
}
else{
printf("WARNING:No Restart.txt file, setting timestep=0 \n");
timestep=0;
}
}
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
FILE *RESTART = fopen(LocalRestartFile,"rb");
if (RESTART==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx");
readID=fread(id,1,N,RESTART);
if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading Restart (rank=%i) \n",rank);
fclose(RESTART);
MPI_Barrier(comm);
}
}
void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
{
int rank=Dm->rank();
int NLABELS=0;
char VALUE=0;
double AFFINITY=0.f;
vector <char> Label;
vector <double> Affinity;
// Read the labels
if (rank==0){
printf("Component labels:\n");
ifstream iFILE("ComponentLabels.csv");
if (iFILE.good()){
int value;
while (!iFILE.eof()){
iFILE>>value;
iFILE>>AFFINITY;
VALUE=char(value);
Label.push_back(value);
Affinity.push_back(AFFINITY);
NLABELS++;
printf("%i %f\n",VALUE,AFFINITY);
}
}
else{
printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n");
// Set default values
VALUE=0; AFFINITY=-1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
VALUE=1; AFFINITY=1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
VALUE=2; AFFINITY=-1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
}
}
MPI_Barrier(comm);
// Broadcast the list
MPI_Bcast(&NLABELS,1,MPI_INT,0,comm);
//printf("rank=%i, NLABELS=%i \n ",rank(),NLABELS);
// Copy into contiguous buffers
char *LabelList;
double * AffinityList;
LabelList=new char[NLABELS];
AffinityList=new double[NLABELS];
if (rank==0){
for (int idx=0; idx < NLABELS; idx++){
VALUE=Label[idx];
AFFINITY=Affinity[idx];
printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank,idx,VALUE,AFFINITY);
LabelList[idx]=VALUE;
AffinityList[idx]=AFFINITY;
}
}
MPI_Barrier(comm);
MPI_Bcast(LabelList,NLABELS,MPI_CHAR,0,comm);
MPI_Bcast(AffinityList,NLABELS,MPI_DOUBLE,0,comm);
// Assign the labels
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
VALUE=Mask->id[n];
// Assign the affinity from the paired list
for (int idx=0; idx < NLABELS; idx++){
//printf("rank=%i, idx=%i, value=%i, %i, \n",rank(),idx, VALUE,LabelList[idx]);
if (VALUE == LabelList[idx]){
AFFINITY=AffinityList[idx];
idx = NLABELS;
}
}
phase[n] = AFFINITY;
}
}
}
}
void ScaLBL_ColorModel::Create(){
/*
* This function creates the variables needed to run a LBM
*/
int rank=Dm->rank();
//.........................................................
// don't perform computations at the eight corners
//id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0;
//id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
//.........................................................
// Initialize communication structures in averaging domain
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
Mask->CommInit();
Np=Mask->PoreCount();
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device (will use optimized layout)
// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
ScaLBL_Comm = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
int Npad=(Np/16 + 2)*16;
if (rank==0) printf ("Set up memory efficient layout \n");
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
//...........................................................................
// LBM variables
if (rank==0) printf ("Allocating distributions \n");
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
int neighborSize=18*(Np*sizeof(int));
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &SolidPotential, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
if (rank==0) printf ("Setting up device map and neighbor list \n");
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
int *TmpMap;
TmpMap=new int[Np];
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
if (!(idx < 0))
TmpMap[idx] = k*Nx*Ny+j*Nx+i;
}
}
}
ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
ScaLBL_DeviceBarrier();
delete [] TmpMap;
}
/********************************************************
* AssignComponentLabels *
********************************************************/
void ScaLBL_ColorModel::AssignSolidPotential(){
if (rank==0) printf("Computing solid interaction potential \n");
double *PhaseLabel;
PhaseLabel=new double [Nx*Ny*Nz];
AssignComponentLabels(PhaseLabel);
double *Tmp;
Tmp=new double[3*Np];
//Averages->UpdateMeshValues(); // this computes the gradient of distance field (among other things)
// Create the distance stencil
// Compute solid forces based on mean field approximation
double *Dst;
Dst = new double [5*5*5];
for (int kk=0; kk<5; kk++){
for (int jj=0; jj<5; jj++){
for (int ii=0; ii<5; ii++){
int index = kk*25+jj*5+ii;
Dst[index] = sqrt(double(ii-2)*double(ii-2) + double(jj-2)*double(jj-2)+ double(kk-2)*double(kk-2));
}
}
}
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
if (!(idx < 0)){
double phi_x = 0.f;
double phi_y = 0.f;
double phi_z = 0.f;
for (int kk=0; kk<5; kk++){
for (int jj=0; jj<5; jj++){
for (int ii=0; ii<5; ii++){
int index = kk*25+jj*5+ii;
double distval= Dst[index];
int idi=i+ii-2;
int idj=j+jj-2;
int idk=k+kk-2;
if (idi < 0) idi=0;
if (idj < 0) idj=0;
if (idk < 0) idk=0;
if (!(idi < Nx)) idi=Nx-1;
if (!(idj < Ny)) idj=Ny-1;
if (!(idk < Nz)) idk=Nz-1;
int nn = idk*Nx*Ny + idj*Nx + idi;
if (!(Mask->id[nn] > 0)){
double vec_x = double(ii-2);
double vec_y = double(jj-2);
double vec_z = double(kk-2);
double ALPHA=PhaseLabel[nn];
double GAMMA=-2.f;
if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance
phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval;
phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval;
phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval;
}
}
}
}
Tmp[idx] = phi_x;
Tmp[idx+Np] = phi_y;
Tmp[idx+2*Np] = phi_z;
/* double d = Averages->SDs(n);
double dx = Averages->SDs_x(n);
double dy = Averages->SDs_y(n);
double dz = Averages->SDs_z(n);
double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d));
Tmp[idx] = value*dx;
Tmp[idx+Np] = value*dy;
Tmp[idx+2*Np] = value*dz;
*/
}
}
}
}
ScaLBL_CopyToDevice(SolidPotential, Tmp, 3*sizeof(double)*Np);
ScaLBL_DeviceBarrier();
delete [] Tmp;
delete [] Dst;
DoubleArray Psx(Nx,Ny,Nz);
DoubleArray Psy(Nx,Ny,Nz);
DoubleArray Psz(Nx,Ny,Nz);
DoubleArray Psnorm(Nx,Ny,Nz);
ScaLBL_Comm->RegularLayout(Map,&SolidPotential[0],Psx);
ScaLBL_Comm->RegularLayout(Map,&SolidPotential[Np],Psy);
ScaLBL_Comm->RegularLayout(Map,&SolidPotential[2*Np],Psz);
for (int n=0; n<N; n++) Psnorm(n) = Psx(n)*Psx(n)+Psy(n)*Psy(n)+Psz(n)*Psz(n);
FILE *PFILE;
sprintf(LocalRankFilename,"Potential.%05i.raw",rank);
PFILE = fopen(LocalRankFilename,"wb");
fwrite(Psnorm.data(),8,N,PFILE);
fclose(PFILE);
}
void ScaLBL_ColorModel::Initialize(){
/*
* This function initializes model
*/
int rank=Dm->rank();
double count_wet=0.f;
double *PhaseLabel;
PhaseLabel=new double [Nx*Ny*Nz];
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
int n = k*Nx*Ny+j*Nx+i;
if (!(idx < 0)){
if (Mask->id[n] == 1)
PhaseLabel[idx] = 1.0;
else {
PhaseLabel[idx] = -1.0;
count_wet+=1.f;
}
}
}
}
}
//printf("sw=%f \n",count_wet/double(Np));
// initialize phi based on PhaseLabel (include solid component labels)
ScaLBL_CopyToDevice(Phi, PhaseLabel, Np*sizeof(double));
//...........................................................................
if (rank==0) printf ("Initializing distributions \n");
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
}
void ScaLBL_ColorModel::Run(){
int nprocs=nprocx*nprocy*nprocz;
int rank=Dm->rank();
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps: %i \n", timestepMax);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
PROFILE_START("Loop");
runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map );
while (timestep < timestepMax ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
// *************ODD TIMESTEP*************
timestep++;
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set BCs
if (BoundaryCondition > 0){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
// *************EVEN TIMESTEP*************
timestep++;
// Compute the Phase indicator field
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set boundary conditions
if (BoundaryCondition > 0){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
//************************************************************************
MPI_Barrier(comm);
PROFILE_STOP("Update");
// Run the analysis
analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den );
}
analysis.finish();
PROFILE_STOP("Loop");
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n");
// ************************************************************************
}
void ScaLBL_ColorModel::WriteDebug(){
// Copy back final phase indicator field and convert to regular layout
DoubleArray PhaseField(Nx,Ny,Nz);
ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField);
FILE *OUTFILE;
sprintf(LocalRankFilename,"Phase.%05i.raw",rank);
OUTFILE = fopen(LocalRankFilename,"wb");
fwrite(PhaseField.data(),8,N,OUTFILE);
fclose(OUTFILE);
}

81
models/ColorModel.h Normal file
View File

@ -0,0 +1,81 @@
/*
Implementation of color lattice boltzmann model
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "analysis/runAnalysis.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
class ScaLBL_ColorModel{
public:
ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM);
~ScaLBL_ColorModel();
// functions in they should be run
void ReadParams(string filename);
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void AssignSolidPotential();
void Run();
void WriteDebug();
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
double tauA,tauB,rhoA,rhoB,alpha,beta;
double Fx,Fy,Fz,flux;
double din,dout,inletA,inletB,outletA,outletB;
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
std::shared_ptr<TwoPhase> Averages;
// input database
std::shared_ptr<Database> db;
std::shared_ptr<Database> domain_db;
std::shared_ptr<Database> color_db;
std::shared_ptr<Database> analysis_db;
IntArray Map;
char *id;
int *NeighborList;
int *dvcMap;
double *fq, *Aq, *Bq;
double *Den, *Phi;
double *SolidPotential;
double *Velocity;
double *Gradient;
double *Pressure;
private:
MPI_Comm comm;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);
void AssignComponentLabels(double *phase);
};

182
models/MRTModel.cpp Normal file
View File

@ -0,0 +1,182 @@
/*
* Multi-relaxation time LBM Model
*/
#include "models/MRTModel.h"
ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
{
}
ScaLBL_MRTModel::~ScaLBL_MRTModel(){
}
void ScaLBL_MRTModel::ReadParams(string filename){
// read the input database
db = std::make_shared<Database>( filename );
domain_db = db->getDatabase( "Domain" );
mrt_db = db->getDatabase( "MRT" );
// Color Model parameters
timestepMax = mrt_db->getScalar<int>( "timestepMax" );
tau = mrt_db->getScalar<double>( "tau" );
Fx = mrt_db->getVector<double>( "F" )[0];
Fy = mrt_db->getVector<double>( "F" )[1];
Fz = mrt_db->getVector<double>( "F" )[2];
Restart = mrt_db->getScalar<bool>( "Restart" );
din = mrt_db->getScalar<double>( "din" );
dout = mrt_db->getScalar<double>( "dout" );
flux = mrt_db->getScalar<double>( "flux" );
// Read domain parameters
auto L = domain_db->getVector<double>( "L" );
auto size = domain_db->getVector<int>( "n" );
auto nproc = domain_db->getVector<int>( "nproc" );
BoundaryCondition = domain_db->getScalar<int>( "BC" );
Nx = size[0];
Ny = size[1];
Nz = size[2];
Lx = L[0];
Ly = L[1];
Lz = L[2];
nprocx = nproc[0];
nprocy = nproc[1];
nprocz = nproc[2];
mu=(tau-0.5)/3.0;
}
void ScaLBL_MRTModel::SetDomain(){
Dm = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // full domain for analysis
Mask = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // mask domain removes immobile phases
Nx+=2; Ny+=2; Nz += 2;
N = Nx*Ny*Nz;
Distance.resize(Nx,Ny,Nz);
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
MPI_Barrier(comm);
Dm->CommInit();
MPI_Barrier(comm);
}
void ScaLBL_MRTModel::ReadInput(){
int rank=Dm->rank();
size_t readID;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
Mask->ReadIDs();
sprintf(LocalRankString,"%05d",Dm->rank());
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
// .......... READ THE INPUT FILE .......................................
//...........................................................................
if (rank == 0) cout << "Reading in signed distance function..." << endl;
//.......................................................................
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Distance.data(), N);
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
}
void ScaLBL_MRTModel::Create(){
/*
* This function creates the variables needed to run a LBM
*/
int rank=Mask->rank();
//.........................................................
// Initialize communication structures in averaging domain
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
Mask->CommInit();
Np=Mask->PoreCount();
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device (will use optimized layout)
// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
ScaLBL_Comm = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
int Npad=(Np/16 + 2)*16;
if (rank==0) printf ("Set up memory efficient layout \n");
Map.resize(Nx,Ny,Nz); Map.fill(-2);
auto neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
//...........................................................................
// LBM variables
if (rank==0) printf ("Allocating distributions \n");
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
int neighborSize=18*(Np*sizeof(int));
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
if (rank==0) printf ("Setting up device map and neighbor list \n");
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
MPI_Barrier(comm);
}
void ScaLBL_MRTModel::Initialize(){
/*
* This function initializes model
*/
if (rank==0) printf ("Initializing distributions \n");
ScaLBL_D3Q19_Init(fq, Np);
}
void ScaLBL_MRTModel::Run(){
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
if (rank==0) printf("Beginning AA timesteps...\n");
if (rank==0) printf("********************************************************\n");
timestep=0;
while (timestep < timestepMax) {
//************************************************************************/
timestep++;
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
//************************************************************************/
}
//************************************************************************/
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
double MLUPS = double(Np*timestep)/cputime/1000000;
// if (rank==0) printf("********************************************************\n");
// if (rank==0) printf("CPU time = %f \n", cputime);
// if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
}
void ScaLBL_MRTModel::VelocityField(double *Vz){
int SIZE=Np*sizeof(double);
ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE);
}

68
models/MRTModel.h Normal file
View File

@ -0,0 +1,68 @@
/*
* Multi-relaxation time LBM Model
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <iostream>
#include <exception>
#include <stdexcept>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
class ScaLBL_MRTModel{
public:
ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM);
~ScaLBL_MRTModel();
// functions in they should be run
void ReadParams(string filename);
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void VelocityField(double *Vz);
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
double tau,mu;
double Fx,Fy,Fz,flux;
double din,dout;
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
// input database
std::shared_ptr<Database> db;
std::shared_ptr<Database> domain_db;
std::shared_ptr<Database> mrt_db;
IntArray Map;
DoubleArray Distance;
int *NeighborList;
double *fq;
double *Velocity;
double *Pressure;
private:
MPI_Comm comm;
// filenames
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);
};

2
tests/BlobAnalyzeParallel.cpp
View File

@ -290,7 +290,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm); // Initialize communications for domains
Dm.CommInit(); // Initialize communications for domains
MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum_global/Dm.Volume;

2
tests/BlobIdentifyParallel.cpp
View File

@ -100,7 +100,7 @@ int main(int argc, char **argv)
readRankData( rank, nx+2, ny+2, nz+2, Phase, SignDist );
// Communication the halos
fillHalo<double> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1);
fillHalo<double> fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1);
fillData.fill(Phase);
fillData.fill(SignDist);

34
tests/CMakeLists.txt
View File

@ -1,17 +1,17 @@
# Copy files for the tests
#ADD_LBPM_EXECUTABLE( lbpm_nonnewtonian_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )
ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )
ADD_LBPM_EXECUTABLE( lbpm_sphere_pp )
ADD_LBPM_EXECUTABLE( lbpm_random_pp )
ADD_LBPM_EXECUTABLE( lbpm_refine_pp )
ADD_LBPM_EXECUTABLE( lbpm_morphdrain_pp )
ADD_LBPM_EXECUTABLE( lbpm_morphopen_pp )
ADD_LBPM_EXECUTABLE( lbpm_morph_pp )
#ADD_LBPM_EXECUTABLE( lbpm_morph_pp )
ADD_LBPM_EXECUTABLE( lbpm_segmented_pp )
#ADD_LBPM_EXECUTABLE( lbpm_block_pp )
ADD_LBPM_EXECUTABLE( lbpm_segmented_decomp )
@ -24,7 +24,7 @@ ADD_LBPM_EXECUTABLE( lbpm_porenetwork_pp )
ADD_LBPM_EXECUTABLE( lbpm_plates_pp )
ADD_LBPM_EXECUTABLE( lbpm_squaretube_pp )
ADD_LBPM_EXECUTABLE( GenerateSphereTest )
ADD_LBPM_EXECUTABLE( ComponentLabel )
#ADD_LBPM_EXECUTABLE( ComponentLabel )
ADD_LBPM_EXECUTABLE( ColorToBinary )
ADD_LBPM_EXECUTABLE( DataAggregator )
ADD_LBPM_EXECUTABLE( BlobAnalysis )
@ -41,25 +41,25 @@ ADD_LBPM_TEST( pmmc_cylinder )
ADD_LBPM_TEST( TestTorus )
ADD_LBPM_TEST( TestFluxBC )
ADD_LBPM_TEST( TestMap )
ADD_LBPM_TEST( TestMRT )
ADD_LBPM_TEST( TestColorGrad )
ADD_LBPM_TEST( TestBubbleDFH )
#ADD_LBPM_TEST( TestMRT )
#ADD_LBPM_TEST( TestColorGrad )
#ADD_LBPM_TEST( TestColorGradDFH )
ADD_LBPM_TEST( TestColorGradDFH )
ADD_LBPM_TEST( TestColorMassBounceback )
ADD_LBPM_TEST( TestPressVel )
ADD_LBPM_TEST( TestPoiseuille )
ADD_LBPM_TEST( TestForceMoments )
ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db)
#ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db)
ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db)
ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db)
ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db)
ADD_LBPM_TEST( TestForceD3Q19 )
ADD_LBPM_TEST( TestMomentsD3Q19 )
ADD_LBPM_TEST( TestInterfaceSpeed )
ADD_LBPM_TEST( TestInterfaceSpeed ../example/Bubble/input.db)
ADD_LBPM_TEST( TestSphereCurvature )
#ADD_LBPM_TEST_1_2_4( TestTwoPhase )
ADD_LBPM_TEST_1_2_4( TestBlobIdentify )
#ADD_LBPM_TEST_PARALLEL( TestTwoPhase 8 )
ADD_LBPM_TEST_PARALLEL( TestBlobAnalyze 8 )
ADD_LBPM_TEST_PARALLEL( TestSegDist 8 )
ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 )
#ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 )
#ADD_LBPM_TEST_PARALLEL( TestMassConservationD3Q7 1 )
ADD_LBPM_TEST_1_2_4( testCommunication )
ADD_LBPM_TEST_1_2_4( testUtilities )

1
tests/ColorToBinary.cpp
View File

@ -189,6 +189,7 @@ int main(int argc, char **argv)
sprintf(LocalRankString,"%05d",proc);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Temp, nx*ny*nz);
for (k=1; k<nz-1; k++){
for (j=1; j<ny-1; j++){
for (i=1; i<nz-1; i++){

65
tests/ComponentLabel.cpp
View File

@ -12,68 +12,7 @@
using namespace std;
/*inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cDistEven, double *cDistOdd, int N)
{
int q,n;
double value;
ifstream File(FILENAME,ios::binary);
for (n=0; n<N; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
cDen[n] = value;
// if (n== 66276) printf("Density a = %f \n",value);
File.read((char*) &value, sizeof(value));
cDen[N+n] = value;
// if (n== 66276) printf("Density b = %f \n",value);
// Read the even distributions
for (q=0; q<10; q++){
File.read((char*) &value, sizeof(value));
cDistEven[q*N+n] = value;
// if (n== 66276) printf("dist even %i = %f \n",q,value);
}
// Read the odd distributions
for (q=0; q<9; q++){
File.read((char*) &value, sizeof(value));
cDistOdd[q*N+n] = value;
// if (n== 66276) printf("dist even %i = %f \n",q,value);
}
}
File.close();
}
inline void ReadBinaryFile(char *FILENAME, double *Data, int N)
{
int n;
double value;
ifstream File(FILENAME,ios::binary);
for (n=0; n<N; n++){
// Write the two density values
File.read((char*) &value, sizeof(value));
Data[n] = value;
}
File.close();
}
inline void SetPeriodicBC(DoubleArray &Scalar, int nx, int ny, int nz){
int i,j,k,in,jn,kn;
for (k=0; k<nz; k++){
for (j=0; j<ny; j++){
for (i=0; i<nx; i++){
in = i; jn=j; kn=k;
if (i==0) in = nx-2 ;
else if (i==nx-1) in = 0;
if (j==0) jn = ny-2;
else if (j==ny-1) jn = 0;
if (k==0) kn = nz-2;
else if (k==nz-1) kn = 0;
Scalar(i,j,k) = Scalar(in,jn,kn);
}
}
}
}
*/
inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, DoubleArray &Pressure, DoubleArray &Vel_x,
DoubleArray &Vel_y, DoubleArray &Vel_z, int nx, int ny, int nz, int iproc, int
jproc, int kproc)
@ -392,7 +331,7 @@ int main(int argc, char **argv)
}
porosity /= (Nx*Ny*Nz*1.0);
printf("Media porosity is %f \n",porosity);
Dm.CommInit(comm);
Dm.CommInit();
/* ****************************************************************
IDENTIFY ALL COMPONENTS FOR BOTH PHASES
@ -431,7 +370,7 @@ int main(int argc, char **argv)
Averages.PrintComponents(timestep);
// Create the MeshDataStruct
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
std::vector<IO::MeshDataStruct> meshData(1);
meshData[0].meshName = "domain";
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) );

211
tests/GenerateSphereTest.cpp
View File

@ -8,6 +8,7 @@
//#include "common/pmmc.h"
#include "common/Domain.h"
#include "common/SpherePack.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
@ -20,42 +21,39 @@
using namespace std;
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
for (idx=0; idx<count; idx++){
n = list[idx];
sendbuf[idx] = ID[n];
}
for (idx=0; idx<count; idx++){
n = list[idx];
sendbuf[idx] = ID[n];
}
}
//***************************************************************************************
inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
for (idx=0; idx<count; idx++){
n = list[idx];
ID[n] = recvbuf[idx];
}
for (idx=0; idx<count; idx++){
n = list[idx];
ID[n] = recvbuf[idx];
}
}
inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny, int nz, int rank, double SW){
int iproc = Dm.iproc;
int jproc = Dm.jproc;
int kproc = Dm.kproc;
int i,j,k,n;
int nprocx=Dm.nprocx;
int nprocy=Dm.nprocy;
int nprocz=Dm.nprocz;
double count,countGlobal,totalGlobal;
count = 0.f;
double maxdist=0.f;
double maxdistGlobal;
int nprocx=Dm.nprocx();
int nprocy=Dm.nprocy();
int nprocz=Dm.nprocz();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -135,7 +133,6 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
int sendtag,recvtag;
sendtag = recvtag = 7;
int x,y,z;
int ii,jj,kk;
int Nx = nx;
int Ny = ny;
@ -217,42 +214,42 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,Dm.Comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);
@ -330,17 +327,7 @@ int main(int argc, char **argv)
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
// MPI ranks for all 18 neighbors
//**********************************
int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
@ -349,73 +336,36 @@ int main(int argc, char **argv)
}
// Variables that specify the computational domain
string FILENAME;
unsigned int nBlocks, nthreads;
string filename;
int Nx,Ny,Nz; // local sub-domain size
int nspheres; // number of spheres in the packing
double Lx,Ly,Lz; // Domain length
double D = 1.0; // reference length for non-dimensionalization
int i,j,k,n;
filename = argv[1];
auto db = std::make_shared<Database>( filename );
auto domain_db = db->getDatabase( "Domain" );
if (argc > 1)
nspheres=atoi(argv[1]);
else nspheres=0;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> Lx;
domain >> Ly;
domain >> Lz;
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(comm);
auto Dm = std::shared_ptr<Domain>(new Domain(domain_db,comm)); // full domain for analysis
Nx = Dm->Nx;
Ny = Dm->Ny;
Nz = Dm->Nz;
Lx = Dm->Lx;
Ly = Dm->Ly;
Lz = Dm->Lz;
iproc = Dm->iproc();
jproc = Dm->jproc();
kproc = Dm->kproc();
nprocx = Dm->nprocx();
nprocy = Dm->nprocy();
nprocz = Dm->nprocz();
nspheres = domain_db->getScalar<int>( "nspheres");
printf("Set domain \n");
int BoundaryCondition=1;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
Nz += 2;
Nx = Ny = Nz; // Cubic domain
//Nz += 2;
//Nx = Ny = Nz; // Cubic domain
int N = Nx*Ny*Nz;
// Define Dm.Communication sub-domain -- everywhere
@ -423,13 +373,11 @@ int main(int argc, char **argv)
for (int j=0; j<Ny; j++){
for (int i=0; i<Nx; i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
Dm->id[n] = 1;
}
}
}
Dm.CommInit(comm);
int dist_mem_size = N*sizeof(double);
Dm->CommInit();
if (rank==0) printf("Number of nodes per side = %i \n", Nx);
if (rank==0) printf("Total Number of nodes = %i \n", N);
@ -444,7 +392,6 @@ int main(int argc, char **argv)
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
char tmpstr[10];
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
@ -500,8 +447,6 @@ int main(int argc, char **argv)
SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
iproc,jproc,kproc,nprocx,nprocy,nprocz);
//.......................................................................
// Assign the phase ID field based on the signed distance
//.......................................................................
for (k=0;k<Nz;k++){
@ -536,7 +481,7 @@ int main(int argc, char **argv)
// Run Morphological opening to initialize 50% saturation
double SW=0.50;
if (rank==0) printf("MorphOpen: Initializing with saturation %f \n",SW);
MorphOpen(SignDist, id, Dm, Nx, Ny, Nz, rank, SW);
MorphOpen(SignDist, id, *Dm, Nx, Ny, Nz, rank, SW);
//.........................................................
// don't perform computations at the eight corners

23
tests/TestBlobAnalyze.cpp
View File

@ -8,6 +8,7 @@
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "analysis/TwoPhase.h"
#include "common/SpherePack.h"
//#include "Domain.h"
@ -55,7 +56,7 @@ inline void WriteBlobs(TwoPhase Averages){
inline void WriteBlobStates(TwoPhase TCAT, double D, double porosity){
int a;
double iVol=1.0/TCAT.Dm.Volume;
double iVol=1.0/TCAT.Dm->Volume;
double PoreVolume;
double nwp_volume,vol_n,pan,pn,pw,pawn,pwn,awn,ans,aws,Jwn,Kwn,lwns,cwns,clwns;
double sw,awnD,awsD,ansD,lwnsDD,JwnD,pc;
@ -215,10 +216,10 @@ int main(int argc, char **argv)
// Set up the domain
int BC=0;
// Get the rank info
Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
std::shared_ptr<Domain> Dm(new Domain(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
// const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
TwoPhase Averages(Dm);
int N = (nx+2)*(ny+2)*(nz+2);
Nx = nx+2;
Ny = ny+2;
Nz = nz+2;
@ -228,12 +229,12 @@ int main(int argc, char **argv)
for ( j=1;j<Ny-1;j++){
for ( i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
Dm->id[n] = 1;
}
}
}
//.......................................................................
Dm.CommInit(comm); // Initialize communications for domains
Dm->CommInit(); // Initialize communications for domains
//.......................................................................
// Read in sphere pack (initialize the non-wetting phase as inside of spheres)
//
@ -263,7 +264,7 @@ int main(int argc, char **argv)
MPI_Barrier(comm);
//.......................................................................
SignedDistance(Averages.Phase.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
Dm.iproc,Dm.jproc,Dm.kproc,Dm.nprocx,Dm.nprocy,Dm.nprocz);
Dm->iproc(),Dm->jproc(),Dm->kproc(),Dm->nprocx(),Dm->nprocy(),Dm->nprocz());
//.......................................................................
// Assign the phase ID field based on the signed distance
//.......................................................................
@ -276,10 +277,10 @@ int main(int argc, char **argv)
Averages.SDs(i,j,k) = 100.0;
Averages.Phase(i,j,k) += 2.0;
if (Averages.Phase(i,j,k) > 0.0){
Dm.id[n] = 2;
Dm->id[n] = 2;
}
else{
Dm.id[n] = 1;
Dm->id[n] = 1;
}
Averages.SDn(i,j,k) = -Averages.Phase(i,j,k);
Averages.Phase(i,j,k) = Averages.SDn(i,j,k);
@ -293,13 +294,11 @@ int main(int argc, char **argv)
}
}
}
double beta = 0.95;
if (rank==0) printf("initializing the system \n");
Averages.UpdateSolid();
Dm.CommunicateMeshHalo(Averages.Phase);
Dm.CommunicateMeshHalo(Averages.SDn);
Dm->CommunicateMeshHalo(Averages.Phase);
Dm->CommunicateMeshHalo(Averages.SDn);
Averages.Initialize();
Averages.UpdateMeshValues();

8
tests/TestBlobIdentify.cpp
View File

@ -133,8 +133,8 @@ void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct
int ngy = ny+2*abs(sy);
int ngz = nz+2*abs(sz);
Array<double> tmp1(nx,ny,nz), tmp2(ngx,ngy,ngz), tmp3(ngx,ngy,ngz);
fillHalo<double> fillData1(comm,rank_info,nx,ny,nz,1,1,1,0,1);
fillHalo<double> fillData2(comm,rank_info,nx,ny,nz,abs(sx),abs(sy),abs(sz),0,1);
fillHalo<double> fillData1(comm,rank_info,{nx,ny,nz},{1,1,1},0,1);
fillHalo<double> fillData2(comm,rank_info,{nx,ny,nz},{abs(sx),abs(sy),abs(sz)},0,1);
fillData1.copy(data,tmp1);
fillData2.copy(tmp1,tmp2);
fillData2.fill(tmp2);
@ -189,7 +189,7 @@ int main(int argc, char **argv)
fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info );
// Communication the halos
fillHalo<double> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1);
fillHalo<double> fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1);
fillData.fill(Phase);
fillData.fill(SignDist);
@ -205,7 +205,7 @@ int main(int argc, char **argv)
// Create the MeshDataStruct
std::vector<IO::MeshDataStruct> meshData(1);
meshData[0].meshName = "domain";
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(rank_info,nx,ny,nz,Lx,Ly,Lz) );
meshData[0].mesh = std::make_shared<IO::DomainMesh>(rank_info,nx,ny,nz,Lx,Ly,Lz);
std::shared_ptr<IO::Variable> PhaseVar( new IO::Variable() );
std::shared_ptr<IO::Variable> SignDistVar( new IO::Variable() );
std::shared_ptr<IO::Variable> BlobIDVar( new IO::Variable() );

159
tests/TestBubble.cpp
View File

@ -16,6 +16,17 @@
using namespace std;
std::shared_ptr<Database> readInput( const std::string& file )
{
if ( exists( file )
return std::make_shared<Database>( file );
auto db = std::make_shared<Database>();
db.putData( "Color", std::make_shared<Database>() );
db.putData( "Domain", std::make_shared<Database>() );
return db;
}
int main(int argc, char **argv)
{
// Initialize MPI
@ -54,13 +65,6 @@ int main(int argc, char **argv)
int nspheres;
double Lx,Ly,Lz;
// Color Model parameters
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
double alpha, beta;
double das, dbs, phi_s;
double din,dout;
double wp_saturation;
bool pBC,Restart;
int i,j,k,n;
// pmmc threshold values
@ -71,109 +75,43 @@ int main(int argc, char **argv)
nthreads = 128;
int RESTART_INTERVAL=1000;
if (rank==0){
//.............................................................
// READ SIMULATION PARMAETERS FROM INPUT FILE
//.............................................................
ifstream input("Color.in");
if (input.good()){
input >> tau; // Viscosity parameter
input >> alpha; // Surface Tension parameter
input >> beta; // Width of the interface
input >> phi_s; // value of phi at the solid surface
input >> wp_saturation;
input >> Fx;
input >> Fy;
input >> Fz;
input >> Restart;
input >> pBC;
input >> din;
input >> dout;
input >> timestepMax; // max no. of timesteps
input >> interval; // restart interval
input >> tol; // error tolerance
}
else {
printf("WARNING: No valid Color.in file, using hard-coded values \n");
tau = 1.0;
alpha = 1e-3;
beta = 0.95;
phi_s = 0.0;
wp_saturation=Fx=Fy=Fz=0;
Restart = 0;
pBC = 0;
din=dout = 1.0;
timestepMax = 3;
interval = 100;
tol = 1e-6;
das = 0.1; dbs = 0.9; // hard coded for density initialization
// should be OK to remove these parameters
// they should have no impact with the
// current boundary condition
}
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else {
//.......................................................................
// Set the domain for single processor test
printf("WARNING: No valid Domain.in file, using hard-coded values \n");
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=50;
nspheres=1;
Lx=Ly=Lz=1;
}
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&interval,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
// MPI_Bcast(&nBlocks,1,MPI_INT,0,comm);
// MPI_Bcast(&nthreads,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
auto db = readInput( "input.in" );
// Read variables from Color
auto color_db = db->getDatabase( "Color" );
auto tau = color_db->getScalarWithDefault<double>( "tau", 1.0 );
auto alpha = color_db->getScalarWithDefault<double>( "alpha", 1e-3 );
auto beta = color_db->getScalarWithDefault<double>( "beta", 0.95 );
auto phi_s = color_db->getScalarWithDefault<double>( "phi_s", 0.0 );
auto Fx = color_db->getVectorWithDefault<double>( "F", { 0, 0, 0 } )[0];
auto Fy = color_db->getScalarWithDefault<double>( "F", { 0, 0, 0 } )[1];
auto Fz = color_db->getScalarWithDefault<double>( "F", { 0, 0, 0 } )[2];
auto Restart = color_db->getScalarWithDefault<bool>( "Restart", 0 );
auto pBC = color_db->getScalarWithDefault<double>( "pBC", 0 );
auto din = color_db->getScalarWithDefault<double>( "din", 1.0 );
auto dout = color_db->getScalarWithDefault<double>( "dout", 1.0 );
auto timestepMax = color_db->getScalarWithDefault<int>( "timestepMax", 3 );
auto interval = color_db->getScalarWithDefault<int>( "interval", 100 );
auto tol = color_db->getScalarWithDefault<double>( "tol", 1e-6 );
auto das = color_db->getScalarWithDefault<double>( "das", 0.1 );
auto dbs = color_db->getScalarWithDefault<double>( "dab", 0.9 );
// Read variables from Domain
auto domain_db = db->getDatabase( "Domain" );
auto n = domain_db->getVectorWithDefault<int>( "n", { 50, 50, 50 } );
auto L = domain_db->getVectorWithDefault<double>( "L", { 1.0, 1.0, 1.0 } );
auto nproc = domain_db->getVectorWithDefault<int>( "nproc", { 1, 1, 1 } );
auto nspheres = domain_db->getScalarWithDefault<int>( "nspheres", 1 );
int Nx = n[0];
int Ny = n[1];
int Nz = n[2];
int nprocx = nproc[0];
int nprocy = nproc[1];
int nprocz = nproc[2];
double Lx = L[0];
double Ly = L[1];
double Lz = L[2];
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
@ -219,7 +157,7 @@ int main(int argc, char **argv)
// Full domain used for averaging (do not use mask for analysis)
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC);
Dm.CommInit(comm);
Dm.CommInit();
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC);
@ -227,7 +165,6 @@ int main(int argc, char **argv)
MPI_Barrier(comm);
Nx+=2; Ny+=2; Nz += 2;
//Nx = Ny = Nz; // Cubic domain
int N = Nx*Ny*Nz;
int dist_mem_size = N*sizeof(double);

462
tests/TestBubbleDFH.cpp
View File

@ -33,11 +33,9 @@ int main(int argc, char **argv)
MPI_Comm_dup(MPI_COMM_WORLD,&comm);
int rank = comm_rank(comm);
int nprocs = comm_size(comm);
int check=0;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int i,j,k,n,Np;
if (rank == 0){
printf("********************************************************\n");
printf("Running DFH/Color LBM \n");
@ -56,229 +54,136 @@ int main(int argc, char **argv)
PROFILE_START("Main");
Utilities::setErrorHandlers();
int ANALYSIS_INTERVAL = 1000;
int BLOBID_INTERVAL = 1000;
std::string analysis_method = "independent";
if (argc >= 3) {
ANALYSIS_INTERVAL = atoi(argv[1]);
BLOBID_INTERVAL = atoi(argv[2]);
}
if (argc >= 4)
analysis_method = std::string(argv[3]);
if ( argc < 2 ) {
std::cerr << "Invalid number of arguments, no input file specified\n";
return -1;
}
auto filename = argv[1];
auto db = std::make_shared<Database>( filename );
auto domain_db = db->getDatabase( "Domain" );
auto color_db = db->getDatabase( "Color" );
auto analysis_db = db->getDatabase( "Analysis" );
// Variables that specify the computational domain
string FILENAME;
int Nx,Ny,Nz,Np; // local sub-domain size
double Lx,Ly,Lz; // Domain length
// Color Model parameters
int timestepMax;
double tauA, tauB, rhoA,rhoB;
double Fx,Fy,Fz,tol,err;
double alpha, beta;
double bns,bws,cns,cws;
int BoundaryCondition;
int InitialCondition;
// bool pBC,Restart;
int i,j,k,n;
double din, dout, flux;
double inletA,inletB,outletA,outletB;
inletA=1.f;
inletB=0.f;
outletA=0.f;
outletB=1.f;
flux = 10.f;
dout=1.f;
if (rank == 0){
printf("********************************************************\n");
printf("Running Color LBM \n");
printf("********************************************************\n");
}
// Initialize compute device
// int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
int RESTART_INTERVAL=20000;
//int ANALYSIS_)INTERVAL=1000;
int BLOB_ANALYSIS_INTERVAL=1000;
int timestep = 0;
PROFILE_ENABLE(1);
//PROFILE_ENABLE_TRACE();
//PROFILE_ENABLE_MEMORY();
PROFILE_SYNCHRONIZE();
PROFILE_START("Main");
Utilities::setErrorHandlers();
if (rank==0){
//.............................................................
// READ SIMULATION PARMAETERS FROM INPUT FILE
//.............................................................
ifstream input("Color.in");
if (input.is_open()){
// Line 1: model parameters (tau, alpha, beta, das, dbs)
input >> tauA; // Viscosity non-wetting
input >> tauB; // Viscosity wetting
input >> rhoA; // density non-wetting
input >> rhoB; // density wetting
input >> alpha; // Surface Tension parameter
input >> beta; // Width of the interface
input >> cws; // solid interaction coefficients
input >> bws; // solid interaction coefficients
input >> cns; // solid interaction coefficients
input >> bns; // solid interaction coefficients
// Line 2: External force components (Fx,Fy, Fz)
input >> Fx;
input >> Fy;
input >> Fz;
// Line 4: Pressure Boundary conditions
input >> InitialCondition;
input >> BoundaryCondition;
input >> din;
input >> dout;
// Line 5: time-stepping criteria
input >> timestepMax; // max no. of timesteps
input >> RESTART_INTERVAL; // restart interval
input >> tol; // error tolerance
// Line 6: Analysis options
input >> BLOB_ANALYSIS_INTERVAL; // interval to analyze blob states
//.............................................................
}
else{
// Set default values
// Print warning
printf("WARNING: No input file provided (Color.in is missing)! Default parameters will be used. \n");
tauA = tauB = 1.0;
rhoA = rhoB = 1.0;
alpha=0.005;
beta= 0.9;
Fx = Fy = Fz = 0.0;
InitialCondition=0;
BoundaryCondition=0;
din=dout=1.0;
timestepMax=0;
}
// Variables that specify the computational domain
string FILENAME;
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (input.is_open()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> Lx;
domain >> Ly;
domain >> Lz;
//.......................................................................
}
else{
// Set default values
// Print warning
printf("WARNING: No input file provided (Domain.in is missing)! Default parameters will be used. \n");
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=10;
Lx=Ly=Lz=1.0;
}
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&cns,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&cws,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&bns,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&bws,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm);
MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
// Color Model parameters
int timestepMax = color_db->getScalar<int>( "timestepMax" );
double tauA = color_db->getScalar<double>( "tauA" );
double tauB = color_db->getScalar<double>( "tauB" );
double rhoA = color_db->getScalar<double>( "rhoA" );
double rhoB = color_db->getScalar<double>( "rhoB" );
double Fx = color_db->getVector<double>( "F" )[0];
double Fy = color_db->getVector<double>( "F" )[1];
double Fz = color_db->getVector<double>( "F" )[2];
double alpha = color_db->getScalar<double>( "alpha" );
double beta = color_db->getScalar<double>( "beta" );
bool Restart = color_db->getScalar<bool>( "Restart" );
double din = color_db->getScalar<double>( "din" );
double dout = color_db->getScalar<double>( "dout" );;
double inletA=1.f;
double inletB=0.f;
double outletA=0.f;
double outletB=1.f;
double flux = 10.f;
flux = 0.f;
if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details
// Read domain values
auto L = domain_db->getVector<double>( "L" );
auto size = domain_db->getVector<int>( "n" );
auto nproc = domain_db->getVector<int>( "nproc" );
int BoundaryCondition = domain_db->getScalar<int>( "BC" );
int Nx = size[0];
int Ny = size[1];
int Nz = size[2];
int nprocx = nproc[0];
int nprocy = nproc[1];
int nprocz = nproc[2];
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
int timestep = 6;
MPI_Barrier(comm);
if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
if (rank==0){
printf("********************************************************\n");
printf("tau (non-wetting) = %f \n", tauA);
printf("tau (wetting) = %f \n", tauB);
printf("density (non-wetting) = %f \n", rhoA);
printf("density (wetting) = %f \n", rhoB);
printf("alpha = %f \n", alpha);
printf("beta = %f \n", beta);
printf("gamma_{wn} = %f \n", 5.796*alpha);
printf("Force(x) = %f \n", Fx);
printf("Force(y) = %f \n", Fy);
printf("Force(z) = %f \n", Fz);
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz);
if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n");
if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n");
if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n");
if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n");
if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n");
if (InitialCondition==0) printf("Initial conditions assigned from phase ID file \n");
if (InitialCondition==1) printf("Initial conditions assigned from restart file \n");
printf("********************************************************\n");
}
MPI_Barrier(comm);
// Initialized domain and averaging framework for Two-Phase Flow
bool pBC,velBC;
if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4)
pBC=true;
else pBC=false;
if (BoundaryCondition==2) velBC=true;
else velBC=false;
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
bool Restart;
if (InitialCondition==1) Restart=true;
else Restart=false;
NULL_USE(pBC); NULL_USE(velBC);
if (rank==0){
printf("********************************************************\n");
printf("tau (non-wetting) = %f \n", tauA);
printf("tau (wetting) = %f \n", tauB);
printf("density (non-wetting) = %f \n", rhoA);
printf("density (wetting) = %f \n", rhoB);
printf("alpha = %f \n", alpha);
printf("beta = %f \n", beta);
printf("gamma_{wn} = %f \n", 5.796*alpha);
printf("Force(x) = %f \n", Fx);
printf("Force(y) = %f \n", Fy);
printf("Force(z) = %f \n", Fz);
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz);
if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n");
if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n");
if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n");
if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n");
if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n");
if (!Restart) printf("Initial conditions assigned from phase ID file \n");
if (Restart) printf("Initial conditions assigned from restart file \n");
printf("********************************************************\n");
}
// Full domain used for averaging (do not use mask for analysis)
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
// TwoPhase Averages(Dm);
Dm.CommInit(comm);
// Initialized domain and averaging framework for Two-Phase Flow
bool pBC;
if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4)
pBC=true;
else
pBC=false;
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
MPI_Barrier(comm);
// Full domain used for averaging (do not use mask for analysis)
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
for (int i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
// TwoPhase Averages(Dm);
Dm->CommInit();
Nx+=2; Ny+=2; Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
// Mask that excludes the solid phase
std::shared_ptr<Domain> Mask(new Domain(domain_db,comm));
MPI_Barrier(comm);
Nx+=2; Ny+=2; Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
//.......................................................................
// Filenames used
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
char tmpstr[10];
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
@ -289,9 +194,6 @@ int main(int argc, char **argv)
char *id;
id = new char[N];
double sum, sum_local;
double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6));
double porosity, pore_vol;
//...........................................................................
if (rank == 0) cout << "Setting up bubble..." << endl;
double BubbleRadius = 15.5; // Radius of the capillary tube
@ -313,13 +215,13 @@ int main(int argc, char **argv)
}
}
// Initialize the bubble
for (k=0;k<Nz;k++){
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
int iglobal= i+(Nx-2)*Dm.iproc;
int jglobal= j+(Ny-2)*Dm.jproc;
int kglobal= k+(Nz-2)*Dm.kproc;
int n = k*Nx*Ny + j*Nz + i;
int iglobal= i+(Nx-2)*Dm->iproc();
int jglobal= j+(Ny-2)*Dm->jproc();
int kglobal= k+(Nz-2)*Dm->kproc();
// Initialize phase position field for parallel bubble test
if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx)
+(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy)
@ -340,22 +242,22 @@ int main(int argc, char **argv)
//.........................................................
// Initialize communication structures in averaging domain
for (i=0; i<Mask.Nx*Mask.Ny*Mask.Nz; i++) Mask.id[i] = id[i];
Mask.CommInit(comm);
for (i=0; i<Mask->Nx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i];
Mask->CommInit();
double *PhaseLabel;
PhaseLabel = new double[N];
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device (will use optimized layout)
ScaLBL_Communicator ScaLBL_Comm(Mask);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Mask));
int Npad=(Np/16 + 2)*16;
if (rank==0) printf ("Set up memory efficient layout Npad=%i \n",Npad);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask.id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
MPI_Barrier(comm);
//...........................................................................
@ -413,6 +315,8 @@ int main(int argc, char **argv)
Tmp=new double[3*Np];
Averages->UpdateMeshValues(); // this computes the gradient of distance field (among other things)
double count_wet=0.f;
double cns,bns,cws,bws;
cns=bns=bws=cws=1.0;
for (k=1; k<Nz-1; k++){
for (j=1; j<Ny-1; j++){
for (i=1; i<Nx-1; i++){
@ -428,8 +332,8 @@ int main(int argc, char **argv)
Tmp[idx+Np] = value*dy;
Tmp[idx+2*Np] = value*dz;
// initialize fluid phases
if (Mask.id[n] == 1) PhaseLabel[idx] = 1.0;
else if (Mask.id[n] == 2){
if (Mask->id[n] == 1) PhaseLabel[idx] = 1.0;
else if (Mask->id[n] == 2){
PhaseLabel[idx] = -1.0;
count_wet +=1.0;
}
@ -454,13 +358,13 @@ int main(int argc, char **argv)
if (rank==0) printf ("Initializing distributions \n");
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np);
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->last_interior, Np);
//.......................................................................
// Once phase has been initialized, map solid to account for 'smeared' interface
//for (i=0; i<N; i++) Averages.SDs(i) -= (1.0);
// Make sure the id match for the two domains
for (i=0; i<N; i++) Dm.id[i] = Mask.id[i];
for (i=0; i<N; i++) Dm->id[i] = Mask->id[i];
//.......................................................................
// Finalize setup for averaging domain
Averages->UpdateSolid();
@ -478,15 +382,16 @@ int main(int argc, char **argv)
//...........................................................................
ScaLBL_DeviceBarrier();
ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double));
ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press);
ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x);
ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y);
ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z);
ScaLBL_Comm->RegularLayout(Map,Pressure,Averages->Press);
ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Averages->Vel_x);
ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Averages->Vel_y);
ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z);
//...........................................................................
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps: %i \n", timestepMax);
double err=1.0;
double tol=1.0e-6;
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
@ -495,17 +400,13 @@ int main(int argc, char **argv)
//.........................................
err = 1.0;
double sat_w_previous = 1.01; // slightly impossible value!
if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol);
if (rank==0){
printf("Analysis intervals: (restart) %i, (TCAT) %i, (blobtracking) %i \n",RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL);
}
//************ MAIN ITERATION LOOP ***************************************/
PROFILE_START("Loop");
runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL,
rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile );
analysis.createThreads( analysis_method, 4 );
//std::shared_ptr<Database> analysis_db;
runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map );
//analysis.createThreads( analysis_method, 4 );
while (timestep < timestepMax && err > tol ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
@ -513,73 +414,73 @@ int main(int argc, char **argv)
timestep++;
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np);
ScaLBL_Comm.RecvGrad(Phi,Gradient);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set BCs
if (BoundaryCondition > 0){
ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (BoundaryCondition == 3){
ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
if (BoundaryCondition == 4){
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np);
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
// *************EVEN TIMESTEP*************
timestep++;
// Compute the Phase indicator field
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np);
ScaLBL_Comm.RecvGrad(Phi,Gradient);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set boundary conditions
if (BoundaryCondition > 0){
ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (BoundaryCondition == 3){
ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 4){
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np);
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
//************************************************************************
MPI_Barrier(comm);
@ -613,7 +514,7 @@ int main(int argc, char **argv)
// Copy back final phase indicator field and convert to regular layout
DoubleArray PhaseField(Nx,Ny,Nz);
ScaLBL_Comm.RegularLayout(Map,Phi,PhaseField);
ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField);
FILE *OUTFILE;
sprintf(LocalRankFilename,"Phase.raw",rank);
OUTFILE = fopen(LocalRankFilename,"wb");
@ -624,9 +525,9 @@ int main(int argc, char **argv)
DoubleArray Cy(Nx,Ny,Nz);
DoubleArray Cz(Nx,Ny,Nz);
DoubleArray GradNorm(Nx,Ny,Nz);
ScaLBL_Comm.RegularLayout(Map,&Gradient[0],Cx);
ScaLBL_Comm.RegularLayout(Map,&Gradient[Np],Cy);
ScaLBL_Comm.RegularLayout(Map,&Gradient[2*Np],Cz);
ScaLBL_Comm->RegularLayout(Map,&Gradient[0],Cx);
ScaLBL_Comm->RegularLayout(Map,&Gradient[Np],Cy);
ScaLBL_Comm->RegularLayout(Map,&Gradient[2*Np],Cz);
for (k=1; k<Nz-1; k++){
for (j=1; j<Ny-1; j++){
for (i=1; i<Nx-1; i++){
@ -635,22 +536,22 @@ int main(int argc, char **argv)
}
}
FILE *GFILE;
sprintf(LocalRankFilename,"Gradient.raw",rank);
sprintf(LocalRankFilename,"Gradient.raw");
GFILE = fopen(LocalRankFilename,"wb");
fwrite(GradNorm.data(),8,N,GFILE);
fclose(GFILE);
DoubleArray Rho1(Nx,Ny,Nz);
DoubleArray Rho2(Nx,Ny,Nz);
ScaLBL_Comm.RegularLayout(Map,&Den[0],Rho1);
ScaLBL_Comm.RegularLayout(Map,&Den[Np],Rho2);
ScaLBL_Comm->RegularLayout(Map,&Den[0],Rho1);
ScaLBL_Comm->RegularLayout(Map,&Den[Np],Rho2);
FILE *RFILE1;
sprintf(LocalRankFilename,"Rho1.raw",rank);
sprintf(LocalRankFilename,"Rho1.raw");
RFILE1 = fopen(LocalRankFilename,"wb");
fwrite(Rho1.data(),8,N,RFILE1);
fclose(RFILE1);
FILE *RFILE2;
sprintf(LocalRankFilename,"Rho2.raw",rank);
sprintf(LocalRankFilename,"Rho2.raw");
RFILE2 = fopen(LocalRankFilename,"wb");
fwrite(Rho2.data(),8,N,RFILE2);
fclose(RFILE2);
@ -662,6 +563,7 @@ int main(int argc, char **argv)
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Comm_free(&comm);
MPI_Finalize();
return check;
}

548
tests/TestColorBubble.cpp
View File

@ -8,82 +8,52 @@
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "models/ColorModel.h"
using namespace std;
inline void AssignComponentLabels(char *id, double *phase, int Nx, int Ny, int Nz, int rank, MPI_Comm comm)
{
int NLABELS=0;
char VALUE=0;
double AFFINITY=0.f;
vector <char> Label;
vector <double> Affinity;
// Read the labels
if (rank==0){
printf("Component labels:\n");
ifstream iFILE("ComponentLabels.csv");\
if (iFILE.good()){
while (!iFILE.eof()){
iFILE>>VALUE;
iFILE>>AFFINITY;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
printf("%i %f\n",VALUE,AFFINITY);
inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){
// initialize a bubble
int i,j,k,n;
int rank = ColorModel.Mask->rank();
int nprocx = ColorModel.Mask->nprocx();
int nprocy = ColorModel.Mask->nprocy();
int nprocz = ColorModel.Mask->nprocz();
int Nx = ColorModel.Mask->Nx;
int Ny = ColorModel.Mask->Ny;
int Nz = ColorModel.Mask->Nz;
if (rank == 0) cout << "Setting up bubble..." << endl;
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
ColorModel.Averages->SDs(i,j,k) = 100.f;
}
}
else{
printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n");
// Set default values
VALUE=0; AFFINITY=-1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
printf("%i %f\n",VALUE,AFFINITY);
VALUE=1; AFFINITY=1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
printf("%i %f\n",VALUE,AFFINITY);
VALUE=2; AFFINITY=-1.0;
Label.push_back(VALUE);
Affinity.push_back(AFFINITY);
NLABELS++;
printf("%i %f\n",VALUE,AFFINITY);
}
}
// Broadcast the list
MPI_Bcast(&NLABELS,1,MPI_INT,0,comm);
// Copy into contiguous buffers
char *LabelList;
double * AffinityList;
LabelList=new char[NLABELS];
AffinityList=new double[NLABELS];
MPI_Bcast(&LabelList,NLABELS,MPI_CHAR,0,comm);
MPI_Bcast(&AffinityList,NLABELS,MPI_DOUBLE,0,comm);
// Assign the labels
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
VALUE=id[n];
// Assign the affinity from the paired list
for (int idx=0; idx < NLABELS; idx++){
if (VALUE == LabelList[idx]){
AFFINITY=AffinityList[idx];
idx = NLABELS;
}
// Initialize the bubble
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
int iglobal= i+(Nx-2)*ColorModel.Mask->iproc();
int jglobal= j+(Ny-2)*ColorModel.Mask->jproc();
int kglobal= k+(Nz-2)*ColorModel.Mask->kproc();
// Initialize phase position field for parallel bubble test
if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx)
+(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy)
+(kglobal-0.5*(Nz-2)*nprocz)*(kglobal-0.5*(Nz-2)*nprocz) < BubbleRadius*BubbleRadius){
ColorModel.Mask->id[n] = 2;
ColorModel.Mask->id[n] = 2;
}
else{
ColorModel.Mask->id[n]=1;
ColorModel.Mask->id[n]=1;
}
phase[n] = AFFINITY;
}
}
}
// initialize the phase indicator field
}
//***************************************************************************************
int main(int argc, char **argv)
{
@ -96,444 +66,28 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Color Model: TestColor \n");
printf("********************************************************\n");
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 50;
//if (rank == 0) printf("dim=%d\n",dim);
int timestep = 0;
int timesteps = 100;
int centralNode = 2;
double tauA = 1.0;
double tauB = 1.0;
double rhoA = 1.0;
double rhoB = 1.0;
double alpha = 0.001;
double beta = 0.95;
double tau = 1.0;
double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
Fx = Fy = 0.f;
Fz = 0.f;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=3; Ny = 1;
Nz = 1;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=2; nprocy=1;
nprocz=1;
Nx=Ny=Nz=dim;
Nx = dim; Ny = dim; Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
MPI_Barrier(comm);
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
// Assign the phase ID field
//.......................................................................
char LocalRankString[8];
sprintf(LocalRankString,"%05d",rank);
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rank);
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nx + i;
Dm.id[n]=1;
}
if ( argc < 2 ) {
std::cerr << "Invalid number of arguments, no input file specified\n";
return -1;
}
}
Dm.CommInit(comm);
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nx + i;
Dm.id[n]=0;
}
}
}
printf("rank=%i, %i,%i,%i \n",rank,Dm.iproc,Dm.jproc,Dm.jproc);
// Initialize a bubble
int BubbleRadius=Nx/3;
int center_x = (Nx-2)*nprocx/2;
int center_y = (Ny-2)*nprocy/2;
int center_z = (Nz-2)*nprocz/2;
if (rank==0) printf("Bubble radius = %i, center=%i,%i,%i \n",BubbleRadius,center_x,center_y,center_z);
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny + j*Nx + i;
int iglobal= i+(Nx-2)*Dm.iproc;
int jglobal= j+(Ny-2)*Dm.jproc;
int kglobal= k+(Nz-2)*Dm.kproc;
// Initialize phase position field for parallel bubble test
if ((iglobal-center_x)*(iglobal-center_x)
+(jglobal-center_y)*(jglobal-center_y)
+(kglobal-center_z)*(kglobal-center_z) < BubbleRadius*BubbleRadius){
Dm.id[n] = 2;
}
else{
Dm.id[n]=1;
}
}
}
}
//.......................................................................
// Compute the media porosity, assign phase labels and solid composition
//.......................................................................
double sum;
double sum_local=0.0, porosity;
int Np=0; // number of local pore nodes
double *PhaseLabel;
PhaseLabel = new double[N];
Dm.AssignComponentLabels(PhaseLabel);
//.......................................................................
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
sum_local+=1.0;
Np++;
}
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
if (rank==0) printf ("Create ScaLBL_Communicator \n");
MPI_Barrier(comm);
// Create a communicator for the device (will use optimized layout)
ScaLBL_Communicator ScaLBL_Comm(Dm);
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular(Dm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm.id, N);
//...........................................................................
if (rank==0){
printf("Total domain size = %i \n",N);
printf("Reduced domain size = %i \n",Np);
}
// LBM variables
if (rank==0) printf ("Set up the neighborlist \n");
int neighborSize=18*Np*sizeof(int);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
if (rank==0) printf ("Allocating distributions \n");
int *NeighborList;
int *dvcMap;
// double *f_even,*f_odd;
double *fq, *Aq, *Bq;
double *Den, *Phi;
double *ColorGrad;
double *Vel;
double *Pressure;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
if (rank==0) printf ("Setting up device map and neighbor list \n");
int *TmpMap;
TmpMap=new int[Np];
for (k=1; k<Nz-1; k++){
for (j=1; j<Ny-1; j++){
for (i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
if (!(idx < 0))
TmpMap[idx] = k*Nx*Ny+j*Nx+i;
}
}
}
//for (int idx=0; idx<Np; idx++) printf("Map=%i\n",TmpMap[idx]);
ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
ScaLBL_DeviceBarrier();
delete [] TmpMap;
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
// initialize phi based on PhaseLabel (include solid component labels)
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
//...........................................................................
if (rank==0) printf ("Initializing distributions \n");
// Initialize the phase field and variables
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np);
//************ MAIN ITERATION LOOP (timing communications)***************************************
if (rank==0) printf("Beginning AA timesteps...\n");
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps for timing: %i \n", timesteps);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
//timesteps=4;
while (timestep < timesteps) {
// ODD TIMESTEP
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Compute the Color Gradient
ScaLBL_Comm_Regular.SendHalo(Phi);
//ScaLBL_D3Q19_ColorGrad(dvcMap, Phi, ColorGrad, ScaLBL_Comm.next, Np, Np, Nx, Ny, Nz);
ScaLBL_Comm_Regular.RecvHalo(Phi);
//ScaLBL_D3Q19_ColorGrad(dvcMap, Phi, ColorGrad, 0,ScaLBL_Comm.next, Np, Nx, Ny, Nz);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
// ScaLBL_D3Q19_AAodd_ColorMomentum(NeighborList, fq, Den, Vel, ColorGrad, rhoA, rhoB, tauA, tauB,
// alpha, beta, Fx, Fy, Fz,ScaLBL_Comm.next, Np, Np);
// ScaLBL_D3Q19_AAodd_ColorMass(NeighborList, Aq, Bq, Den, Vel, ColorGrad, beta, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
// ScaLBL_D3Q19_AAodd_ColorMomentum(NeighborList, fq, Den, Vel, ColorGrad, rhoA, rhoB, tauA, tauB,
// alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np);
// ScaLBL_D3Q19_AAodd_ColorMass(NeighborList, Aq, Bq, Den, Vel, ColorGrad, beta, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
// EVEN TIMESTEP
// Compute the Phase indicator field
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Compute the Color Gradient
ScaLBL_Comm_Regular.SendHalo(Phi);
//ScaLBL_D3Q19_ColorGrad(dvcMap, Phi, ColorGrad, ScaLBL_Comm.next, Np, Np, Nx, Ny, Nz);
ScaLBL_Comm_Regular.RecvHalo(Phi);
//ScaLBL_D3Q19_ColorGrad(dvcMap, Phi, ColorGrad, 0, ScaLBL_Comm.next, Np, Nx, Ny, Nz);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
// ScaLBL_D3Q19_AAeven_ColorMomentum(fq, Den, Vel, ColorGrad, rhoA, rhoB, tauA, tauB,
// alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.next, Np, Np);
// ScaLBL_D3Q19_AAeven_ColorMass(Aq, Bq, Den, Vel, ColorGrad, beta, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
// ScaLBL_D3Q19_AAeven_ColorMomentum(fq, Den, Vel, ColorGrad, rhoA, rhoB, tauA, tauB,
// alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np);
// ScaLBL_D3Q19_AAeven_ColorMass(Aq, Bq, Den, Vel, ColorGrad, beta, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************
}
//************************************************************************
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
double MLUPS = double(Np*timestep)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n");
// Number of memory references for color model
double MemoryRefs = double(Np)*(77*8+(9+7+7)*4); // extra memory refs to read from neighborlist (every other timestep)
// number of memory references for the swap algorithm - GigaBytes / second
if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*timestep/1e9/cputime);
// Report bandwidth in Gigabits per second
// communication bandwidth includes both send and recieve
if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
double *VEL;
VEL= new double [3*Np];
int SIZE=3*Np*sizeof(double);
ScaLBL_D3Q19_Momentum(fq,Vel,Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_CopyToHost(&VEL[0],&Vel[0],SIZE);
sum_local=0.f;
sum = 0.f;
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
int idx = Map(i,j,k);
sum_local+=VEL[2*Np+idx];
}
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
double PoreVel = sum*iVol_global;
if (rank==0) printf("Velocity = %f \n",PoreVel);
/*
double *PHASE;
PHASE= new double [Nx*Ny*Nz];
SIZE=Nx*Ny*Nz*sizeof(double);
ScaLBL_CopyToHost(&PHASE[0],&Phi[0],SIZE);
FILE *OUTFILE;
sprintf(LocalRankFilename,"Phase.%05i.raw",rank);
OUTFILE = fopen(LocalRankFilename,"wb");
fwrite(PHASE,8,N,OUTFILE);
fclose(OUTFILE);
*/
auto filename = argv[1];
ScaLBL_ColorModel ColorModel(rank,nprocs,comm);
ColorModel.ReadParams(filename);
ColorModel.SetDomain();
//ColorModel.ReadInput();
double radius=15.5;
InitializeBubble(ColorModel,radius);
ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables
ColorModel.Initialize(); // initializing the model will set initial conditions for variables
ColorModel.Run();
ColorModel.WriteDebug();
}
// ****************************************************
MPI_Barrier(comm);

2
tests/TestColorGrad.cpp
View File

@ -174,7 +174,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");

139
tests/TestColorGradDFH.cpp
View File

@ -11,6 +11,16 @@
using namespace std;
std::shared_ptr<Database> loadInputs( int nprocs )
{
auto db = std::make_shared<Database>();
db->putScalar<int>( "BC", 0 );
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 100, 100, 100 } );
db->putScalar<int>( "nspheres", 1 );
db->putVector<double>( "L", { 1, 1, 1 } );
return db;
}
//***************************************************************************************
int main(int argc, char **argv)
@ -24,13 +34,9 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Color Model: TestColorGradDFH \n");
@ -39,13 +45,11 @@ int main(int argc, char **argv)
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 3;
//if (rank == 0) printf("dim=%d\n",dim);
@ -67,92 +71,24 @@ int main(int argc, char **argv)
Fx = Fy = 0.f;
Fz = 0.f;
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=3;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=2; nprocy=1;
nprocz=1;
Nx=Ny=Nz=dim;
Nx = dim; Ny = dim; Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
MPI_Barrier(comm);
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
// Get the rank info
std::shared_ptr<Domain> Dm(new Domain(db,comm));
Nx += 2;
Ny += 2;
Nz += 2;
@ -165,21 +101,20 @@ int main(int argc, char **argv)
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n]=1;
Dm->id[n]=1;
// Initialize gradient ColorGrad = (1,2,3)
double value=double(3*k+2*j+i);
PhaseLabel[n]= value;
}
}
}
Dm.CommInit(comm);
Dm->CommInit();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular(Dm);
ScaLBL_Communicator ScaLBL_Comm(Dm);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Dm));
// LBM variables
if (rank==0) printf ("Set up the neighborlist \n");
@ -196,7 +131,7 @@ int main(int argc, char **argv)
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
@ -253,10 +188,10 @@ int main(int argc, char **argv)
//...........................................................................
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, 0, ScaLBL_Comm.first_interior, Np);
ScaLBL_Comm.RecvGrad(Phi,ColorGrad);
ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, 0, ScaLBL_Comm->first_interior, Np);
ScaLBL_Comm->RecvGrad(Phi,ColorGrad);
double *COLORGRAD;
COLORGRAD= new double [3*Np];
@ -267,7 +202,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
printf("%i ",idx);
}
@ -282,7 +217,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
CX=COLORGRAD[idx];
CY=COLORGRAD[Np+idx];

424
tests/TestColorMassBounceback.cpp
View File

@ -24,139 +24,110 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int i,j,k,n,Npad;
auto filename = argv[1];
auto db = std::make_shared<Database>( filename );
auto domain_db = db->getDatabase( "Domain" );
auto color_db = db->getDatabase( "Color" );
auto analysis_db = db->getDatabase( "Analysis" );
if (rank == 0){
printf("********************************************************\n");
printf("TestColorMassBounceback \n");
printf("********************************************************\n");
}
// Initialize compute device
// int device=ScaLBL_SetDevice(rank);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
Utilities::setErrorHandlers();
if (rank == 0){
printf("********************************************************\n");
printf("Running Color Model: TestColor \n");
printf("********************************************************\n");
}
// Variables that specify the computational domain
string FILENAME;
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 3;
//if (rank == 0) printf("dim=%d\n",dim);
int timestep = 0;
int timesteps = 100;
int centralNode = 2;
// Color Model parameters
double tauA = color_db->getScalar<double>( "tauA" );
double tauB = color_db->getScalar<double>( "tauB" );
double rhoA = color_db->getScalar<double>( "rhoA" );
double rhoB = color_db->getScalar<double>( "rhoB" );
double Fx = color_db->getVector<double>( "F" )[0];
double Fy = color_db->getVector<double>( "F" )[1];
double Fz = color_db->getVector<double>( "F" )[2];
double alpha = color_db->getScalar<double>( "alpha" );
double beta = color_db->getScalar<double>( "beta" );
bool Restart = color_db->getScalar<bool>( "Restart" );
double din = color_db->getScalar<double>( "din" );
double tauA = 1.0;
double tauB = 1.0;
double rhoA = 1.0;
double rhoB = 1.0;
double alpha = 0.005;
double beta = 0.95;
double tau = 1.0;
double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
// Read domain values
auto size = domain_db->getVector<int>( "n" );
auto nproc = domain_db->getVector<int>( "nproc" );
int BoundaryCondition = domain_db->getScalar<int>( "BC" );
int Nx = size[0];
int Ny = size[1];
int Nz = size[2];
int nprocx = nproc[0];
int nprocy = nproc[1];
int nprocz = nproc[2];
Fx = Fy = 0.f;
Fz = 0.f;
int timestep = 6;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=3;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=2; nprocy=1;
nprocz=1;
Nx=Ny=Nz=dim;
Nx = dim; Ny = dim; Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
MPI_Barrier(comm);
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
MPI_Barrier(comm);
if (rank==0){
printf("********************************************************\n");
printf("tau (non-wetting) = %f \n", tauA);
printf("tau (wetting) = %f \n", tauB);
printf("density (non-wetting) = %f \n", rhoA);
printf("density (wetting) = %f \n", rhoB);
printf("alpha = %f \n", alpha);
printf("beta = %f \n", beta);
printf("gamma_{wn} = %f \n", 5.796*alpha);
printf("Force(x) = %f \n", Fx);
printf("Force(y) = %f \n", Fy);
printf("Force(z) = %f \n", Fz);
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz);
if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n");
if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n");
if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n");
if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n");
if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n");
if (!Restart) printf("Initial conditions assigned from phase ID file \n");
if (Restart) printf("Initial conditions assigned from restart file \n");
printf("********************************************************\n");
}
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
// Initialized domain and averaging framework for Two-Phase Flow
bool pBC;
if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4)
pBC=true;
else
pBC=false;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
// Full domain used for averaging (do not use mask for analysis)
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
for (int i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1;
Dm->CommInit();
MPI_Barrier(comm);
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
Nx+=2; Ny+=2; Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
int Np=0; // number of local pore nodes
double *PhaseLabel;
@ -166,7 +137,7 @@ int main(int argc, char **argv)
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n]=0;
Dm->id[n]=0;
}
}
}
@ -174,21 +145,21 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n]=1;
Dm->id[n]=1;
Np++;
// constant color
PhaseLabel[n]= -1.0;
}
}
}
Dm.CommInit(comm);
Dm->CommInit();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular(Dm);
ScaLBL_Communicator ScaLBL_Comm(Dm);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm));
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Dm));
// LBM variables
if (rank==0) printf ("Set up the neighborlist \n");
@ -196,29 +167,26 @@ int main(int argc, char **argv)
int neighborSize=18*Np*sizeof(int);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
Npad=Np+32;
neighborList= new int[18*Npad];
Np=ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
if (rank==0) printf ("Allocating distributions \n");
int *NeighborList;
int *dvcMap;
// double *f_even,*f_odd;
double *fq, *Aq, *Bq;
double *Den, *Phi;
double *ColorGrad;
double *Gradient;
double *SolidPotential;
double *Vel;
double *Pressure;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size);
@ -226,7 +194,8 @@ int main(int argc, char **argv)
ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &SolidPotential, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
@ -276,33 +245,35 @@ int main(int argc, char **argv)
}
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
if (rank==0) printf ("Initializing distributions \n");
// Initialize the phase field and variables
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np);
if (rank==0) printf ("Initializing distributions \n");
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np);
// *************ODD TIMESTEP*************
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
printf("Check after odd time \n");
@ -317,7 +288,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
nA=DENSITY[idx];
nB=DENSITY[Np+idx];
@ -341,29 +312,29 @@ int main(int argc, char **argv)
}
}
}
// EVEN TIMESTEP
// Compute the Phase indicator field
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// *************EVEN TIMESTEP*************
// Compute the Phase indicator field
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************
printf("Check after even time \n");
SIZE=2*Np*sizeof(double);
@ -377,7 +348,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
nA=DENSITY[idx];
nB=DENSITY[Np+idx];
@ -415,36 +386,39 @@ int main(int argc, char **argv)
}
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
if (rank==0) printf ("Initializing distributions \n");
// Initialize the phase field and variables
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np);
if (rank==0) printf ("Initializing distributions \n");
ScaLBL_D3Q19_Init(fq, Np);
if (rank==0) printf ("Initializing phase field \n");
ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np);
// *************ODD TIMESTEP*************
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
printf("Check after odd time \n");
SIZE=2*Np*sizeof(double);
ScaLBL_CopyToHost(&DENSITY[0],&Den[0],SIZE);
@ -457,7 +431,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
nA=DENSITY[idx];
nB=DENSITY[Np+idx];
@ -482,28 +456,29 @@ int main(int argc, char **argv)
}
}
// EVEN TIMESTEP
// Compute the Phase indicator field
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// compute the gradient
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->SendHalo(Phi);
ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->RecvGrad(Phi,Gradient);
// *************EVEN TIMESTEP*************
// Compute the Phase indicator field
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************
printf("Check after even time \n");
SIZE=2*Np*sizeof(double);
@ -517,7 +492,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
nA=DENSITY[idx];
nB=DENSITY[Np+idx];
@ -551,7 +526,6 @@ int main(int argc, char **argv)
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
return check;
}

583
tests/TestColorSquareTube.cpp
View File

@ -8,9 +8,78 @@
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "models/ColorModel.h"
using namespace std;
std::shared_ptr<Database> loadInputs( int nprocs )
{
auto db = std::make_shared<Database>( "Domain.in" );
const int dim = 50;
db->putScalar<int>( "BC", 0 );
if ( nprocs == 1 ){
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 3, 1, 1 } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if ( nprocs == 2 ) {
db->putVector<int>( "nproc", { 2, 1, 1 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if ( nprocs == 4 ) {
db->putVector<int>( "nproc", { 2, 2, 1 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if (nprocs==8){
db->putVector<int>( "nproc", { 2, 2, 2 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
}
return db;
}
void InitializeSquareTube(ScaLBL_ColorModel &ColorModel){
int i,j,k,n;
int rank = ColorModel.Mask->rank();
int Nx = ColorModel.Mask->Nx;
int Ny = ColorModel.Mask->Ny;
int Nz = ColorModel.Mask->Nz;
int nprocx = ColorModel.Mask->rank_info.nx;
int nprocy = ColorModel.Mask->rank_info.ny;
int iproc = ColorModel.Mask->rank_info.ix;
int jproc = ColorModel.Mask->rank_info.jy;
int kproc = ColorModel.Mask->rank_info.kz;
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nx + i;
ColorModel.Mask->id[n]=0;
}
}
}
printf("rank=%i, %i,%i,%i \n",rank,iproc,jproc,kproc);
// Initialize a square tube
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny + j*Nx + i;
int iglobal= i+(Nx-2)*iproc;
int jglobal= j+(Ny-2)*jproc;
int kglobal= k+(Nz-2)*kproc;
// Initialize phase position field for parallel bubble test
if (iglobal < 2) ColorModel.Mask->id[n]=0;
else if (iglobal > (Nx-2)*nprocx-2) ColorModel.Mask->id[n]=0;
else if (jglobal < 2) ColorModel.Mask->id[n]=0;
else if (jglobal > (Ny-2)*nprocy-2) ColorModel.Mask->id[n]=0;
else if (kglobal < 20) ColorModel.Mask->id[n]=1;
else ColorModel.Mask->id[n]=2;
}
}
}
}
//***************************************************************************************
int main(int argc, char **argv)
@ -24,513 +93,23 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Color Model: TestColor \n");
printf("********************************************************\n");
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 50;
//if (rank == 0) printf("dim=%d\n",dim);
int timestep = 1;
int timesteps = 100;
int centralNode = 2;
double tauA = 1.0;
double tauB = 1.0;
double rhoA = 1.0;
double rhoB = 1.0;
double alpha = 0.001;
double beta = 0.95;
double tau = 1.0;
double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
Fx = Fy = 0.f;
Fz = 0.f;
int typeBC;
double din, dout, flux;
double inletA,inletB,outletA,outletB;
inletA=1.f;
inletB=0.f;
outletA=0.f;
outletB=1.f;
typeBC=4;
flux = 10.f;
dout=1.f;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=3; Ny = 1;
Nz = 1;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=2; nprocy=1;
nprocz=1;
Nx=Ny=Nz=dim;
Nx = dim; Ny = dim; Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
MPI_Barrier(comm);
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
// Assign the phase ID field
//.......................................................................
char LocalRankString[8];
sprintf(LocalRankString,"%05d",rank);
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rank);
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nx + i;
Dm.id[n]=0;
}
}
}
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocx*jproc;
printf("rank=%i, %i,%i,%i \n",rank,iproc,jproc,kproc);
// Initialize a square tube
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny + j*Nx + i;
int iglobal= i+(Nx-2)*iproc;
int jglobal= j+(Ny-2)*jproc;
int kglobal= k+(Nz-2)*kproc;
// Initialize phase position field for parallel bubble test
if (iglobal < 2) Dm.id[n]=0;
else if (iglobal > (Nx-2)*nprocx-2) Dm.id[n]=0;
else if (jglobal < 2) Dm.id[n]=0;
else if (jglobal > (Ny-2)*nprocy-2) Dm.id[n]=0;
else if (kglobal < 20) Dm.id[n]=1;
else Dm.id[n]=2;
}
}
}
Dm.CommInit(comm);
//.......................................................................
// Compute the media porosity, assign phase labels and solid composition
//.......................................................................
double sum;
double sum_local=0.0, porosity;
int Np=0; // number of local pore nodes
double *PhaseLabel;
PhaseLabel = new double[N];
Dm.AssignComponentLabels(PhaseLabel);
//.......................................................................
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
sum_local+=1.0;
Np++;
}
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
if (rank==0) printf ("Create ScaLBL_Communicator \n");
MPI_Barrier(comm);
// Create a communicator for the device (will use optimized layout)
ScaLBL_Communicator ScaLBL_Comm(Dm);
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular(Dm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm.id, N);
//...........................................................................
if (rank==0){
printf("Total domain size = %i \n",N);
printf("Reduced domain size = %i \n",Np);
}
// LBM variables
if (rank==0) printf ("Set up the neighborlist \n");
int neighborSize=18*Np*sizeof(int);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
if (rank==0) printf ("Allocating distributions \n");
int *NeighborList;
int *dvcMap;
// double *f_even,*f_odd;
double *fq, *Aq, *Bq;
double *Den, *Phi;
double *ColorGrad;
double *Vel;
double *Pressure;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np);
ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np);
//...........................................................................
// Update GPU data structures
if (rank==0) printf ("Setting up device map and neighbor list \n");
int *TmpMap;
TmpMap=new int[Np];
for (k=1; k<Nz-1; k++){
for (j=1; j<Ny-1; j++){
for (i=1; i<Nx-1; i++){
int idx=Map(i,j,k);
if (!(idx < 0))
TmpMap[idx] = k*Nx*Ny+j*Nx+i;
}
}
}
//for (int idx=0; idx<Np; idx++) printf("Map=%i\n",TmpMap[idx]);
ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
ScaLBL_DeviceBarrier();
delete [] TmpMap;
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
// initialize phi based on PhaseLabel (include solid component labels)
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
//...........................................................................
if (rank==0) printf ("Initializing distributions \n");
// Initialize the phase field and variables
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np);
if (Dm.kproc==0){
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2);
}
if (Dm.kproc == nprocz-1){
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3);
}
//************ MAIN ITERATION LOOP (timing communications)***************************************
if (rank==0) printf("Beginning AA timesteps...\n");
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps for timing: %i \n", timesteps);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
//timesteps=20;
//timestep=1;
while (timestep < timesteps) {
// *************ODD TIMESTEP*************
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set BCs
if (typeBC > 0){
ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (typeBC == 3){
ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
if (typeBC == 4){
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
// *************EVEN TIMESTEP*************
// Compute the Phase indicator field
ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np);
// Halo exchange for phase field
ScaLBL_Comm_Regular.SendHalo(Phi);
ScaLBL_Comm_Regular.RecvHalo(Phi);
// Perform the collision operation
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
// Set boundary conditions
if (typeBC > 0){
ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
if (typeBC == 3){
ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (typeBC == 4){
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************
}
//************************************************************************
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
double MLUPS = double(Np*timestep)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n");
// Number of memory references for color model
double MemoryRefs = double(Np)*(77*8+(9+7+7)*4); // extra memory refs to read from neighborlist (every other timestep)
// number of memory references for the swap algorithm - GigaBytes / second
if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*timestep/1e9/cputime);
// Report bandwidth in Gigabits per second
// communication bandwidth includes both send and recieve
if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
double *VEL;
VEL= new double [3*Np];
int SIZE=3*Np*sizeof(double);
ScaLBL_D3Q19_Momentum(fq,Vel,Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_CopyToHost(&VEL[0],&Vel[0],SIZE);
sum_local=0.f;
sum = 0.f;
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
int idx = Map(i,j,k);
sum_local+=VEL[2*Np+idx];
}
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
double PoreVel = sum*iVol_global;
if (rank==0) printf("Average velocity = %f \n",PoreVel);
if (rank==0){
printf("Printing inlet velocity for rank=0 \n");
k=1;
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
int idx = Map(i,j,k);
double vz = VEL[2*Np+idx];
printf("%f ",vz);
}
}
printf("\n");
}
}
/*
double *PHASE;
PHASE= new double [Nx*Ny*Nz];
SIZE=Nx*Ny*Nz*sizeof(double);
ScaLBL_CopyToHost(&PHASE[0],&Phi[0],SIZE);
FILE *OUTFILE;
sprintf(LocalRankFilename,"Phase.%05i.raw",rank);
OUTFILE = fopen(LocalRankFilename,"wb");
fwrite(PHASE,8,N,OUTFILE);
fclose(OUTFILE);
double *DENA, *DENB, *TMPDAT;
SIZE=Np*sizeof(double);
TMPDAT = new double [Np];
DENA= new double [Nx*Ny*Nz];
DENB= new double [Nx*Ny*Nz];
ScaLBL_CopyToHost(&TMPDAT[0],&Den[0],SIZE);
ScaLBL_Comm.RegularLayout(Map,TMPDAT,DENA);
ScaLBL_CopyToHost(&TMPDAT[0],&Den[Np],SIZE);
ScaLBL_Comm.RegularLayout(Map,TMPDAT,DENB);
FILE *AFILE;
sprintf(LocalRankFilename,"na.%05i.raw",rank);
AFILE = fopen(LocalRankFilename,"wb");
fwrite(DENA,8,N,AFILE);
fclose(AFILE);
FILE *BFILE;
sprintf(LocalRankFilename,"nb.%05i.raw",rank);
BFILE = fopen(LocalRankFilename,"wb");
fwrite(DENB,8,N,BFILE);
fclose(BFILE);
double *CG;
CG= new double [3*Np];
ScaLBL_D3Q19_Gradient(dvcMap, Phi, ColorGrad, 0, Np, Np, Nx, Ny, Nz);
ScaLBL_CopyToHost(&CG[0],&ColorGrad[0],3*SIZE);
for (int idx=0; idx<Np; idx++){
double C=CG[idx]*CG[idx]+CG[Np+idx]*CG[Np+idx]+CG[2*Np+idx]*CG[2*Np+idx];
TMPDAT[idx]=C;
}
ScaLBL_Comm.RegularLayout(Map,TMPDAT,DENB);
FILE *CGFILE;
sprintf(LocalRankFilename,"cgrad.%05i.raw",rank);
CGFILE = fopen(LocalRankFilename,"wb");
fwrite(DENB,8,N,CGFILE);
fclose(CGFILE);
*/
auto filename = argv[1];
ScaLBL_ColorModel ColorModel(rank,nprocs,comm);
ColorModel.ReadParams(filename);
ColorModel.SetDomain();
//ColorModel.ReadInput();
InitializeSquareTube(ColorModel);
ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables
ColorModel.Initialize(); // initializing the model will set initial conditions for variables
ColorModel.Run();
ColorModel.WriteDebug();
}
// ****************************************************

122
tests/TestCommD3Q19.cpp
View File

@ -11,6 +11,36 @@
using namespace std;
std::shared_ptr<Database> loadInputs( int nprocs )
{
auto db = std::make_shared<Database>( "Domain.in" );
const int dim = 50;
db->putScalar<int>( "BC", 0 );
if ( nprocs == 1 ){
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 3, 1, 1 } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if ( nprocs == 2 ) {
db->putVector<int>( "nproc", { 2, 1, 1 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if ( nprocs == 4 ) {
db->putVector<int>( "nproc", { 2, 2, 1 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
} else if (nprocs==8){
db->putVector<int>( "nproc", { 2, 2, 2 } );
db->putVector<int>( "n", { dim, dim, dim } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
}
return db;
}
extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int Nx, int Ny, int Nz,
int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
{
@ -176,88 +206,13 @@ int main(int argc, char **argv)
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=50;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=nprocy=1;
nprocz=2;
Nx=Ny=Nz=50;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=50;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=50;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nBlocks,1,MPI_UNSIGNED,0,comm);
MPI_Bcast(&nthreads,1,MPI_UNSIGNED,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
if (rank==0){
printf("********************************************************\n");
@ -272,8 +227,9 @@ int main(int argc, char **argv)
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
Domain Dm(db);
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
@ -315,7 +271,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
//.......................................................................
// Compute the media porosity

90
tests/TestFluxBC.cpp
View File

@ -3,6 +3,17 @@
#include "common/Utilities.h"
#include "common/ScaLBL.h"
std::shared_ptr<Database> loadInputs( int nprocs )
{
//auto db = std::make_shared<Database>( "Domain.in" );
auto db = std::make_shared<Database>();
db->putScalar<int>( "BC", 0 );
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 16, 16, 16 } );
db->putScalar<int>( "nspheres", 1 );
db->putVector<double>( "L", { 1, 1, 1 } );
return db;
}
int main (int argc, char **argv)
{
@ -20,22 +31,25 @@ int main (int argc, char **argv)
ASSERT(nprocs==1);
}
{
int i,j,k,n;
int Nx,Ny,Nz,Np;
int i,j,k,n,Np;
bool pBC=true;
int nprocx,nprocy,nprocz;
double Lx,Ly,Lz;
Nx = Ny = Nz = 16;
Lx = Ly = Lz = 1.f;
nprocx=nprocy=nprocz=1;
double din,dout;
int BC=1;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Nz += 2;
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
std::shared_ptr<Domain> Dm(new Domain(db,comm));
Nx += 2; Ny+=2; Nz += 2;
Nx = Ny = Nz; // Cubic domain
int N = Nx*Ny*Nz;
//.......................................................................
@ -67,25 +81,13 @@ int main (int argc, char **argv)
id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
//.........................................................
// Initialize communication structures in averaging domain
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
Dm.CommInit(comm);
Np=0; // number of local pore nodes
//.......................................................................
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
Np++;
}
}
}
}
//...........................................................................
for (i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = id[i];
Dm->CommInit();
Np=Dm->PoreCount();
//................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
ScaLBL_Communicator ScaLBL_Comm(Dm);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Dm));
if (rank==0) printf ("Set up the neighborlist \n");
int Npad=Np+32;
@ -94,7 +96,7 @@ int main (int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
@ -142,7 +144,7 @@ int main (int argc, char **argv)
double flux = 1.0;
int timestep=0;
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
printf("Computed pressure for flux = %f\n",din);
@ -165,7 +167,7 @@ int main (int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
Q += VEL[2*Np+idx];
}
@ -194,21 +196,21 @@ int main (int argc, char **argv)
while (timestep < 2000) {
ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************/
@ -226,7 +228,7 @@ int main (int argc, char **argv)
for (k=1;k<Nz-1;k++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
double vz = VEL[2*Np+idx];
printf("%f ",vz);
@ -242,7 +244,7 @@ int main (int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
double vz = VEL[2*Np+idx];
printf("%f ",vz);
@ -256,7 +258,7 @@ int main (int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
int idx = Map(i,j,k);
Q += VEL[2*Np+idx];
}

181
tests/TestForceMoments.cpp
View File

@ -11,11 +11,8 @@
using namespace std;
extern void PrintNeighborList(int * neighborList, int Np, int rank) {
if (rank == 0) {
int n;
int neighbor;
for (int i = 0; i < Np; i++) {
@ -30,6 +27,22 @@ extern void PrintNeighborList(int * neighborList, int Np, int rank) {
}
}
std::shared_ptr<Database> loadInputs( int nprocs )
{
auto db = std::make_shared<Database>( "Domain.in" );
db->putScalar<int>( "BC", 0 );
if ( nprocs == 1 ){
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 3, 3, 3 } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
}
return db;
}
//***************************************************************************************
int main(int argc, char **argv)
{
@ -42,13 +55,11 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Unit Test: TestForceMoments \n");
@ -56,79 +67,31 @@ int main(int argc, char **argv)
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
double tau,Fx,Fy,Fz;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 3; if (rank == 0) printf("dim=%d\n",dim);
int timestep = 0;
int timesteps = 2;
tau =1.0;
double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
Fx = Fy = 1.0;
Fz = 1.0;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
printf("domain.good == true \n");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=3; Ny = 3;
Nz = 3;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
// Load inputs
string FILENAME = argv[1];
// Load inputs
if (rank==0) printf("Loading input database \n");
auto db = std::make_shared<Database>( FILENAME );
auto domain_db = db->getDatabase( "Domain" );
int Nx = domain_db->getVector<int>( "n" )[0];
int Ny = domain_db->getVector<int>( "n" )[1];
int Nz = domain_db->getVector<int>( "n" )[2];
int nprocx = domain_db->getVector<int>( "nproc" )[0];
int nprocy = domain_db->getVector<int>( "nproc" )[1];
int nprocz = domain_db->getVector<int>( "nproc" )[2];
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
@ -149,10 +112,7 @@ int main(int argc, char **argv)
printf("\n\n");
}
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
Nx += 2;
Ny += 2;
@ -169,7 +129,7 @@ int main(int argc, char **argv)
/*
FILE *IDFILE = fopen(LocalRankFilename,"rb");
if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx");
fread(Dm.id,1,N,IDFILE);
fread(Dm->id,1,N,IDFILE);
fclose(IDFILE);
*/
@ -178,11 +138,11 @@ int main(int argc, char **argv)
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n]=1;
Dm->id[n]=1;
}
}
}
Dm.CommInit(comm);
Dm->CommInit();
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
@ -191,7 +151,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
if (Dm->id[n] > 0){
Np++;
}
}
@ -199,16 +159,14 @@ int main(int argc, char **argv)
}
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
ScaLBL_Communicator ScaLBL_Comm(Dm);
auto ScaLBL_Comm = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Dm));
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm.id, N);
ScaLBL_CopyToDevice(ID, Dm->id, N);
//...........................................................................
if (rank==0){
@ -225,7 +183,7 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
if (rank == 0) PrintNeighborList(neighborList,Np, rank);
@ -259,8 +217,8 @@ int main(int argc, char **argv)
starttime = MPI_Wtime();
/************ MAIN ITERATION LOOP (timing communications)***************************************/
//ScaLBL_Comm.SendD3Q19(dist, &dist[10*Np]);
//ScaLBL_Comm.RecvD3Q19(dist, &dist[10*Np]);
//ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]);
//ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
if (rank==0) printf("Beginning AA timesteps...\n");
@ -269,16 +227,20 @@ int main(int argc, char **argv)
while (timestep < 2) {
ScaLBL_D3Q19_AAeven_MRT(dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_Comm->SendD3Q19AA(dist); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************/
timestep++;
@ -297,43 +259,8 @@ int main(int argc, char **argv)
MLUPS *= nprocs;
if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n");
// Number of memory references from the swap algorithm (per timestep)
// 18 reads and 18 writes for each lattice site
double MemoryRefs = Np*38;
int SIZE=Np*sizeof(double);
/*
double *Vz;
Vz= new double [Np];
double *Vx;
Vx= new double [Np];
double *Vy;
Vy= new double [Np];
ScaLBL_D3Q19_AA_Velocity(dist, &dist[10*Np],Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_CopyToHost(&Vx[0],&Velocity[0],SIZE);
ScaLBL_CopyToHost(&Vy[0],&Velocity[Np],SIZE);
ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE);
printf("Force: %f,%f,%f \n",Fx,Fy,Fz);
double vz;
double W = 1.f*Nx-4;
j=Ny/2; k=Nz/2;
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
//printf("%i ",Dm.id[n]);
n = Map(i,j,k);
//printf("%i,%i,%i; %i :",i,j,k,n);
if (n<0) vz =0.f;
else vz=Vz[n];
printf("%f ",vz);
}
printf("\n");
}
*/
int SIZE=Np*sizeof(double);
double *DIST;
DIST= new double [19*Np];
ScaLBL_CopyToHost(&DIST[0],&dist[0],19*SIZE);
@ -348,7 +275,7 @@ int main(int argc, char **argv)
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
n = k*Nx*Ny+j*Nx+i;
//printf("%i ",Dm.id[n]);
//printf("%i ",Dm->id[n]);
n = Map(i,j,k);
double fa = DIST[(2*q+1)*Np+n];
double fb = DIST[2*(q+1)*Np+n];
@ -369,7 +296,7 @@ int main(int argc, char **argv)
for (k=1;k<Nz-1;k++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
//printf("%i ",Dm.id[n]);
//printf("%i ",Dm->id[n]);
n = Map(i,j,k);
double fa = DIST[(2*q+1)*Np+n];
double fb = DIST[2*(q+1)*Np+n];
@ -389,7 +316,7 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
//printf("%i ",Dm.id[n]);
//printf("%i ",Dm->id[n]);
n = Map(i,j,k);
double fa = DIST[(2*q+1)*Np+n];
double fb = DIST[2*(q+1)*Np+n];

89
tests/TestInterfaceSpeed.cpp
View File

@ -24,28 +24,29 @@ int main (int argc, char *argv[])
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int npx,npy,npz;
int i,j,k,n;
int Nx,Ny,Nz;
double Lx,Ly,Lz;
Nx=Ny=Nz=N;
npx=npy=npz=1;
Lx=Ly=Lz=1.0;
int BC=0; // periodic boundary condition
Domain Dm(Nx,Ny,Nz,rank,npx,npy,npz,Lx,Ly,Lz,BC);
// Load inputs
string FILENAME = argv[1];
// Load inputs
if (rank==0) printf("Loading input database \n");
auto db = std::make_shared<Database>( FILENAME );
auto domain_db = db->getDatabase( "Domain" );
int Nx = domain_db->getVector<int>( "n" )[0];
int Ny = domain_db->getVector<int>( "n" )[1];
int Nz = domain_db->getVector<int>( "n" )[2];
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
Dm.CommInit(comm);
Nx+=2; Ny+=2; Nz+=2;
TwoPhase Averages(Dm);
for (i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages(new TwoPhase(Dm));
int timestep=0;
Nx = Dm.Nx;
Ny = Dm.Ny;
Nz = Dm.Nz;
double Cx,Cy,Cz;
double dist1,dist2;
@ -56,7 +57,7 @@ int main (int argc, char *argv[])
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
dist2 = fabs(Cz-k)-HEIGHT;
Averages.Phase_tminus(i,j,k) = dist2;
Averages->Phase_tminus(i,j,k) = dist2;
}
}
}
@ -69,9 +70,9 @@ int main (int argc, char *argv[])
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
dist2 = fabs(Cz-k)-HEIGHT;
Averages.SDs(i,j,k) = -dist1;
Averages.Phase(i,j,k) = dist2;
Averages.SDn(i,j,k) = dist2;
Averages->SDs(i,j,k) = -dist1;
Averages->Phase(i,j,k) = dist2;
Averages->SDn(i,j,k) = dist2;
}
}
}
@ -82,7 +83,7 @@ int main (int argc, char *argv[])
dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
dist2 = fabs(Cz-k)-HEIGHT;
Averages.Phase_tplus(i,j,k) = dist2;
Averages->Phase_tplus(i,j,k) = dist2;
}
}
}
@ -91,58 +92,58 @@ int main (int argc, char *argv[])
//....................................................................
// The following only need to be done once
//Averages.SetupCubes(Dm);
Averages.UpdateSolid(); // unless the solid is deformable!
//Averages->SetupCubes(Dm);
Averages->UpdateSolid(); // unless the solid is deformable!
//....................................................................
// The following need to be called each time new averages are computed
Averages.Initialize();
Averages.UpdateMeshValues();
Averages.ComputeLocal();
Averages.Reduce();
Averages.PrintAll(timestep);
Averages->Initialize();
Averages->UpdateMeshValues();
Averages->ComputeLocal();
Averages->Reduce();
Averages->PrintAll(timestep);
//....................................................................
printf("-------------------------------- \n");
printf("NWP volume = %f \n", Averages.nwp_volume);
printf("Area wn = %f, Analytical = %f \n", Averages.awn,2*PI*RADIUS*RADIUS);
printf("Area ns = %f, Analytical = %f \n", Averages.ans, 2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT);
printf("Area ws = %f, Analytical = %f \n", Averages.aws, 4*PI*RADIUS*HEIGHT);
printf("Area s = %f, Analytical = %f \n", Averages.As, 2*PI*RADIUS*(Nz-2));
printf("Length wns = %f, Analytical = %f \n", Averages.lwns, 4*PI*RADIUS);
printf("Geodesic curvature (wns) = %f, Analytical = %f \n", Averages.KGwns_global, 0.0);
printf("Normal curvature (wns) = %f, Analytical = %f \n", Averages.KNwns_global, 1.0/RADIUS);
printf("NWP volume = %f \n", Averages->nwp_volume);
printf("Area wn = %f, Analytical = %f \n", Averages->awn,2*PI*RADIUS*RADIUS);
printf("Area ns = %f, Analytical = %f \n", Averages->ans, 2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT);
printf("Area ws = %f, Analytical = %f \n", Averages->aws, 4*PI*RADIUS*HEIGHT);
printf("Area s = %f, Analytical = %f \n", Averages->As, 2*PI*RADIUS*(Nz-2));
printf("Length wns = %f, Analytical = %f \n", Averages->lwns, 4*PI*RADIUS);
printf("Geodesic curvature (wns) = %f, Analytical = %f \n", Averages->KGwns_global, 0.0);
printf("Normal curvature (wns) = %f, Analytical = %f \n", Averages->KNwns_global, 1.0/RADIUS);
// printf("Cos(theta_wns) = %f, Analytical = %f \n",efawns/lwns,1.0*RADIUS/CAPRAD);
printf("Interface Velocity = %f,%f,%f \n",Averages.vawn_global(0),Averages.vawn_global(1),Averages.vawn_global(2));
printf("Common Curve Velocity = %f,%f,%f \n",Averages.vawns_global(0),Averages.vawns_global(1),Averages.vawns_global(2));
printf("Interface Velocity = %f,%f,%f \n",Averages->vawn_global(0),Averages->vawn_global(1),Averages->vawn_global(2));
printf("Common Curve Velocity = %f,%f,%f \n",Averages->vawns_global(0),Averages->vawns_global(1),Averages->vawns_global(2));
printf("-------------------------------- \n");
//.........................................................................
int toReturn = 0;
if (fabs(Averages.awn - 2*PI*RADIUS*RADIUS)/(2*PI*RADIUS*RADIUS) > 0.02){
if (fabs(Averages->awn - 2*PI*RADIUS*RADIUS)/(2*PI*RADIUS*RADIUS) > 0.02){
toReturn = 1;
printf("TestCylinderArea.cpp: error tolerance exceeded for wn area \n");
}
if (fabs(Averages.ans - (2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT))/(2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT)> 0.02 ){
if (fabs(Averages->ans - (2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT))/(2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT)> 0.02 ){
toReturn = 2;
printf("TestCylinderArea.cpp: error tolerance exceeded for ns area \n");
}
if (fabs(Averages.aws - 4*PI*RADIUS*HEIGHT)/(4*PI*RADIUS*HEIGHT) > 0.02 ){
if (fabs(Averages->aws - 4*PI*RADIUS*HEIGHT)/(4*PI*RADIUS*HEIGHT) > 0.02 ){
toReturn = 3;
printf("TestCylinderArea.cpp: error tolerance exceeded for ws area \n");
}
if (fabs(Averages.As - 2*PI*RADIUS*(Nz-2))/(2*PI*RADIUS*(Nz-2)) > 0.02 ){
if (fabs(Averages->As - 2*PI*RADIUS*(Nz-2))/(2*PI*RADIUS*(Nz-2)) > 0.02 ){
toReturn = 4;
printf("TestCylinderArea.cpp: error tolerance exceeded for solid area \n");
}
if (fabs(Averages.lwns - 4*PI*RADIUS)/(4*PI*RADIUS) > 0.02 ){
if (fabs(Averages->lwns - 4*PI*RADIUS)/(4*PI*RADIUS) > 0.02 ){
toReturn = 5;
printf("TestCylinderArea.cpp: error tolerance exceeded for common curve length \n");
}
if ( fabs(Averages.vawn_global(2)+0.2) > 0.01){
if ( fabs(Averages->vawn_global(2)+0.25) > 0.01){
printf("TestInterfaceSpeed: Error too high for kinematic velocity of wn interface \n");
toReturn = 6;
}
if ( fabs(Averages.vawns_global(2)+0.2) > 0.01){
if ( fabs(Averages->vawns_global(2)+0.25) > 0.01){
printf("TestInterfaceSpeed: Error too high for kinematic velocity of common curve \n");
toReturn = 7;
}

2
tests/TestMRT.cpp
View File

@ -651,7 +651,7 @@ int main(int argc, char **argv)
fclose(IDFILE);
MPI_Barrier(comm);
Dm.CommInit(comm);
Dm.CommInit();
//.......................................................................
// Compute the media porosity

159
tests/TestMap.cpp
View File

@ -11,6 +11,18 @@
using namespace std;
std::shared_ptr<Database> loadInputs( int nprocs )
{
//auto db = std::make_shared<Database>( "Domain.in" );
auto db = std::make_shared<Database>();
db->putScalar<int>( "BC", 0 );
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 5, 5, 5 } );
db->putScalar<int>( "nspheres", 1 );
db->putVector<double>( "L", { 1, 1, 1 } );
return db;
}
//***************************************************************************************
int main(int argc, char **argv)
{
@ -23,142 +35,54 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Color Model: TestColor \n");
printf("********************************************************\n");
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim=5;
static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=5;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
nprocx=2; nprocy=1;
nprocz=1;
Nx=Ny=Nz=dim;
Nx = dim; Ny = dim; Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==4){
nprocx=nprocy=2;
nprocz=1;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==8){
nprocx=nprocy=nprocz=2;
Nx=Ny=Nz=dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
if (rank == 0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("Running unit test: TestMap \n");
printf("********************************************************\n");
}
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
MPI_Barrier(comm);
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
std::shared_ptr<Domain> Dm(new Domain(db,comm));
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
int Np = 0;
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
Dm->id[n] = 1;
Np++;
}
}
}
Dm.CommInit(comm);
Dm->CommInit();
// Create a communicator for the device (will use optimized layout)
ScaLBL_Communicator ScaLBL_Comm(Dm);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Dm));
//Create a second communicator based on the regular data layout
ScaLBL_Communicator ScaLBL_Comm_Regular(Dm);
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm));
if (rank==0){
printf("Total domain size = %i \n",N);
@ -173,16 +97,29 @@ int main(int argc, char **argv)
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
// Check the neighborlist
printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior());
for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
for (int q=0; q<18; q++){
int nn = neighborList[q*Np+idx]%Np;
if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx);
}
}
for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
for (int q=0; q<18; q++){
int nn = neighborList[q*Np+idx]%Np;
if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx);
}
}
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
if (rank==0) printf ("Allocating distributions \n");
int *NeighborList;
int *dvcMap;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Npad);
@ -236,7 +173,7 @@ int main(int argc, char **argv)
// Loop over the distributions for interior lattice sites
if (rank==0) printf ("Loop over distributions \n");
for (int idx=ScaLBL_Comm.first_interior; idx<ScaLBL_Comm.last_interior; idx++){
for (int idx=ScaLBL_Comm->first_interior; idx<ScaLBL_Comm->last_interior; idx++){
n = TmpMap[idx];
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;

339
tests/TestPoiseuille.cpp
View File

@ -8,8 +8,25 @@
#include <fstream>
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
#include "models/MRTModel.h"
using namespace std;
void ParallelPlates(ScaLBL_MRTModel &MRT){
// initialize empty domain
int i,j,k,n;
int Nx = MRT.Nx;
int Ny = MRT.Ny;
int Nz = MRT.Nz;
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
if (i<2) MRT.Mask->id[n] = 0;
else if (i>Nx-3) MRT.Mask->id[n] = 0;
else MRT.Mask->id[n]=1;
}
}
}
}
//***************************************************************************************
int main(int argc, char **argv)
@ -23,312 +40,44 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Unit Test: TestPoiseuille \n");
printf("********************************************************\n");
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 12; if (rank == 0) printf("dim=%d\n",dim);
int timestep = 0;
tau = 1.0;
double mu=(tau-0.5)/3.0;
double rlx_setA=1.0/tau;
double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
Fx = 0; Fy = 0;
Fz = 1e-3; //1.f; // 1e-3;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
printf("domain.good == true \n");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
printf("domain.good == false - using predefined parameters \n");
nprocx=nprocy=nprocz=1;
Nx=dim; Ny = dim;
Nz = dim;
nspheres=0;
Lx=Ly=Lz=1;
}
else if (nprocs==2){
printf("domain.good == false - using predefined parameters \n");
nprocx=2;
nprocy=nprocz=1;
Nx=0.5*dim; Ny = 0.5*dim;
Nz = 0.5*dim;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
if (rank == 0) {
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
}
MPI_Barrier(comm);
if (rank == 1){
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
printf("\n\n");
}
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
// Assign the phase ID field
//.......................................................................
char LocalRankString[8];
sprintf(LocalRankString,"%05d",rank);
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rank);
/*
FILE *IDFILE = fopen(LocalRankFilename,"rb");
if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx");
fread(Dm.id,1,N,IDFILE);
fclose(IDFILE);
*/
// initialize empty domain
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
if (i<2) Dm.id[n] = 0;
else if (i>Nx-3) Dm.id[n] = 0;
else Dm.id[n]=1;
}
}
}
Dm.CommInit(comm);
MPI_Barrier(comm);
//.......................................................................
// Compute the media porosity
//.......................................................................
double sum;
double sum_local=0.0, porosity;
int Np=0; // number of local pore nodes
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
if (Dm.id[n] > 0){
sum_local+=1.0;
Np++;
}
}
}
}
MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = sum*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
ScaLBL_Communicator ScaLBL_Comm(Dm);
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm.id, N);
//...........................................................................
if (rank==0){
printf("Total domain size = %i \n",N);
printf("Reduced domain size = %i \n",Np);
}
// LBM variables
if (rank==0) printf ("Allocating distributions \n");
if (rank==0) printf ("Set up the neighborlist \n");
int Npad=Np+32;
int neighborSize=18*Npad*sizeof(int);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Npad];
Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
int dist_mem_size = Np*sizeof(double);
int *NeighborList;
// double *f_even,*f_odd;
double * dist;
double * Velocity;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &dist, 19*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
//...........................................................................
/*
* AA Algorithm begins here
*
*/
ScaLBL_D3Q19_Init(dist, Np);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
starttime = MPI_Wtime();
/************ MAIN ITERATION LOOP (timing communications)***************************************/
// ScaLBL_Comm.SendD3Q19(dist, &dist[10*Np]);
// ScaLBL_Comm.RecvD3Q19(dist, &dist[10*Np]);
// ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
//
if (rank==0) printf("Beginning AA timesteps...\n");
if (rank==0) printf("********************************************************\n");
while (timestep < 2000) {
ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL
ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
timestep++;
//************************************************************************/
}
//************************************************************************/
stoptime = MPI_Wtime();
// cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
cputime = stoptime - starttime;
// cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl;
double MLUPS = double(Np*timestep)/cputime/1000000;
// if (rank==0) printf("********************************************************\n");
// if (rank==0) printf("CPU time = %f \n", cputime);
// if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
// if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS);
// if (rank==0) printf("********************************************************\n");
// Number of memory references from the swap algorithm (per timestep)
// 18 reads and 18 writes for each lattice site
double MemoryRefs = Np*38;
// number of memory references for the swap algorithm - GigaBytes / second
// if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9/cputime);
// Report bandwidth in Gigabits per second
// communication bandwidth includes both send and recieve
//if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
// if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime);
double *Vz;
Vz= new double [Np];
int SIZE=Np*sizeof(double);
ScaLBL_D3Q19_Momentum(dist,Velocity, Np);
ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE);
if (rank == 0) printf("Force: %f,%f,%f \n",Fx,Fy,Fz);
ScaLBL_MRTModel MRT(rank,nprocs,comm);
auto filename = argv[1];
MRT.ReadParams(filename);
MRT.SetDomain(); // this reads in the domain
ParallelPlates(MRT);
MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables
MRT.Initialize(); // initializing the model will set initial conditions for variables
MRT.Run();
double *Vz; Vz= new double [MRT.Np];
MRT.VelocityField(Vz);
if (rank == 0) printf("Force: %f,%f,%f \n",MRT.Fx,MRT.Fy,MRT.Fz);
double mu = MRT.mu;
int Nx = MRT.Nx;
int Ny = MRT.Ny;
int Nz = MRT.Nz;
double Fz = MRT.Fz;
double vz;
double W = 1.f*Nx-4.f;
j=Ny/2; k=Nz/2;
if (rank == 0) printf("Channel width=%f \n",W);
if (rank == 0) printf("ID flag vz analytical\n");
MPI_Barrier(comm);
if (rank == 0) {
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
printf("%i ",Dm.id[n]);
n = Map(i,j,k);
printf("%i ",MRT.Mask->id[n]);
n = MRT.Map(i,j,k);
//printf("%i,%i,%i; %i :",i,j,k,n);
if (n<0) {vz =0.f; printf(" b "); }
else { vz=Vz[n]; printf(" a "); }
@ -341,14 +90,11 @@ int main(int argc, char **argv)
}
printf("\n");
}
if (rank == 1) {
for (i=0;i<Nx;i++){
n = k*Nx*Ny+j*Nx+i;
printf("%i ",Dm.id[n]);
n = Map(i,j,k);
printf("%i ",MRT.Mask->id[n]);
n = MRT.Map(i,j,k);
//printf("%i,%i,%i; %i :",i,j,k,n);
if (n<0) {vz =0.f; printf(" b "); }
else { vz=Vz[n]; printf(" a "); }
@ -361,9 +107,6 @@ int main(int argc, char **argv)
}
printf("\n");
}
}
// ****************************************************

138
tests/TestPressVel.cpp
View File

@ -9,7 +9,6 @@
#include "common/ScaLBL.h"
#include "common/MPI_Helpers.h"
using namespace std;
//***************************************************************************************
int main(int argc, char **argv)
@ -23,87 +22,28 @@ int main(int argc, char **argv)
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
int check;
int check=0;
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
if (rank == 0){
printf("********************************************************\n");
printf("Running Unit Test: TestPressVel \n");
printf("********************************************************\n");
}
// BGK Model parameters
string FILENAME;
unsigned int nBlocks, nthreads;
int timestepMax, interval;
double tau,Fx,Fy,Fz,tol;
// Domain variables
double Lx,Ly,Lz;
int nspheres;
int Nx,Ny,Nz;
int i,j,k,n;
int dim = 50; if (rank == 0) printf("dim=%d\n",dim);
double rlx_setA=1.0;
double rlx_setB=1.0;
Fx = Fy = 0.f;
Fz = 1.0e-4;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (domain.good()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
}
else if (nprocs==1){
nprocx=nprocy=nprocz=1;
Nx=4; Ny = 4;
Nz = 4;
nspheres=0;
Lx=Ly=Lz=1;
}
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
// Load inputs
string FILENAME = argv[1];
// Load inputs
if (rank==0) printf("Loading input database \n");
auto db = std::make_shared<Database>( FILENAME );
auto domain_db = db->getDatabase( "Domain" );
int Nx = domain_db->getVector<int>( "n" )[0];
int Ny = domain_db->getVector<int>( "n" )[1];
int Nz = domain_db->getVector<int>( "n" )[2];
int nprocx = domain_db->getVector<int>( "nproc" )[0];
int nprocy = domain_db->getVector<int>( "nproc" )[1];
int nprocz = domain_db->getVector<int>( "nproc" )[2];
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
@ -111,9 +51,9 @@ int main(int argc, char **argv)
}
MPI_Barrier(comm);
kproc = rank/(nprocx*nprocy);
jproc = (rank-nprocx*nprocy*kproc)/nprocx;
iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
int kproc = rank/(nprocx*nprocy);
int jproc = (rank-nprocx*nprocy*kproc)/nprocx;
int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
if (rank == 0) {
printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
@ -125,10 +65,9 @@ int main(int argc, char **argv)
}
double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz;
int BoundaryCondition=0;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
std::shared_ptr<Domain> Dm (new Domain(domain_db,comm));
Dm->CommInit();
Nx += 2;
Ny += 2;
@ -142,14 +81,8 @@ int main(int argc, char **argv)
sprintf(LocalRankString,"%05d",rank);
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rank);
/*
FILE *IDFILE = fopen(LocalRankFilename,"rb");
if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx");
fread(Dm.id,1,N,IDFILE);
fclose(IDFILE);
*/
Dm.CommInit(comm);
Dm->CommInit();
//.......................................................................
// Compute the media porosity
@ -161,8 +94,8 @@ int main(int argc, char **argv)
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
if (Dm.id[n] > 0){
Dm->id[n] = 1;
if (Dm->id[n] > 0){
sum_local+=1.0;
Np++;
}
@ -178,14 +111,13 @@ int main(int argc, char **argv)
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
ScaLBL_Communicator ScaLBL_Comm(Dm);
auto ScaLBL_Comm = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Dm));
//...........device phase ID.................................................
if (rank==0) printf ("Copying phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Copy to the device
ScaLBL_CopyToDevice(ID, Dm.id, N);
ScaLBL_CopyToDevice(ID, Dm->id, N);
//...........................................................................
if (rank==0){
@ -193,25 +125,20 @@ int main(int argc, char **argv)
printf("Reduced domain size = %i \n",Np);
}
// LBM variables
if (rank==0) printf ("Allocating distributions \n");
int neighborSize=18*Np*sizeof(int);
if (rank==0) printf ("Set up the neighborlist \n");
int Npad=Np+32;
int neighborSize=18*Npad*sizeof(int);
int *neighborList;
IntArray Map(Nx,Ny,Nz);
neighborList= new int[18*Np];
ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np);
// ScaLBL_Comm.MemoryDenseLayoutFull(Map,neighborList,Dm.id,Np); // this was how I tested for correctness
neighborList= new int[18*Npad];
Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np);
MPI_Barrier(comm);
//......................device distributions.................................
if (rank==0) printf ("Allocating distributions \n");
int dist_mem_size = Np*sizeof(double);
int *NeighborList;
// double *f_even,*f_odd;
double * dist;
double * Velocity;
//...........................................................................
@ -225,7 +152,6 @@ int main(int argc, char **argv)
* AA Algorithm begins here
*
*/
//ScaLBL_D3Q19_Init(ID, dist, &dist[10*Np], Np, 1, 1);
double *DIST;
DIST = new double [19*Np];
double VALUE=0.1;
@ -253,10 +179,8 @@ int main(int argc, char **argv)
DIST[17*Np + n] = 1.0;
DIST[18*Np + n] = 1.0;
}
ScaLBL_CopyToDevice(dist, DIST, 19*Np*sizeof(double));
double *Vz;
Vz= new double [Np];
size_t SIZE=Np*sizeof(double);
@ -273,4 +197,6 @@ int main(int argc, char **argv)
MPI_Barrier(comm);
MPI_Finalize();
// ****************************************************
return check;
}

232
tests/TestSegDist.cpp
View File

@ -11,138 +11,138 @@
#include <fstream>
#include "common/Array.h"
#include "common/Domain.h"
#include "analysis/eikonal.h"
#include "IO/Writer.h"
#include "analysis/distance.h"
std::shared_ptr<Database> loadInputs( int nprocs )
{
std::vector<int> nproc;
if ( nprocs == 1 ) {
nproc = { 1, 1, 1 };
} else if ( nprocs == 8 ) {
nproc = { 2, 2, 2 };
} else {
ERROR("TestSegDist: Unsupported number of processors");
}
auto db = std::make_shared<Database>( );
db->putScalar<int>( "BC", 0 );
db->putVector<int>( "nproc", nproc );
db->putVector<int>( "n", { 200, 200, 200 } );
db->putScalar<int>( "nspheres", 0 );
db->putVector<double>( "L", { 1, 1, 1 } );
return db;
}
//***************************************************************************************
int main(int argc, char **argv)
{
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
// Initialize MPI
int rank, nprocs;
MPI_Init(&argc,&argv);
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
int i,j,k,n,nn;
int iproc,jproc,kproc;
int nx,ny,nz;
int Nx, Ny, Nz, N;
int nprocx, nprocy, nprocz, nspheres;
double Lx, Ly, Lz;
Nx = Ny = Nz = 50;
nx = ny = nz = 50;
N = Nx*Ny*Nz;
nprocx=nprocy=nprocz=2;
Lx = Ly = Lz = 1.0;
int BC=0;
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
if (nprocs != 8){
ERROR("TestSegDist: Number of MPI processes must be equal to 8");
}
if (nprocx !=2 || nprocz !=2 || nprocy !=2 ){
ERROR("TestSegDist: MPI process grid must be 2x2x2");
}
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
// Get the rank info
Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
for (k=0;k<nz;k++){
for (j=0;j<ny;j++){
for (i=0;i<nx;i++){
n = k*nx*ny+j*nx+i;
Dm.id[n] = 1;
}
}
}
Dm.CommInit(comm);
Domain Dm(db,comm);
for (int k=0;k<Nz;k++){
for (int j=0;j<Ny;j++){
for (int i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
}
}
}
Dm.CommInit();
nx+=2; ny+=2; nz+=2;
N = nx*ny*nz;
int count = 0;
int nx = Nx+2;
int ny = Ny+2;
int nz = Nz+2;
char *id;
id = new char [N];
double BubbleRadius = 25;
// Initialize the bubble
double x,y,z , Cx,Cy,Cz;
// Initialize the bubble
double BubbleRadius = 0.15*Nx*Dm.nprocx();
double Cx = 0.40*Nx*Dm.nprocx();
double Cy = 0.45*Nx*Dm.nprocy();
double Cz = 0.50*Nx*Dm.nprocy();
Cx = 1.0*nx;
Cy = 1.0*ny;
Cz = 1.0*nz;
DoubleArray Distance(nx,ny,nz);
DoubleArray TrueDist(nx,ny,nz);
for (k=1;k<nz-1;k++){
for (j=1;j<ny-1;j++){
for (i=1;i<nx-1;i++){
// True signed distance
x = (nx-2)*Dm.iproc+i-1;
y = (ny-2)*Dm.jproc+j-1;
z = (nz-2)*Dm.kproc+k-1;
TrueDist(i,j,k) = sqrt((x-Cx)*(x-Cx)+(y-Cy)*(y-Cy)+(z-Cz)*(z-Cz)) - BubbleRadius;
n = k*nx*ny+j*nx+i;
// Initialize phase positions
if (TrueDist(i,j,k) < 0.0){
id[n] = 0;
}
else{
id[n]=1;
}
}
}
}
// Initialize the signed distance function
for (k=0;k<nz;k++){
for (j=0;j<ny;j++){
for (i=0;i<nx;i++){
n=k*nx*ny+j*nx+i;
// Initialize distance to +/- 1
Distance(i,j,k) = 1.0*id[n]-0.5;
}
}
}
if (rank==0) printf("Nx = %i \n",(int)Distance.size(0));
if (rank==0) printf("Ny = %i \n",(int)Distance.size(1));
if (rank==0) printf("Nz = %i \n",(int)Distance.size(2));
MPI_Barrier(comm);
if (rank==0) printf("Initialized! Converting to Signed Distance function \n");
double starttime,stoptime,cputime;
starttime = MPI_Wtime();
Eikonal(Distance,id,Dm,10);
stoptime = MPI_Wtime();
cputime = (stoptime - starttime);
if (rank==0) printf("Total time: %f seconds \n",cputime);
double Error=0.0;
int Count = 0;
for (k=0;k<nz;k++){
for (j=0;j<ny;j++){
for (i=0;i<nx;i++){
if (fabs(TrueDist(i,j,k)) < 3.0){
Error += (Distance(i,j,k)-TrueDist(i,j,k))*(Distance(i,j,k)-TrueDist(i,j,k));
Count += 1;
}
}
}
}
Error = sqrt(Error)/(double (Count));
if (rank==0) printf("Mean error %f \n", Error);
DoubleArray TrueDist(nx,ny,nz);
Array<char> id(nx,ny,nz);
id.fill(0);
for (int k=1; k<nz-1; k++) {
double z = k - 0.5 + Dm.kproc()*Nz;
for (int j=1; j<ny-1; j++) {
double y = j - 0.5 + Dm.jproc()*Ny;
for (int i=1; i<nx-1; i++) {
double x = i - 0.5 + Dm.iproc()*Nx;
// True signed distance
TrueDist(i,j,k) = sqrt((x-Cx)*(x-Cx)+(y-Cy)*(y-Cy)+(z-Cz)*(z-Cz)) - BubbleRadius;
// Initialize phase positions
if (TrueDist(i,j,k) < 0.0){
id(i,j,k) = 0;
} else{
id(i,j,k)=1;
}
}
}
}
MPI_Barrier(comm);
if (rank==0) printf("Initialized! Converting to Signed Distance function \n");
double t1 = MPI_Wtime();
DoubleArray Distance(nx,ny,nz);
CalcDist(Distance,id,Dm,{false,false,false});
double t2 = MPI_Wtime();
if (rank==0)
printf("Total time: %f seconds \n",t2-t1);
double err = 0.0;
for (int i=1; i<Nx-1; i++) {
for (int j=1; j<Ny-1; j++) {
for (int k=1; k<Nz-1; k++) {
err += (Distance(i,j,k)-TrueDist(i,j,k))*(Distance(i,j,k)-TrueDist(i,j,k));
}
}
}
err = sumReduce( Dm.Comm, err );
err = sqrt( err / (nx*ny*nz*nprocs) );
if (rank==0)
printf("Mean error %0.4f \n", err);
// Write the results to visit
Array<int> ID0(id.size());
ID0.copy( id );
Array<double> ID(Nx,Ny,Nz);
Array<double> dist1(Nx,Ny,Nz);
Array<double> dist2(Nx,Ny,Nz);
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1);
fillData.copy( ID0, ID );
fillData.copy( TrueDist, dist1 );
fillData.copy( Distance, dist2 );
std::vector<IO::MeshDataStruct> data(1);
data[0].meshName = "mesh";
data[0].mesh.reset( new IO::DomainMesh( Dm.rank_info, Nx, Ny, Nz, Dm.Lx, Dm.Ly, Dm.Lz ) );
data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "ID", ID ) );
data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "TrueDist", dist1 ) );
data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "Distance", dist2 ) );
data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "error", dist2-dist1 ) );
IO::initialize( "", "silo", false );
IO::writeData( "testSegDist", data, MPI_COMM_WORLD );
}
MPI_Barrier(comm);
MPI_Finalize();
return 0;

1
tests/TestSphereCurvature.cpp
View File

@ -2,6 +2,7 @@
#include <math.h>
#include "analysis/pmmc.h"
#include "common/Domain.h"
#include "common/SpherePack.h"
using namespace std;

162
tests/TestTorus.cpp
View File

@ -9,10 +9,18 @@
#include "analysis/analysis.h"
#include "analysis/TwoPhase.h"
//#include "Domain.h"
using namespace std;
std::shared_ptr<Database> loadInputs( int nprocs )
{
//auto db = std::make_shared<Database>( "Domain.in" );
auto db = std::make_shared<Database>();
db->putScalar<int>( "BC", 0 );
db->putVector<int>( "nproc", { 1, 1, 1 } );
db->putVector<int>( "n", { 100, 100, 100 } );
db->putScalar<int>( "nspheres", 1 );
db->putVector<double>( "L", { 1, 1, 1 } );
return db;
}
int main(int argc, char **argv)
@ -34,76 +42,42 @@ int main(int argc, char **argv)
//.......................................................................
// Reading the domain information file
//.......................................................................
int nprocx, nprocy, nprocz, nx, ny, nz, nspheres;
double Lx, Ly, Lz;
int Nx,Ny,Nz;
int i,j,k,n;
// Load inputs
auto db = loadInputs( nprocs );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
if (rank==0){
/* ifstream domain("Domain.in");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> nx;
domain >> ny;
domain >> nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
*/
// Set the domain for single processor test
nprocx=nprocy=nprocz=1;
nx=ny=nz=100;
nspheres=1;
Lx=Ly=Lz=1;
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("********************************************************\n");
}
MPI_Barrier(comm);
// Computational domain
MPI_Bcast(&nx,1,MPI_INT,0,comm);
MPI_Bcast(&ny,1,MPI_INT,0,comm);
MPI_Bcast(&nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// Check that the number of processors >= the number of ranks
if ( rank==0 ) {
printf("Number of MPI ranks required: %i \n", nprocx*nprocy*nprocz);
printf("Number of MPI ranks used: %i \n", nprocs);
printf("Full domain size: %i x %i x %i \n",nx*nprocx,ny*nprocy,nz*nprocz);
}
if ( nprocs < nprocx*nprocy*nprocz )
ERROR("Insufficient number of processors");
// Set up the domain
int BC=0;
// Get the rank info
Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
std::shared_ptr<Domain> Dm(new Domain(db,comm));
// const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
TwoPhase Averages(Dm);
int N = (nx+2)*(ny+2)*(nz+2);
Nx = nx+2;
Ny = ny+2;
Nz = nz+2;
if (rank == 0) cout << "Domain set." << endl;
std::shared_ptr<TwoPhase> Averages(new TwoPhase(Dm));
Nx += 2;
Ny += 2;
Nz += 2;
int N = Nx*Ny*Nz;
//.......................................................................
for ( k=1;k<Nz-1;k++){
for ( j=1;j<Ny-1;j++){
for ( i=1;i<Nx-1;i++){
n = k*Nx*Ny+j*Nx+i;
Dm.id[n] = 1;
Dm->id[n] = 1;
}
}
}
//.......................................................................
Dm.CommInit(comm); // Initialize communications for domains
Dm->CommInit(); // Initialize communications for domains
//.......................................................................
//.......................................................................
@ -128,42 +102,42 @@ int main(int argc, char **argv)
n = k*Nx*Ny+j*Nx+i;
// global position relative to center
x = Dm.iproc*Nx+i - CX;
y = Dm.jproc*Ny+j - CY;
z = Dm.kproc*Nz+k - CZ;
x = Dm->iproc()*Nx+i - CX;
y = Dm->jproc()*Ny+j - CY;
z = Dm->kproc()*Nz+k - CZ;
// Shrink the sphere sizes by two voxels to make sure they don't touch
Averages.SDs(i,j,k) = 100.0;
Averages->SDs(i,j,k) = 100.0;
//..............................................................................
// Single torus
Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2;
Averages->Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2;
// Double torus
/* y = Dm.jproc*Ny+j - CY1;
//z = Dm.kproc*Nz+k - CZ +R1;
Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2;
/* y = Dm->jproc()*Ny+j - CY1;
//z = Dm->kproc()*Nz+k - CZ +R1;
Averages->Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2;
y = Dm.jproc*Ny+j - CY2;
//z = Dm.kproc*Nz+k - CZ-R1;
Averages.Phase(i,j,k) = min(Averages.Phase(i,j,k),
y = Dm->jproc()*Ny+j - CY2;
//z = Dm->kproc()*Nz+k - CZ-R1;
Averages->Phase(i,j,k) = min(Averages->Phase(i,j,k),
sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2);
*///..............................................................................
//Averages.Phase(i,j,k) = - Averages.Phase(i,j,k);
if (Averages.Phase(i,j,k) > 0.0){
Dm.id[n] = 2;
//Averages->Phase(i,j,k) = - Averages->Phase(i,j,k);
if (Averages->Phase(i,j,k) > 0.0){
Dm->id[n] = 2;
}
else{
Dm.id[n] = 1;
Dm->id[n] = 1;
}
Averages.SDn(i,j,k) = Averages.Phase(i,j,k);
Averages.Phase(i,j,k) = Averages.SDn(i,j,k);
Averages.Phase_tplus(i,j,k) = Averages.SDn(i,j,k);
Averages.Phase_tminus(i,j,k) = Averages.SDn(i,j,k);
Averages.DelPhi(i,j,k) = 0.0;
Averages.Press(i,j,k) = 0.0;
Averages.Vel_x(i,j,k) = 0.0;
Averages.Vel_y(i,j,k) = 0.0;
Averages.Vel_z(i,j,k) = 0.0;
Averages->SDn(i,j,k) = Averages->Phase(i,j,k);
Averages->Phase(i,j,k) = Averages->SDn(i,j,k);
Averages->Phase_tplus(i,j,k) = Averages->SDn(i,j,k);
Averages->Phase_tminus(i,j,k) = Averages->SDn(i,j,k);
Averages->DelPhi(i,j,k) = 0.0;
Averages->Press(i,j,k) = 0.0;
Averages->Vel_x(i,j,k) = 0.0;
Averages->Vel_y(i,j,k) = 0.0;
Averages->Vel_z(i,j,k) = 0.0;
}
}
}
@ -171,25 +145,25 @@ int main(int argc, char **argv)
double beta = 0.95;
if (rank==0) printf("initializing the system \n");
Averages.UpdateSolid();
Dm.CommunicateMeshHalo(Averages.Phase);
Dm.CommunicateMeshHalo(Averages.SDn);
Averages->UpdateSolid();
Dm->CommunicateMeshHalo(Averages->Phase);
Dm->CommunicateMeshHalo(Averages->SDn);
Averages.Initialize();
Averages.UpdateMeshValues();
Averages->Initialize();
Averages->UpdateMeshValues();
if (rank==0) printf("computing local averages \n");
Averages.AssignComponentLabels();
Averages.ComponentAverages();
Averages.PrintComponents(int(5));
Averages->AssignComponentLabels();
Averages->ComponentAverages();
Averages->PrintComponents(int(5));
if (rank==0) printf("reducing averages \n");
Averages.Initialize();
Averages.ComputeLocal();
Averages.Reduce();
Averages.PrintAll(int(5));
// Averages.Reduce();
Averages->Initialize();
Averages->ComputeLocal();
Averages->Reduce();
Averages->PrintAll(int(5));
// Averages->Reduce();
} // Limit scope so variables that contain communicators will free before MPI_Finialize
MPI_Barrier(comm);

2
tests/TestTwoPhase.cpp
View File

@ -49,7 +49,7 @@ int main(int argc, char **argv)
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
Dm.CommInit(comm);
Dm.CommInit();
TwoPhase Averages(Dm);
int timestep=0;

2
tests/lbpm_BGK_simulator.cpp
View File

@ -153,7 +153,7 @@ int main(int argc, char **argv)
int BC=pBC;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
Dm.CommInit(comm);
Dm.CommInit();
TwoPhase Averages(Dm);
// Mask that excludes the solid phase

92
tests/lbpm_captube_pp.cpp
View File

@ -11,6 +11,17 @@
#include "analysis/TwoPhase.h"
#include "common/MPI_Helpers.h"
std::shared_ptr<Database> loadInputs( )
{
auto db = std::make_shared<Database>( "Domain.in" );
const int dim = 50;
db->putScalar<int>( "BC", 0 );
return db;
}
//***************************************************************************************
int main(int argc, char **argv)
{
//*****************************************
@ -23,10 +34,6 @@ int main(int argc, char **argv)
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
{
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
// MPI ranks for all 18 neighbors
//**********************************
@ -35,8 +42,6 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
double TubeRadius =15.0;
int BC;
@ -54,55 +59,24 @@ int main(int argc, char **argv)
// Variables that specify the computational domain
string FILENAME;
int Nx,Ny,Nz; // local sub-domain size
int nspheres; // number of spheres in the packing
double Lx,Ly,Lz; // Domain length
int i,j,k,n;
// pmmc threshold values
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
// Load inputs
auto db = loadInputs( );
int Nx = db->getVector<int>( "n" )[0];
int Ny = db->getVector<int>( "n" )[1];
int Nz = db->getVector<int>( "n" )[2];
int nprocx = db->getVector<int>( "nproc" )[0];
int nprocy = db->getVector<int>( "nproc" )[1];
int nprocz = db->getVector<int>( "nproc" )[2];
int kproc = rank/(nprocx*nprocy);
int jproc = (rank-nprocx*nprocy*kproc)/nprocx;
int iproc = rank-nprocx*nprocy*kproc-nprocx*jproc;
if (rank==0){
printf("********************************************************\n");
printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz);
@ -110,10 +84,12 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
// Initialized domain and averaging framework for Two-Phase Flow
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Dm.CommInit(comm);
TwoPhase Averages(Dm);
double Lx=1.f;
double Ly=1.f;
double Lz=1.f;
std::shared_ptr<Domain> Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
@ -155,18 +131,18 @@ int main(int argc, char **argv)
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
// Cylindrical capillary tube aligned with the z direction
Averages.SDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
Averages->SDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
+ (j-Ny/2)*(j-Ny/2)));
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Dm.kproc*Nz+k<BubbleBottom){
else if (Dm->kproc()*Nz+k<BubbleBottom){
id[n] = 2;
sum++;
}
else if (Dm.kproc*Nz+k<BubbleTop){
else if (Dm->kproc()*Nz+k<BubbleTop){
id[n] = 1;
sum++;
}
@ -199,7 +175,7 @@ int main(int argc, char **argv)
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST);
fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST);
fclose(DIST);
sprintf(LocalRankFilename,"ID.%05i",rank);

14
tests/lbpm_color_macro_simulator.cpp
View File

@ -258,7 +258,7 @@ int main(int argc, char **argv)
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
// TwoPhase Averages(Dm);
Dm.CommInit(comm);
Dm.CommInit();
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
@ -367,7 +367,7 @@ int main(int argc, char **argv)
if (rank==0) printf("Media porosity = %f \n",porosity);
//.........................................................
// If external boundary conditions are applied remove solid
if (BoundaryCondition > 0 && Dm.kproc == 0){
if (BoundaryCondition > 0 && Dm.kproc() == 0){
for (k=0; k<3; k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
@ -378,7 +378,7 @@ int main(int argc, char **argv)
}
}
}
if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){
if (BoundaryCondition > 0 && Dm.kproc() == nprocz-1){
for (k=Nz-3; k<Nz; k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
@ -488,12 +488,12 @@ int main(int argc, char **argv)
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np);
if (BoundaryCondition >0 ){
if (Dm.kproc==0){
if (Dm.kproc()==0){
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2);
}
if (Dm.kproc == nprocz-1){
if (Dm.kproc() == nprocz-1){
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3);
@ -550,8 +550,8 @@ int main(int argc, char **argv)
//************ MAIN ITERATION LOOP ***************************************/
PROFILE_START("Loop");
runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL,
rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile );
std::shared_ptr<Database> analysis_db;
runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map );
analysis.createThreads( analysis_method, 4 );
while (timestep < timestepMax && err > tol ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }

947
tests/lbpm_color_simulator.cpp
View File

@ -6,15 +6,7 @@
#include <stdexcept>
#include <fstream>
#include "common/ScaLBL.h"
#include "common/Communication.h"
#include "analysis/TwoPhase.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
#include "lbpm_color_simulator.h"
#include "models/ColorModel.h"
//#define WRE_SURFACES
@ -28,75 +20,6 @@ using namespace std;
//*************************************************************************
// Implementation of Two-Phase Immiscible LBM using CUDA
//*************************************************************************
inline void PackID(int *list, int count, char *sendbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
for (idx=0; idx<count; idx++){
n = list[idx];
sendbuf[idx] = ID[n];
}
}
//***************************************************************************************
inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
for (idx=0; idx<count; idx++){
n = list[idx];
ID[n] = recvbuf[idx];
}
}
//***************************************************************************************
inline void ZeroHalo(double *Data, int Nx, int Ny, int Nz)
{
int i,j,k,n;
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
i=0;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
i=Nx-1;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
}
}
for (k=0;k<Nz;k++){
for (i=0;i<Nx;i++){
j=0;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
j=Ny-1;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
}
}
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
k=0;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
k=Nz-1;
n = k*Nx*Ny+j*Nx+i;
Data[2*n] = 0.0;
Data[2*n+1] = 0.0;
}
}
}
//***************************************************************************************
int main(int argc, char **argv)
{
@ -111,18 +34,11 @@ int main(int argc, char **argv)
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
if (rank == 0){
printf("********************************************************\n");
printf("Running Color LBM \n");
printf("********************************************************\n");
}
PROFILE_ENABLE(1);
//PROFILE_ENABLE_TRACE();
//PROFILE_ENABLE_MEMORY();
@ -130,859 +46,16 @@ int main(int argc, char **argv)
PROFILE_START("Main");
Utilities::setErrorHandlers();
// Variables that specify the computational domain
string FILENAME;
unsigned int nBlocks, nthreads;
int Nx,Ny,Nz; // local sub-domain size
int nspheres; // number of spheres in the packing
double Lx,Ly,Lz; // Domain length
double D = 1.0; // reference length for non-dimensionalization
// Color Model parameters
int timestepMax;
double tau,Fx,Fy,Fz,tol,err;
double alpha, beta;
double das, dbs, phi_s;
double din,dout;
double wp_saturation;
int BoundaryCondition;
int InitialCondition;
// bool pBC,Restart;
int i,j,k;
// pmmc threshold values
//double fluid_isovalue,solid_isovalue;
//fluid_isovalue = 0.0;
//solid_isovalue = 0.0;
auto filename = argv[1];
ScaLBL_ColorModel ColorModel(rank,nprocs,comm);
ColorModel.ReadParams(filename);
ColorModel.SetDomain();
ColorModel.ReadInput();
ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables
ColorModel.Initialize(); // initializing the model will set initial conditions for variables
ColorModel.Run();
ColorModel.WriteDebug();
int RESTART_INTERVAL=20000;
//int ANALYSIS_)INTERVAL=1000;
int BLOB_ANALYSIS_INTERVAL=1000;
int timestep = -1;
if (rank==0){
//.............................................................
// READ SIMULATION PARMAETERS FROM INPUT FILE
//.............................................................
ifstream input("Color.in");
if (input.is_open()){
// Line 1: model parameters (tau, alpha, beta, das, dbs)
input >> tau; // Viscosity parameter
input >> alpha; // Surface Tension parameter
input >> beta; // Width of the interface
input >> phi_s; // value of phi at the solid surface
// Line 2: wetting phase saturation to initialize
input >> wp_saturation;
// Line 3: External force components (Fx,Fy, Fz)
input >> Fx;
input >> Fy;
input >> Fz;
// Line 4: Pressure Boundary conditions
input >> InitialCondition;
input >> BoundaryCondition;
input >> din;
input >> dout;
// Line 5: time-stepping criteria
input >> timestepMax; // max no. of timesteps
input >> RESTART_INTERVAL; // restart interval
input >> tol; // error tolerance
// Line 6: Analysis options
input >> BLOB_ANALYSIS_INTERVAL; // interval to analyze blob states
//.............................................................
}
else{
// Set default values
// Print warning
printf("WARNING: No input file provided (Color.in is missing)! Default parameters will be used. \n");
tau = 1.0;
alpha=0.005;
beta= 0.9;
Fx = Fy = Fz = 0.0;
InitialCondition=0;
BoundaryCondition=0;
din=dout=1.0;
timestepMax=0;
}
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
if (input.is_open()){
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
//.......................................................................
}
else{
// Set default values
// Print warning
printf("WARNING: No input file provided (Domain.in is missing)! Default parameters will be used. \n");
nprocx=nprocy=nprocz=1;
Nx=Ny=Nz=10;
nspheres=0;
Lx=Ly=Lz=1.0;
}
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&das,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm);
MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm);
MPI_Bcast(&din,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&timestepMax,1,MPI_INT,0,comm);
MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm);
MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm);
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
// Get the rank info
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
MPI_Barrier(comm);
// **************************************************************
// **************************************************************
double Ps = -(das-dbs)/(das+dbs);
double rlxA = 1.f/tau;
double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA);
//double xIntPos = log((1.0+phi_s)/(1.0-phi_s))/(2.0*beta);
// Set the density values inside the solid based on the input value phi_s
das = (phi_s+1.0)*0.5;
dbs = 1.0 - das;
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
printf("nprocz = %i \n",nprocz);
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
if (rank==0){
printf("********************************************************\n");
printf("tau = %f \n", tau);
printf("alpha = %f \n", alpha);
printf("beta = %f \n", beta);
printf("das = %f \n", das);
printf("dbs = %f \n", dbs);
printf("gamma_{wn} = %f \n", 5.796*alpha);
printf("Force(x) = %f \n", Fx);
printf("Force(y) = %f \n", Fy);
printf("Force(z) = %f \n", Fz);
printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz);
printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz);
if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n");
if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n");
if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n");
if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n");
if (InitialCondition==0) printf("Initial conditions assigned from phase ID file \n");
if (InitialCondition==1) printf("Initial conditions assigned from restart file \n");
printf("********************************************************\n");
}
// Initialized domain and averaging framework for Two-Phase Flow
bool pBC,velBC;
if (BoundaryCondition==1 || BoundaryCondition==3)
pBC=true;
else pBC=false;
if (BoundaryCondition==2) velBC=true;
else velBC=false;
bool Restart;
if (InitialCondition==1) Restart=true;
else Restart=false;
NULL_USE(pBC); NULL_USE(velBC);
// Full domain used for averaging (do not use mask for analysis)
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
Dm.CommInit(comm);
// Mask that excludes the solid phase
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition);
MPI_Barrier(comm);
Nx+=2; Ny+=2; Nz += 2;
//Nx = Ny = Nz; // Cubic domain
int N = Nx*Ny*Nz;
int dist_mem_size = N*sizeof(double);
//.......................................................................
if (rank == 0) printf("Read input media... \n");
//.......................................................................
//.......................................................................
// Filenames used
char LocalRankString[8];
char LocalRankFilename[40];
char LocalRestartFile[40];
char tmpstr[10];
sprintf(LocalRankString,"%05d",rank);
sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
// printf("Local File Name = %s \n",LocalRankFilename);
// .......... READ THE INPUT FILE .......................................
// char value;
char *id;
id = new char[N];
int sum = 0;
double sum_local;
double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6));
double porosity, pore_vol;
//...........................................................................
if (rank == 0) cout << "Reading in domain from signed distance function..." << endl;
//.......................................................................
sprintf(LocalRankString,"%05d",rank);
// sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
// WriteLocalSolidID(LocalRankFilename, id, N);
sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N);
MPI_Barrier(comm);
if (rank == 0) cout << "Domain set." << endl;
//.......................................................................
// Assign the phase ID field based on the signed distance
//.......................................................................
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
id[n] = 0;
}
}
}
sum=0;
pore_vol = 0.0;
for ( k=0;k<Nz;k++){
for ( j=0;j<Ny;j++){
for ( i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
if (Averages->SDs(n) > 0.0){
id[n] = 2;
}
// compute the porosity (actual interface location used)
if (Averages->SDs(n) > 0.0){
sum++;
}
}
}
}
if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n");
sprintf(LocalRankFilename,"ID.%05i",rank);
size_t readID;
FILE *IDFILE = fopen(LocalRankFilename,"rb");
if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx");
readID=fread(id,1,N,IDFILE);
if (readID != size_t(N)) printf("lbpm_segmented_pp: Error reading ID (rank=%i) \n",rank);
fclose(IDFILE);
/* for ( k=0;k<Nz;k++){
for ( j=0;j<Ny;j++){
for ( i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
// The following turns off communication if external BC are being set
if (BoundaryCondition > 0){
if (kproc==0 && k==0) id[n]=0;
if (kproc==0 && k==1) id[n]=0;
if (kproc==nprocz-1 && k==Nz-2) id[n]=0;
if (kproc==nprocz-1 && k==Nz-1) id[n]=0;
}
}
}
}
*/
// Set up kstart, kfinish so that the reservoirs are excluded from averaging
int kstart,kfinish;
kstart = 1;
kfinish = Nz-1;
if (BoundaryCondition > 0 && Dm.kproc==0) kstart = 4;
if (BoundaryCondition > 0 && Dm.kproc==nprocz-1) kfinish = Nz-4;
// Compute the pore volume
sum_local = 0.0;
for ( k=kstart;k<kfinish;k++){
for ( j=1;j<Ny-1;j++){
for ( i=1;i<Nx-1;i++){
int n = k*Nx*Ny+j*Nx+i;
if (id[n] > 0){
sum_local += 1.0;
}
}
}
}
MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm);
// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
porosity = pore_vol*iVol_global;
if (rank==0) printf("Media porosity = %f \n",porosity);
//.........................................................
// If external boundary conditions are applied remove solid
if (BoundaryCondition > 0 && Dm.kproc == 0){
for (k=0; k<3; k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
//id[n] = 1;
Averages->SDs(n) = max(Averages->SDs(n),1.0*(2.5-k));
}
}
}
}
if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){
for (k=Nz-3; k<Nz; k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
//id[n] = 2;
Averages->SDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5));
}
}
}
}
//.........................................................
// don't perform computations at the eight corners
id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0;
id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
//.........................................................
// Initialize communication structures in averaging domain
for (i=0; i<Mask.Nx*Mask.Ny*Mask.Nz; i++) Mask.id[i] = id[i];
Mask.CommInit(comm);
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");
// Create a communicator for the device
ScaLBL_Communicator ScaLBL_Comm(Mask);
// set reservoirs
if (BoundaryCondition > 0){
for ( k=0;k<Nz;k++){
for ( j=0;j<Ny;j++){
for ( i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
if (Dm.kproc==0 && k==0) id[n]=1;
if (Dm.kproc==0 && k==1) id[n]=1;
if (Dm.kproc==nprocz-1 && k==Nz-2) id[n]=2;
if (Dm.kproc==nprocz-1 && k==Nz-1) id[n]=2;
Mask.id[n] = id[n];
}
}
}
}
//...........device phase ID.................................................
if (rank==0) printf ("Copy phase ID to device \n");
char *ID;
ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory
// Don't compute in the halo
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
int n = k*Nx*Ny+j*Nx+i;
if (i==0 || i==Nx-1 || j==0 || j==Ny-1 || k==0 || k==Nz-1) id[n] = 0;
}
}
}
// Copy to the device
ScaLBL_CopyToDevice(ID, id, N);
ScaLBL_DeviceBarrier();
//...........................................................................
//...........................................................................
// MAIN VARIABLES ALLOCATED HERE
//...........................................................................
// LBM variables
if (rank==0) printf ("Allocating distributions \n");
//......................device distributions.................................
double *f_even,*f_odd;
double *A_even,*A_odd,*B_even,*B_odd;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &f_even, 10*dist_mem_size); // Allocate device memory
ScaLBL_AllocateDeviceMemory((void **) &f_odd, 9*dist_mem_size); // Allocate device memory
ScaLBL_AllocateDeviceMemory((void **) &A_even, 4*dist_mem_size); // Allocate device memory
ScaLBL_AllocateDeviceMemory((void **) &A_odd, 3*dist_mem_size); // Allocate device memory
ScaLBL_AllocateDeviceMemory((void **) &B_even, 4*dist_mem_size); // Allocate device memory
ScaLBL_AllocateDeviceMemory((void **) &B_odd, 3*dist_mem_size); // Allocate device memory
//...........................................................................
double *Phi,*Den;
double *ColorGrad, *Velocity, *Pressure, *dvcSignDist;
//...........................................................................
ScaLBL_AllocateDeviceMemory((void **) &Phi, dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Pressure, dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &dvcSignDist, dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*dist_mem_size);
ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*dist_mem_size);
//...........................................................................
// Copy signed distance for device initialization
ScaLBL_CopyToDevice(dvcSignDist, Averages->SDs.data(), dist_mem_size);
//...........................................................................
int logcount = 0; // number of surface write-outs
//...........................................................................
// MAIN VARIABLES INITIALIZED HERE
//...........................................................................
//...........................................................................
//...........................................................................
if (rank==0) printf("Setting the distributions, size = %i\n", N);
//...........................................................................
ScaLBL_DeviceBarrier();
ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz);
ScaLBL_Color_Init(ID, Den, Phi, das, dbs, Nx, Ny, Nz);
ScaLBL_DeviceBarrier();
//......................................................................
if (Restart == true){
if (rank==0){
printf("Reading restart file! \n");
ifstream restart("Restart.txt");
if (restart.is_open()){
restart >> timestep;
printf("Restarting from timestep =%i \n",timestep);
}
else{
printf("WARNING:No Restart.txt file, setting timestep=0 \n");
timestep=5;
}
}
MPI_Bcast(&timestep,1,MPI_INT,0,comm);
// Read in the restart file to CPU buffers
double *cDen = new double[2*N];
double *cDistEven = new double[10*N];
double *cDistOdd = new double[9*N];
ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N);
// Copy the restart data to the GPU
ScaLBL_CopyToDevice(f_even,cDistEven,10*N*sizeof(double));
ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double));
ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double));
ScaLBL_DeviceBarrier();
delete [] cDen;
delete [] cDistEven;
delete [] cDistOdd;
MPI_Barrier(comm);
}
//......................................................................
ScaLBL_D3Q7_Init(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz);
ScaLBL_D3Q7_Init(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
//.......................................................................
// Once phase has been initialized, map solid to account for 'smeared' interface
//for (i=0; i<N; i++) Averages->SDs(i) -= (1.0);
// Make sure the id match for the two domains
for (i=0; i<N; i++) Dm.id[i] = Mask.id[i];
//.......................................................................
// Finalize setup for averaging domain
//Averages->SetupCubes(Mask);
Averages->UpdateSolid();
//.......................................................................
//*************************************************************************
// Compute the phase indicator field and reset Copy, Den
//*************************************************************************
ScaLBL_ComputePhaseField(ID, Phi, Den, N);
//*************************************************************************
ScaLBL_DeviceBarrier();
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_Comm.RecvHalo(Phi);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
//*************************************************************************
if (rank==0 && BoundaryCondition==1){
printf("Setting inlet pressure = %f \n", din);
printf("Setting outlet pressure = %f \n", dout);
}
if (BoundaryCondition==1 && Mask.kproc == 0) {
ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
}
if (BoundaryCondition==1 && Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
}
if (rank==0 && BoundaryCondition==2){
printf("Setting inlet velocity = %f \n", din);
printf("Setting outlet velocity = %f \n", dout);
}
if (BoundaryCondition==2 && Mask.kproc == 0) {
ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
}
if (BoundaryCondition==2 && Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
}
// Set dynamic pressure boundary conditions
double dp, slope;
dp = slope = 0.0;
if (BoundaryCondition==3){
slope = abs(dout-din)/timestepMax;
dp = abs(timestep)*slope;
if (rank==0) printf("Change in pressure / time =%.3e \n",slope);
// set the initial value
din = 1.0+0.5*dp;
dout = 1.0-0.5*dp;
// set the initial boundary conditions
if (Mask.kproc == 0) {
ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
}
if (Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
}
}
ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
//...........................................................................
// Copy the phase indicator field for the earlier timestep
ScaLBL_DeviceBarrier();
ScaLBL_CopyToHost(Averages->Phase_tplus.data(),Phi,N*sizeof(double));
//...........................................................................
//...........................................................................
// Copy the data for for the analysis timestep
//...........................................................................
// Copy the phase from the GPU -> CPU
//...........................................................................
ScaLBL_DeviceBarrier();
ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double));
ScaLBL_CopyToHost(Averages->Press.data(),Pressure,N*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_x.data(),&Velocity[0],N*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_y.data(),&Velocity[N],N*sizeof(double));
ScaLBL_CopyToHost(Averages->Vel_z.data(),&Velocity[2*N],N*sizeof(double));
//...........................................................................
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps: %i \n", timestepMax);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
starttime = MPI_Wtime();
//.........................................
err = 1.0;
double sat_w_previous = 1.01; // slightly impossible value!
if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol);
// Create the thread pool
int N_threads = 4;
if ( provided_thread_support < MPI_THREAD_MULTIPLE )
N_threads = 0;
if ( N_threads > 0 ) {
// Set the affinity
int N_procs = ThreadPool::getNumberOfProcessors();
std::vector<int> procs(N_procs);
for (int i=0; i<N_procs; i++)
procs[i] = i;
ThreadPool::setProcessAffinity(procs);
}
ThreadPool tpool(N_threads);
// Create the MeshDataStruct
fillHalo<double> fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1);
std::vector<IO::MeshDataStruct> meshData(1);
meshData[0].meshName = "domain";
meshData[0].mesh = std::shared_ptr<IO::DomainMesh>( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) );
std::shared_ptr<IO::Variable> PhaseVar( new IO::Variable() );
std::shared_ptr<IO::Variable> PressVar( new IO::Variable() );
std::shared_ptr<IO::Variable> SignDistVar( new IO::Variable() );
std::shared_ptr<IO::Variable> BlobIDVar( new IO::Variable() );
PhaseVar->name = "phase";
PhaseVar->type = IO::VariableType::VolumeVariable;
PhaseVar->dim = 1;
PhaseVar->data.resize(Nx-2,Ny-2,Nz-2);
meshData[0].vars.push_back(PhaseVar);
PressVar->name = "Pressure";
PressVar->type = IO::VariableType::VolumeVariable;
PressVar->dim = 1;
PressVar->data.resize(Nx-2,Ny-2,Nz-2);
meshData[0].vars.push_back(PressVar);
SignDistVar->name = "SignDist";
SignDistVar->type = IO::VariableType::VolumeVariable;
SignDistVar->dim = 1;
SignDistVar->data.resize(Nx-2,Ny-2,Nz-2);
meshData[0].vars.push_back(SignDistVar);
BlobIDVar->name = "BlobID";
BlobIDVar->type = IO::VariableType::VolumeVariable;
BlobIDVar->dim = 1;
BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2);
meshData[0].vars.push_back(BlobIDVar);
//************ MAIN ITERATION LOOP ***************************************/
PROFILE_START("Loop");
BlobIDstruct last_ids, last_index;
BlobIDList last_id_map;
writeIDMap(ID_map_struct(),0,id_map_filename);
AnalysisWaitIdStruct work_ids;
while (timestep < timestepMax && err > tol ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
//*************************************************************************
// Fused Color Gradient and Collision
//*************************************************************************
ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad,
Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz);
//*************************************************************************
ScaLBL_DeviceBarrier();
//*************************************************************************
// Pack and send the D3Q19 distributions
ScaLBL_Comm.SendD3Q19(f_even, f_odd);
//*************************************************************************
//*************************************************************************
// Carry out the density streaming step for mass transport
//*************************************************************************
ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi,
ColorGrad, Velocity, beta, N, pBC);
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
//*************************************************************************
// Swap the distributions for momentum transport
//*************************************************************************
ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz);
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
//*************************************************************************
// Wait for communications to complete and unpack the distributions
ScaLBL_Comm.RecvD3Q19(f_even, f_odd);
//*************************************************************************
ScaLBL_DeviceBarrier();
//*************************************************************************
// Pack and send the D3Q7 distributions
ScaLBL_Comm.BiSendD3Q7(A_even, A_odd, B_even, B_odd);
//*************************************************************************
ScaLBL_DeviceBarrier();
ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz);
ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz);
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
//*************************************************************************
// Wait for communication and unpack the D3Q7 distributions
ScaLBL_Comm.BiRecvD3Q7(A_even, A_odd, B_even, B_odd);
//*************************************************************************
ScaLBL_DeviceBarrier();
//..................................................................................
ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz);
ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz);
//*************************************************************************
// Compute the phase indicator field
//*************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
ScaLBL_ComputePhaseField(ID, Phi, Den, N);
//*************************************************************************
ScaLBL_Comm.SendHalo(Phi);
ScaLBL_DeviceBarrier();
ScaLBL_Comm.RecvHalo(Phi);
//*************************************************************************
ScaLBL_DeviceBarrier();
// Pressure boundary conditions
if (BoundaryCondition==1 && Mask.kproc == 0) {
ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
}
if (BoundaryCondition==1 && Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
}
// Velocity boundary conditions
if (BoundaryCondition==2 && Mask.kproc == 0) {
ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,0);
}
if (BoundaryCondition==2 && Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,Nz-1);
}
if (BoundaryCondition==3){
// Increase the pressure difference
dp += slope;
din = 1.0+0.5*dp;
dout = 1.0-0.5*dp;
// set the initial boundary conditions
if (Mask.kproc == 0) {
ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz);
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
}
if (Mask.kproc == nprocz-1){
ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2));
ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz);
ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
}
}
//...................................................................................
MPI_Barrier(comm);
PROFILE_STOP("Update");
// Timestep completed!
timestep++;
// Run the analysis, blob identification, and write restart files
// if (BLOB_ANALYSIS_INTERVAL > 0){
//if (timestep > 5)
run_analysis(timestep,RESTART_INTERVAL,rank_info,*Averages,last_ids,last_index,last_id_map,
Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den,
LocalRestartFile,meshData,fillData,tpool,work_ids);
/* }
else{
ComputeMacroscaleAverages(timestep,ANALYSIS_INTERVAL,RESTART_INTERVAL,rank_info,*Averages,
Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den,
LocalRestartFile,meshData,fillData,tpool,work_ids);
}
*/
// Save the timers
if ( timestep%50==0 )
PROFILE_SAVE("lbpm_color_simulator",1);
}
tpool.wait_pool_finished();
PROFILE_STOP("Loop");
//************************************************************************
ScaLBL_DeviceBarrier();
MPI_Barrier(comm);
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
// Performance obtained from each node
double MLUPS = double(Nx*Ny*Nz)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
MLUPS *= nprocs;
if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
if (rank==0) printf("********************************************************\n");
// ************************************************************************
/* // Perform component averaging and write tcat averages
Averages->Initialize();
Averages->ComponentAverages();
Averages->SortBlobs();
Averages->PrintComponents(timestep);
// ************************************************************************
int NumberComponents_NWP = ComputeGlobalPhaseComponent(Mask.Nx-2,Mask.Ny-2,Mask.Nz-2,Mask.rank_info,Averages->PhaseID,1,Averages->Label_NWP);
printf("Number of non-wetting phase components: %i \n ",NumberComponents_NWP);
ScaLBL_DeviceBarrier();
ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double));
*/
/* Averages->WriteSurfaces(0);
sprintf(LocalRankFilename,"%s%s","Phase.",LocalRankString);
FILE *PHASE;
PHASE = fopen(LocalRankFilename,"wb");
fwrite(Averages->SDn.data(),8,N,PHASE);
fclose(PHASE);
*/
/* sprintf(LocalRankFilename,"%s%s","Pressure.",LocalRankString);
FILE *PRESS;
PRESS = fopen(LocalRankFilename,"wb");
fwrite(Averages->Press.data(),8,N,PRESS);
fclose(PRESS);
ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double));
double * Grad;
Grad = new double [3*N];
ScaLBL_CopyToHost(Grad,ColorGrad,3*N*sizeof(double));
sprintf(LocalRankFilename,"%s%s","ColorGrad.",LocalRankString);
FILE *GRAD;
GRAD = fopen(LocalRankFilename,"wb");
fwrite(Grad,8,3*N,GRAD);
fclose(GRAD);
*/
PROFILE_STOP("Main");
PROFILE_SAVE("lbpm_color_simulator",1);
// ****************************************************

1332
tests/lbpm_dfh_simulator.cpp
View File

File diff suppressed because it is too large Load Diff

2
tests/lbpm_disc_pp.cpp
View File

@ -10,7 +10,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/MPI_Helpers.h" // This includes mpi.h
#include "common/SpherePack.h"
/*
* Pre-Processor to generate signed distance function from disc packing

23
tests/lbpm_inkbottle_pp.cpp
View File

@ -26,7 +26,6 @@ int main(int argc, char **argv)
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
// MPI ranks for all 18 neighbors
//**********************************
@ -35,8 +34,6 @@ int main(int argc, char **argv)
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
//**********************************
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
double UpperTubeRadius =15.0;
double LowerTubeRadius =15.0;
@ -117,9 +114,9 @@ int main(int argc, char **argv)
}
// Initialized domain and averaging framework for Two-Phase Flow
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Dm.CommInit(comm);
TwoPhase Averages(Dm);
std::shared_ptr<Domain> Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
@ -163,25 +160,25 @@ int main(int argc, char **argv)
n = k*Nx*Ny + j*Nz + i;
// Cylindrical capillary tube aligned with the z direction
if (k<Nz/2)
Averages.SDs(i,j,k) = LowerTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
Averages->SDs(i,j,k) = LowerTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
+ (j-Ny/2)*(j-Ny/2)));
else
Averages.SDs(i,j,k) = UpperTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
Averages->SDs(i,j,k) = UpperTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)
+ (j-Ny/2)*(j-Ny/2)));
BulbDist = BulbRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)+ (j-Ny/2)*(j-Ny/2) + (k-Nz/2)*(k-Nz/2)));
if (BulbDist > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = BulbDist;
if (BulbDist > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = BulbDist;
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Dm.kproc*Nz+k<BubbleBottom){
else if (Dm->kproc()*Nz+k<BubbleBottom){
id[n] = 2;
sum++;
}
else if (Dm.kproc*Nz+k<BubbleTop){
else if (Dm->kproc()*Nz+k<BubbleTop){
id[n] = 1;
sum++;
}
@ -214,7 +211,7 @@ int main(int argc, char **argv)
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST);
fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST);
fclose(DIST);
sprintf(LocalRankFilename,"ID.%05i",rank);

2
tests/lbpm_juanes_bench_disc_pp.cpp
View File

@ -10,7 +10,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/MPI_Helpers.h" // This includes mpi.h
#include "common/SpherePack.h"
/*
* Pre-Processor to generate signed distance function from disc packing

78
tests/lbpm_morph_pp.cpp
View File

@ -168,9 +168,9 @@ int main(int argc, char **argv)
Dm.CommInit(comm);
int iproc = Dm.iproc;
int jproc = Dm.jproc;
int kproc = Dm.kproc;
int iproc = Dm.iproc();
int jproc = Dm.jproc();
int kproc = Dm.kproc();
// Generate the NWP configuration
//if (rank==0) printf("Performing morphological drainage with critical radius %f \n", Rcrit);
@ -343,42 +343,42 @@ int main(int argc, char **argv)
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE);
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);

84
tests/lbpm_morphdrain_pp.cpp
View File

@ -148,14 +148,14 @@ int main(int argc, char **argv)
}
}
*/
Dm.CommInit(comm);
Dm.CommInit();
int xdim,ydim,zdim;
xdim=Dm.Nx-2;
ydim=Dm.Ny-2;
zdim=Dm.Nz-2;
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
fillHalo<double> fillData(Dm.Comm, Dm.rank_info,{xdim,ydim,zdim},{1,1,1},0,1);
DoubleArray SignDist(nx,ny,nz);
// Read the signed distance from file
@ -177,10 +177,10 @@ int main(int argc, char **argv)
fclose(IDFILE);
Dm.CommInit(comm);
int iproc = Dm.iproc;
int jproc = Dm.jproc;
int kproc = Dm.kproc;
Dm.CommInit();
int iproc = Dm.iproc();
int jproc = Dm.jproc();
int kproc = Dm.kproc();
// Generate the NWP configuration
//if (rank==0) printf("Performing morphological drainage with critical radius %f \n", Rcrit);
@ -360,42 +360,42 @@ int main(int argc, char **argv)
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE);
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);

82
tests/lbpm_morphopen_pp.cpp
View File

@ -131,7 +131,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
DoubleArray SignDist(nx,ny,nz);
// Read the signed distance from file
@ -270,10 +270,10 @@ int main(int argc, char **argv)
MPI_Barrier(comm);
*/
Dm.CommInit(comm);
int iproc = Dm.iproc;
int jproc = Dm.jproc;
int kproc = Dm.kproc;
Dm.CommInit();
int iproc = Dm.iproc();
int jproc = Dm.jproc();
int kproc = Dm.kproc();
// Generate the NWP configuration
//if (rank==0) printf("Initializing morphological distribution with critical radius %f \n", Rcrit);
@ -415,42 +415,42 @@ int main(int argc, char **argv)
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);

2
tests/lbpm_nondarcy_simulator.cpp
View File

@ -347,7 +347,7 @@ int main(int argc, char **argv)
//.........................................................
// Initialize communication structures in averaging domain
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
Dm.CommInit(comm);
Dm.CommInit();
//...........................................................................
if (rank==0) printf ("Create ScaLBL_Communicator \n");

4
tests/lbpm_nonnewtonian_simulator.cpp
View File

@ -308,7 +308,7 @@ int main(int argc, char **argv)
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); /* 1 */
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
Dm.CommInit(comm); /* 2 */
Dm.CommInit(); /* 2 */
Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
@ -482,7 +482,7 @@ int main(int argc, char **argv)
// Initialize communication structures in averaging domain
// for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = id[i];
// Dm.CommInit(comm);
// Dm.CommInit();
for (i=0; i<Mask.Nx*Mask.Ny*Mask.Nz; i++) Mask.id[i] = id[i];
Mask.CommInit(comm);

2
tests/lbpm_permeability_simulator.cpp
View File

@ -155,7 +155,7 @@ int main(int argc, char **argv)
int BC=pBC;
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
for (i=0; i<Dm.Nx*Dm.Ny*Dm.Nz; i++) Dm.id[i] = 1;
Dm.CommInit(comm);
Dm.CommInit();
TwoPhase Averages(Dm);
// Mask that excludes the solid phase

16
tests/lbpm_plates_pp.cpp
View File

@ -110,9 +110,11 @@ int main(int argc, char **argv)
}
// Initialized domain and averaging framework for Two-Phase Flow
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Dm.CommInit(comm);
TwoPhase Averages(Dm);
std::shared_ptr<Domain> Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
MPI_Barrier(comm);
@ -149,13 +151,13 @@ int main(int argc, char **argv)
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
// Cylindrical capillary tube aligned with the z direction
Averages.SDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)));
Averages->SDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)));
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Averages.SDs(i,j,k) < WIDTH){
else if (Averages->SDs(i,j,k) < WIDTH){
id[n] = 2;
sum++;
}
@ -188,7 +190,7 @@ int main(int argc, char **argv)
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST);
fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST);
fclose(DIST);
sprintf(LocalRankFilename,"ID.%05i",rank);

22
tests/lbpm_porenetwork_pp.cpp
View File

@ -100,10 +100,14 @@ int main(int argc, char **argv)
}
// Initialized domain and averaging framework for Two-Phase Flow
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Dm.CommInit(comm);
TwoPhase Averages(Dm);
// Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
//Dm.CommInit();
//TwoPhase Averages(Dm);
std::shared_ptr<Domain> Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
MPI_Barrier(comm);
Nx += 2; Ny += 2; Nz += 2;
@ -180,7 +184,7 @@ int main(int argc, char **argv)
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
Averages.SDs(i,j,k) = -2.f*Nx;
Averages->SDs(i,j,k) = -2.f*Nx;
}
}
}
@ -208,7 +212,7 @@ int main(int argc, char **argv)
double zk = double(k) - z;
// value of s along center line {x=alpha*s, y = beta*s, z=gamma*s} that is closest to i,j,k
double s = (alpha*xi + beta*yj + gamma*zk)/(alpha*alpha + beta*beta + gamma*gamma);
double distance=Averages.SDs(i,j,k);
double distance=Averages->SDs(i,j,k);
if (s > length){
//distance = radius - sqrt((xi-X)*(xi-X) + (yj-Y)*(yj-Y) + (zk-Z)*(zk-Z));
}
@ -222,7 +226,7 @@ int main(int argc, char **argv)
distance = radius - sqrt((xi-xs)*(xi-xs) + (yj-ys)*(yj-ys) + (zk-zs)*(zk-zs));
//if (distance>0)printf("s=%f,alpha=%f,beta=%f,gamma=%f,distance=%f\n",s,alpha,beta,gamma,distance);
}
if (distance > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = distance;
if (distance > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = distance;
}
// Compute the distance to each sphere
@ -237,7 +241,7 @@ int main(int argc, char **argv)
double distance = radius - sqrt((xi-x)*(xi-x) + (yj-y)*(yj-y) + (zk-z)*(zk-z));
if (distance > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = distance;
if (distance > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = distance;
}
}
}
@ -247,7 +251,7 @@ int main(int argc, char **argv)
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else{
@ -279,7 +283,7 @@ int main(int argc, char **argv)
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST);
fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST);
fclose(DIST);
sprintf(LocalRankFilename,"ID.%05i",rank);

82
tests/lbpm_random_pp.cpp
View File

@ -141,7 +141,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
DoubleArray SignDist(nx,ny,nz);
// Read the signed distance from file
@ -171,10 +171,10 @@ int main(int argc, char **argv)
float porosity=float(totalGlobal)/(nprocx*nprocy*nprocz*(nx-2)*(ny-2)*(nz-2));
if (rank==0) printf("Media Porosity: %f \n",porosity);
Dm.CommInit(comm);
int iproc = Dm.iproc;
int jproc = Dm.jproc;
int kproc = Dm.kproc;
Dm.CommInit();
int iproc = Dm.iproc();
int jproc = Dm.jproc();
int kproc = Dm.kproc();
int bin, binCount;
ifstream Dist("BlobSize.in");
@ -346,42 +346,42 @@ int main(int argc, char **argv)
PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id);
PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag,
recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag,
recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag,
recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag,
recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag,
recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag,
recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag,
recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag,
recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag,
recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag,
recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag,
recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag,
recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag,
recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag,
recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag,
recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag,
recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag,
recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag,
recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE);
//......................................................................................
UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id);
UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id);

21
tests/lbpm_refine_pp.cpp
View File

@ -87,7 +87,7 @@ int main(int argc, char **argv)
// Communication the halos
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
fillHalo<double> fillData(comm,rank_info,rnx,rny,rnz,1,1,1,0,1);
fillHalo<double> fillData(comm,rank_info,{rnx,rny,rnz},{1,1,1},0,1);
nx+=2; ny+=2; nz+=2;
rnx+=2; rny+=2; rnz+=2;
@ -102,7 +102,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
DoubleArray SignDist(nx,ny,nz);
// Read the signed distance from file
@ -170,7 +170,7 @@ int main(int argc, char **argv)
// Write output blocks with the same sub-domain size as origina
// refinement increases the size of the process grid
writerank = 8*Dm.kproc*nprocx*nprocy + 4*Dm.jproc*nprocx + 2*Dm.iproc;
writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -198,7 +198,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = 8*Dm.kproc*nprocx*nprocy + 4*Dm.jproc*nprocx + 2*Dm.iproc+1;
writerank = 8*Dm.kproc()*nprocx*nprocy + 4*Dm.jproc()*nprocx + 2*Dm.iproc()+1;
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -227,7 +227,7 @@ int main(int argc, char **argv)
fclose(WRITEID);
writerank = (2*Dm.kproc)*4*nprocx*nprocy + (2*Dm.jproc+1)*2*nprocx + 2*Dm.iproc+1;
writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1;
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -255,7 +255,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = (2*Dm.kproc)*4*nprocx*nprocy + (2*Dm.jproc+1)*2*nprocx + 2*Dm.iproc;
writerank = (2*Dm.kproc())*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -283,8 +283,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = (2*Dm.kproc+1)*4*nprocx*nprocy + (2*Dm.jproc)*2*nprocx + 2*Dm.iproc;
writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -312,7 +311,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = (2*Dm.kproc+1)*4*nprocx*nprocy + (2*Dm.jproc)*2*nprocx + 2*Dm.iproc+1;
writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc())*2*nprocx + 2*Dm.iproc()+1;
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -340,7 +339,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = (2*Dm.kproc+1)*4*nprocx*nprocy + (2*Dm.jproc+1)*2*nprocx + 2*Dm.iproc;
writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){
@ -368,7 +367,7 @@ int main(int argc, char **argv)
fwrite(id,1,nx*ny*nz,WRITEID);
fclose(WRITEID);
writerank = (2*Dm.kproc+1)*4*nprocx*nprocy + (2*Dm.jproc+1)*2*nprocx + 2*Dm.iproc+1;
writerank = (2*Dm.kproc()+1)*4*nprocx*nprocy + (2*Dm.jproc()+1)*2*nprocx + 2*Dm.iproc()+1;
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
for (int i=0; i<nx; i++){

10
tests/lbpm_segmented_decomp.cpp
View File

@ -142,7 +142,7 @@ int main(int argc, char **argv)
}
}
}
Dm.CommInit(comm);
Dm.CommInit();
// number of sites to use for periodic boundary condition transition zone
int z_transition_size = (nprocz*nz - (Nz - zStart))/2;
@ -151,7 +151,7 @@ int main(int argc, char **argv)
// Set up the sub-domains
if (rank==0){
printf("Distributing subdomains across %i processors \n",nprocs);
printf("Process grid: %i x %i x %i \n",Dm.nprocx,Dm.nprocy,Dm.nprocz);
printf("Process grid: %i x %i x %i \n",Dm.nprocx(),Dm.nprocy(),Dm.nprocz());
printf("Subdomain size: %i \n",N);
printf("Size of transition region: %i \n", z_transition_size);
char *tmp;
@ -160,7 +160,7 @@ int main(int argc, char **argv)
for (int jp=0; jp<nprocy; jp++){
for (int ip=0; ip<nprocx; ip++){
// rank of the process that gets this subdomain
int rnk = kp*Dm.nprocx*Dm.nprocy + jp*Dm.nprocx + ip;
int rnk = kp*Dm.nprocx()*Dm.nprocy() + jp*Dm.nprocx() + ip;
// Pack and send the subdomain for rnk
for (k=0;k<nz+2;k++){
for (j=0;j<ny+2;j++){
@ -342,8 +342,8 @@ int main(int argc, char **argv)
if (rank==0) printf("Writing symmetric domain reflection\n");
MPI_Barrier(comm);
int symrank,sympz;
sympz = 2*nprocz - Dm.kproc -1;
symrank = sympz*nprocx*nprocy + Dm.jproc*nprocx + Dm.iproc;
sympz = 2*nprocz - Dm.kproc() -1;
symrank = sympz*nprocx*nprocy + Dm.jproc()*nprocx + Dm.iproc();
// DoubleArray SymDist(nx,ny,nz);
char *symid;

45
tests/lbpm_segmented_pp.cpp
View File

@ -12,7 +12,7 @@
#include "common/Array.h"
#include "common/Domain.h"
#include "analysis/TwoPhase.h"
#include "analysis/eikonal.h"
#include "analysis/distance.h"
inline void MeanFilter(DoubleArray &Mesh){
for (int k=1; k<(int)Mesh.size(2)-1; k++){
@ -44,7 +44,6 @@ double ReadFromBlock( char *ID, int iproc, int jproc, int kproc, int Nx, int Ny,
char sx[2];
char sy[2];
char sz[2];
char tmpstr[10];
// array to store ids read from block
char *id;
@ -62,9 +61,8 @@ double ReadFromBlock( char *ID, int iproc, int jproc, int kproc, int Nx, int Ny,
//fflush(stdout);
// Read the file
size_t readID;
FILE *IDFILE = fopen(LocalRankFilename,"rb");
readID=fread(id,1,1024*1024*1024,IDFILE);
fread(id,1,1024*1024*1024,IDFILE);
fclose(IDFILE);
//printf("Loading data ... \n");
@ -128,7 +126,6 @@ int main(int argc, char **argv)
//.......................................................................
int nprocx, nprocy, nprocz, nx, ny, nz, nspheres;
double Lx, Ly, Lz;
int Nx,Ny,Nz;
int i,j,k,n;
int BC=0;
// char fluidValue,solidValue;
@ -189,9 +186,11 @@ int main(int argc, char **argv)
char LocalRankFilename[40];
int N = (nx+2)*(ny+2)*(nz+2);
Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
for (n=0; n<N; n++) Dm.id[n]=1;
Dm.CommInit(comm);
std::shared_ptr<Domain> Dm (new Domain(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
for (n=0; n<N; n++) Dm->id[n]=1;
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
nx+=2; ny+=2; nz+=2;
// Read the phase ID
@ -200,7 +199,7 @@ int main(int argc, char **argv)
if (READ_FROM_BLOCK == 0){
size_t readID;
FILE *ID = fopen(LocalRankFilename,"rb");
readID=fread(Dm.id,1,N,ID);
readID=fread(Dm->id,1,N,ID);
if (readID != size_t(N)) printf("lbpm_segmented_pp: Error reading ID \n");
fclose(ID);
}
@ -208,13 +207,13 @@ int main(int argc, char **argv)
// Read from the large block and write the local ID file
if (rank==0) printf("Reading ID file from blocks \n");
fflush(stdout);
porosity = ReadFromBlock(Dm.id,Dm.iproc,Dm.jproc,Dm.kproc,nx,ny,nz);
porosity = ReadFromBlock(Dm->id,Dm->iproc(),Dm->jproc(),Dm->kproc(),nx,ny,nz);
MPI_Barrier(MPI_COMM_WORLD);
if (rank==0) printf("Writing local ID files (poros=%f) \n",porosity);
fflush(stdout);
FILE *ID = fopen(LocalRankFilename,"wb");
fwrite(Dm.id,1,N,ID);
fwrite(Dm->id,1,N,ID);
fclose(ID);
if (rank==0) printf("Succeeded! \n");
fflush(stdout);
@ -222,9 +221,8 @@ int main(int argc, char **argv)
}
// Initialize the domain and communication
char *id;
id = new char [N];
TwoPhase Averages(Dm);
Array<char> id(nx,ny,nz);
//TwoPhase Averages(Dm);
// DoubleArray Distance(nx,ny,nz);
// DoubleArray Phase(nx,ny,nz);
@ -235,8 +233,8 @@ int main(int argc, char **argv)
for (i=0;i<nx;i++){
n = k*nx*ny+j*nx+i;
// Initialize the solid phase
if (Dm.id[n] == 0) id[n] = 0;
else id[n] = 1;
if (Dm->id[n] == 0) id(i,j,k) = 0;
else id(i,j,k) = 1;
}
}
}
@ -246,25 +244,18 @@ int main(int argc, char **argv)
for (i=0;i<nx;i++){
n=k*nx*ny+j*nx+i;
// Initialize distance to +/- 1
Averages.SDs(i,j,k) = 2.0*double(id[n])-1.0;
Averages->SDs(i,j,k) = 2.0*double(id(i,j,k))-1.0;
}
}
}
MeanFilter(Averages.SDs);
MeanFilter(Averages->SDs);
double LocalVar, TotalVar;
if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
int Maxtime=10*max(max(Dm.Nx*Dm.nprocx,Dm.Ny*Dm.nprocy),Dm.Nz*Dm.nprocz);
Maxtime=min(Maxtime,MAXTIME);
LocalVar = Eikonal(Averages.SDs,id,Dm,Maxtime);
MPI_Allreduce(&LocalVar,&TotalVar,1,MPI_DOUBLE,MPI_SUM,comm);
TotalVar /= nprocs;
if (rank==0) printf("Final variation in signed distance function %f \n",TotalVar);
CalcDist(Averages->SDs,id,*Dm);
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,N,DIST);
fwrite(Averages->SDs.data(),8,N,DIST);
fclose(DIST);
}

70
tests/lbpm_sphere_pp.cpp
View File

@ -8,6 +8,7 @@
#include "analysis/pmmc.h"
#include "common/Domain.h"
#include "common/SpherePack.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
@ -19,7 +20,6 @@
using namespace std;
int main(int argc, char **argv)
{
//*****************************************
@ -32,7 +32,6 @@ int main(int argc, char **argv)
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&nprocs);
// parallel domain size (# of sub-domains)
int nprocx,nprocy,nprocz;
int iproc,jproc,kproc;
int sendtag,recvtag;
//*****************************************
@ -52,53 +51,27 @@ int main(int argc, char **argv)
printf("********************************************************\n");
}
// Variables that specify the computational domain
string FILENAME;
unsigned int nBlocks, nthreads;
int Nx,Ny,Nz; // local sub-domain size
int nspheres; // number of spheres in the packing
double Lx,Ly,Lz; // Domain length
double D = 1.0; // reference length for non-dimensionalization
// Load inputs
string FILENAME = argv[1];
// Load inputs
if (rank==0) printf("Loading input database \n");
auto db = std::make_shared<Database>( FILENAME );
auto domain_db = db->getDatabase( "Domain" );
int Nx = domain_db->getVector<int>( "n" )[0];
int Ny = domain_db->getVector<int>( "n" )[1];
int Nz = domain_db->getVector<int>( "n" )[2];
int nprocx = domain_db->getVector<int>( "nproc" )[0];
int nprocy = domain_db->getVector<int>( "nproc" )[1];
int nprocz = domain_db->getVector<int>( "nproc" )[2];
int nspheres = domain_db->getScalar<int>( "nsphere" );
int Lx = domain_db->getVector<double>( "L" )[0];
int Ly = domain_db->getVector<double>( "L" )[1];
int Lz = domain_db->getVector<double>( "L" )[2];
int i,j,k,n;
if (rank==0){
//.......................................................................
// Reading the domain information file
//.......................................................................
ifstream domain("Domain.in");
domain >> nprocx;
domain >> nprocy;
domain >> nprocz;
domain >> Nx;
domain >> Ny;
domain >> Nz;
domain >> nspheres;
domain >> Lx;
domain >> Ly;
domain >> Lz;
//.......................................................................
}
// **************************************************************
// Broadcast simulation parameters from rank 0 to all other procs
MPI_Barrier(comm);
//.................................................
// Computational domain
MPI_Bcast(&Nx,1,MPI_INT,0,comm);
MPI_Bcast(&Ny,1,MPI_INT,0,comm);
MPI_Bcast(&Nz,1,MPI_INT,0,comm);
MPI_Bcast(&nprocx,1,MPI_INT,0,comm);
MPI_Bcast(&nprocy,1,MPI_INT,0,comm);
MPI_Bcast(&nprocz,1,MPI_INT,0,comm);
MPI_Bcast(&nspheres,1,MPI_INT,0,comm);
MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm);
MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm);
//.................................................
MPI_Barrier(comm);
// **************************************************************
if (nprocs != nprocx*nprocy*nprocz){
printf("nprocx = %i \n",nprocx);
printf("nprocy = %i \n",nprocy);
@ -106,13 +79,6 @@ int main(int argc, char **argv)
INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!");
}
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz,
rank_yz, rank_YZ, rank_yZ, rank_Yz );
MPI_Barrier(comm);
Nz += 2;
Nx = Ny = Nz; // Cubic domain

38
tests/lbpm_squaretube_pp.cpp
View File

@ -116,9 +116,9 @@ int main(int argc, char **argv)
}
// Initialized domain and averaging framework for Two-Phase Flow
Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC);
Dm.CommInit(comm);
TwoPhase Averages(Dm);
std::shared_ptr<Domain> Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC));
Dm->CommInit();
std::shared_ptr<TwoPhase> Averages(new TwoPhase(Dm));
InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc,
rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z,
@ -163,17 +163,17 @@ int main(int argc, char **argv)
if (ORIENTATION==0){
// square capillary tube aligned with the x direction
Averages.SDs(i,j,k) = TubeWidth/2 - fabs(j-0.5*Ny);
Averages.SDs(i,j,k) = min(Averages.SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz));
Averages->SDs(i,j,k) = TubeWidth/2 - fabs(j-0.5*Ny);
Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz));
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Dm.iproc*Nx+k<BubbleBottom){
else if (Dm->iproc()*Nx+k<BubbleBottom){
id[n] = 2;
sum++;
}
else if (Dm.iproc*Nx+k<BubbleTop){
else if (Dm->iproc()*Nx+k<BubbleTop){
id[n] = 1;
sum++;
}
@ -184,17 +184,17 @@ int main(int argc, char **argv)
}
else if (ORIENTATION==1){
// square capillary tube aligned with the y direction
Averages.SDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx);
Averages.SDs(i,j,k) = min(Averages.SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz));
Averages->SDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx);
Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz));
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Dm.jproc*Ny+k<BubbleBottom){
else if (Dm->jproc()*Ny+k<BubbleBottom){
id[n] = 2;
sum++;
}
else if (Dm.jproc*Ny+k<BubbleTop){
else if (Dm->jproc()*Ny+k<BubbleTop){
id[n] = 1;
sum++;
}
@ -205,17 +205,17 @@ int main(int argc, char **argv)
}
else { //
// square capillary tube aligned with the z direction
Averages.SDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx);
Averages.SDs(i,j,k) = min(Averages.SDs(i,j,k),TubeWidth/2-fabs(j-0.5*Ny));
Averages->SDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx);
Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(j-0.5*Ny));
// Initialize phase positions
if (Averages.SDs(i,j,k) < 0.0){
if (Averages->SDs(i,j,k) < 0.0){
id[n] = 0;
}
else if (Dm.kproc*Nz+k<BubbleBottom){
else if (Dm->kproc()*Nz+k<BubbleBottom){
id[n] = 2;
sum++;
}
else if (Dm.kproc*Nz+k<BubbleTop){
else if (Dm->kproc()*Nz+k<BubbleTop){
id[n] = 1;
sum++;
}
@ -249,7 +249,7 @@ int main(int argc, char **argv)
sprintf(LocalRankFilename,"SignDist.%05i",rank);
FILE *DIST = fopen(LocalRankFilename,"wb");
fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST);
fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST);
fclose(DIST);
sprintf(LocalRankFilename,"ID.%05i",rank);

2
tests/testCommunication.cpp
View File

@ -221,7 +221,7 @@ int testHalo( MPI_Comm comm, int nprocx, int nprocy, int nprocz, int depth )
}
// Communicate the halo
fillHalo<TYPE> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,depth);
fillHalo<TYPE> fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,depth);
fillData.fill(array);
// Check the results