From 8911435b10eb945529a3b90ba86b73f257a882de Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 5 May 2018 23:42:13 -0400 Subject: [PATCH 001/124] getting ready to switch over to dfh as main color type simulator --- common/ScaLBL.cpp | 4 ++-- gpu/Color.cu | 17 +++++++++++------ tests/lbpm_dfh_simulator.cpp | 34 +++++++++++++++++----------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 37409c57..f2809d07 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -4159,7 +4159,7 @@ void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double if (kproc == 0) { // Set the phase indicator field and density on the z inlet ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N); - ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,0); + //ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,0); } } @@ -4168,7 +4168,7 @@ void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double if (kproc == nprocz-1){ // Set the phase indicator field and density on the Z outlet ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N); - ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,Nz-1); + //ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,Nz-1); } } diff --git a/gpu/Color.cu b/gpu/Color.cu index 47251a43..6bd76534 100644 --- a/gpu/Color.cu +++ b/gpu/Color.cu @@ -136,8 +136,10 @@ __global__ void dvc_ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *D Den[n] = vA; Den[Np+n] = vB; - nm = Map[n]; - Phi[nm] = (vA-vB)/(vA+vB); + //nm = Map[n]; + //Phi[nm] = (vA-vB)/(vA+vB); + Phi[n] = (vA-vB)/(vA+vB); + } } __global__ void dvc_ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np) @@ -151,8 +153,9 @@ __global__ void dvc_ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double Den[Np+n] = vB; //double valB = Den[Np+n]; // mass that reaches inlet is conserved - nm = Map[n]; - Phi[nm] = (vA-vB)/(vA+vB); + //nm = Map[n]; + //Phi[nm] = (vA-vB)/(vA+vB); + Phi[n] = (vA-vB)/(vA+vB); } } @@ -167,8 +170,10 @@ __global__ void dvc_ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double Den[n] = vA; Den[Np+n] = vB; - nm = Map[n]; - Phi[nm] = (vA-vB)/(vA+vB); + //nm = Map[n]; + //Phi[nm] = (vA-vB)/(vA+vB); + Phi[n] = (vA-vB)/(vA+vB); + } } //************************************************************************* diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index d5568b13..a62da5fc 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -492,7 +492,7 @@ int main(int argc, char **argv) ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); ScaLBL_DeviceBarrier(); delete [] TmpMap; - + // Compute the solid interaction potential and copy result to device if (rank==0) printf("Computing solid interaction potential \n"); double *Tmp; @@ -523,14 +523,14 @@ int main(int argc, char **argv) for (int kk=0; kk<5; kk++){ for (int jj=0; jj<5; jj++){ for (int ii=0; ii<5; ii++){ - + int index = kk*25+jj*5+ii; double distval= Dst[index]; int idi=i+ii-2; int idj=j+jj-2; int idk=k+kk-2; - + if (idi < 0) idi=0; if (idj < 0) idj=0; if (idk < 0) idk=0; @@ -540,16 +540,16 @@ int main(int argc, char **argv) int nn = idk*Nx*Ny + idj*Nx + idi; if (!(Mask.id[nn] > 0)){ - double vec_x = double(ii-2); - double vec_y = double(jj-2); - double vec_z = double(kk-2); - - double ALPHA=PhaseLabel[nn]; - double GAMMA=-2.f; - if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance - phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; - phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; - phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; + double vec_x = double(ii-2); + double vec_y = double(jj-2); + double vec_z = double(kk-2); + + double ALPHA=PhaseLabel[nn]; + double GAMMA=-2.f; + if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance + phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; + phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; + phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; } } } @@ -557,17 +557,17 @@ int main(int argc, char **argv) Tmp[idx] = phi_x; Tmp[idx+Np] = phi_y; Tmp[idx+2*Np] = phi_z; - + /* double d = Averages->SDs(n); double dx = Averages->SDs_x(n); double dy = Averages->SDs_y(n); double dz = Averages->SDs_z(n); double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d)); - + Tmp[idx] = value*dx; Tmp[idx+Np] = value*dy; Tmp[idx+2*Np] = value*dz; - */ + */ } } } @@ -642,7 +642,7 @@ int main(int argc, char **argv) // Copy the phase from the GPU -> CPU //........................................................................... ScaLBL_DeviceBarrier(); - ScaLBL_CopyToHost(Averages->Phase.data(),Phi,Np*sizeof(double)); + ScaLBL_Comm.RegularLayout(Map,Phi,Averages->Phase); ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press); ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x); ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y); From 865c0b57771511d7cdd65b4b97b636a2dc1b067e Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 5 May 2018 23:56:58 -0400 Subject: [PATCH 002/124] updated runanalysis for phi --- analysis/runAnalysis.cpp | 5 ++++- tests/lbpm_dfh_simulator.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 43e46b82..27c7d5b4 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -492,10 +492,13 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, matches(type,AnalysisType::IdentifyBlobs) ) { phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); - ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); + //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); + ScaLBL_Comm.RegularLayout(Map,Phi,phase.get()); + } if ( matches(type,AnalysisType::CopyPhaseIndicator) ) { memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double)); + //Averages.ColorToSignedDistance(d_beta,Averages.Phase,Averages.Phase_tplus); } if ( matches(type,AnalysisType::ComputeAverages) ) { diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index a62da5fc..d4daf942 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -751,7 +751,7 @@ int main(int argc, char **argv) PROFILE_STOP("Update"); // Run the analysis - // analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); + analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); } analysis.finish(); From 1ad1b0811910ec1efe74a7a0e1fc82f968384a78 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sun, 6 May 2018 00:22:23 -0400 Subject: [PATCH 003/124] trying to figure out shared pointer workaround --- analysis/runAnalysis.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 27c7d5b4..2943b6a0 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -493,7 +493,9 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, { phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); - ScaLBL_Comm.RegularLayout(Map,Phi,phase.get()); + DoubleArray PHASE(d_N[0],d_N[1],d_N[2])); + ScaLBL_Comm.RegularLayout(Map,Phi,PHASE); + memcpy(PHASE.data(),phase->data(),N*sizeof(double)); } if ( matches(type,AnalysisType::CopyPhaseIndicator) ) { From 4fd78883e8b7144b06e60d80fd191c7fef6545cb Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sun, 6 May 2018 00:35:37 -0400 Subject: [PATCH 004/124] refactoring to mkae things work hopefullly --- analysis/runAnalysis.cpp | 4 ++-- common/ScaLBL.cpp | 2 +- common/ScaLBL.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 2943b6a0..f09e8958 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -493,8 +493,8 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, { phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); - DoubleArray PHASE(d_N[0],d_N[1],d_N[2])); - ScaLBL_Comm.RegularLayout(Map,Phi,PHASE); + DoubleArray PHASE(d_N[0],d_N[1],d_N[2]); + d_ScaLBL_Comm.RegularLayout(d_Map,Phi,PHASE); memcpy(PHASE.data(),phase->data(),N*sizeof(double)); } diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index f2809d07..d7db9dc4 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -4125,7 +4125,7 @@ void ScaLBL_Communicator::RecvHalo(double *data){ //................................................................................... } -void ScaLBL_Communicator::RegularLayout(IntArray map, double *data, DoubleArray ®data){ +void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray ®data){ // Gets data from the device and stores in regular layout int i,j,k,n,idx; int Nx = map.size(0); diff --git a/common/ScaLBL.h b/common/ScaLBL.h index caf67012..2a818cf7 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -297,7 +297,7 @@ public: void SendHalo(double *data); void RecvHalo(double *data); void RecvGrad(double *Phi, double *Gradient); - void RegularLayout(IntArray map, double *data, DoubleArray ®data); + void RegularLayout(IntArray map, const double *data, DoubleArray ®data); // Routines to set boundary conditions void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB); From 82d0b6848f3d98618950aa2c81b808425e4fd406 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sun, 6 May 2018 22:17:45 -0400 Subject: [PATCH 005/124] analysis now working with dfh simultor --- analysis/runAnalysis.cpp | 29 ++++++++++++++++++++++++----- tests/lbpm_dfh_simulator.cpp | 5 +++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index f09e8958..a96b8390 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -465,7 +465,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep ) * Run the analysis * ******************************************************************/ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, - double *Pressure, double *Velocity, double *fq, double *Den ) + double *Pressure, double *Velocity, double *fq, double *Den) { int N = d_N[0]*d_N[1]*d_N[2]; @@ -493,10 +493,29 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, { phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); - DoubleArray PHASE(d_N[0],d_N[1],d_N[2]); - d_ScaLBL_Comm.RegularLayout(d_Map,Phi,PHASE); - memcpy(PHASE.data(),phase->data(),N*sizeof(double)); - + // try 2 d_ScaLBL_Comm.RegularLayout(d_Map,Phi,Averages.Phase); + // memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double)); + int Nx = d_N[0]; + int Ny = d_N[1]; + int Nz = d_N[2]; + double *TmpDat; + double value; + TmpDat = new double [d_Np]; + ScaLBL_CopyToHost(&TmpDat[0],&Phi[0], d_Np*sizeof(double)); + for (int k=0; kdata()[n]=value; + } + } + } + } + delete [] TmpDat; } if ( matches(type,AnalysisType::CopyPhaseIndicator) ) { memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double)); diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index d4daf942..1ad4c245 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -96,7 +96,7 @@ int main(int argc, char **argv) int RESTART_INTERVAL=20000; //int ANALYSIS_)INTERVAL=1000; int BLOB_ANALYSIS_INTERVAL=1000; - int timestep = 6; + int timestep = 0; if (rank==0){ //............................................................. @@ -782,7 +782,8 @@ int main(int argc, char **argv) FILE *OUTFILE; sprintf(LocalRankFilename,"Phase.%05i.raw",rank); OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,OUTFILE); + //fwrite(PhaseField.data(),8,N,OUTFILE); + fwrite(Averages->Phase.data(),8,N,OUTFILE); fclose(OUTFILE); PROFILE_STOP("Main"); From d5509ad7e6e2fcd6e3b93519053f5030c7cfe34c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 7 May 2018 06:57:24 -0400 Subject: [PATCH 006/124] fixed bug in analysis --- analysis/runAnalysis.cpp | 59 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index a96b8390..46ed8772 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -437,7 +437,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep ) type |= AnalysisType::IdentifyBlobs; } #endif */ - if ( timestep%d_analysis_interval + 4 == 0 ) { + if ( timestep%d_analysis_interval + 4 == d_analysis_interval ) { // Copy the averages to the CPU (and identify blobs) type |= AnalysisType::CopySimState; type |= AnalysisType::IdentifyBlobs; @@ -487,39 +487,38 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, PROFILE_START("Copy data to host",1); std::shared_ptr phase; if ( matches(type,AnalysisType::CopyPhaseIndicator) || - matches(type,AnalysisType::ComputeAverages) || - matches(type,AnalysisType::CopySimState) || - matches(type,AnalysisType::IdentifyBlobs) ) + matches(type,AnalysisType::ComputeAverages) || + matches(type,AnalysisType::CopySimState) || + matches(type,AnalysisType::IdentifyBlobs) ) { - phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); - //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); - // try 2 d_ScaLBL_Comm.RegularLayout(d_Map,Phi,Averages.Phase); - // memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double)); - int Nx = d_N[0]; - int Ny = d_N[1]; - int Nz = d_N[2]; - double *TmpDat; - double value; - TmpDat = new double [d_Np]; - ScaLBL_CopyToHost(&TmpDat[0],&Phi[0], d_Np*sizeof(double)); - for (int k=0; kdata()[n]=value; - } - } - } - } - delete [] TmpDat; + phase = std::shared_ptr(new DoubleArray(d_N[0],d_N[1],d_N[2])); + //ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double)); + // try 2 d_ScaLBL_Comm.RegularLayout(d_Map,Phi,Averages.Phase); + // memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double)); + int Nx = d_N[0]; + int Ny = d_N[1]; + int Nz = d_N[2]; + double *TmpDat; + double value; + TmpDat = new double [d_Np]; + ScaLBL_CopyToHost(&TmpDat[0],&Phi[0], d_Np*sizeof(double)); + for (int k=0; kdata()[n]=value; + } + } + } + } + delete [] TmpDat; } if ( matches(type,AnalysisType::CopyPhaseIndicator) ) { memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double)); - //Averages.ColorToSignedDistance(d_beta,Averages.Phase,Averages.Phase_tplus); } if ( matches(type,AnalysisType::ComputeAverages) ) { From 4a4029c48d07292616cfa964badf3b45d151f032 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 7 May 2018 20:37:32 -0400 Subject: [PATCH 007/124] updated write time --- analysis/TwoPhase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index c07f4772..72acce40 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -1225,7 +1225,7 @@ void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT) void TwoPhase::PrintAll(int timestep) { if (Dm.rank==0){ - fprintf(TIMELOG,"%i %.5g ",timestep-5,dEs); // change in surface energy + fprintf(TIMELOG,"%i %.5g ",timestep,dEs); // change in surface energy fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw_global,pan_global); // saturation and pressure fprintf(TIMELOG,"%.5g %.5g %.5g ",awn_global,ans_global,aws_global); // interfacial areas fprintf(TIMELOG,"%.5g %.5g ",Jwn_global, Kwn_global); // curvature of wn interface @@ -1250,7 +1250,7 @@ void TwoPhase::PrintAll(int timestep) else{ sat_w = 1.0 - nwp_volume/(nwp_volume+wp_volume); - fprintf(TIMELOG,"%i ",timestep-5); // change in surface energy + fprintf(TIMELOG,"%i ",timestep); // change in surface energy fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw,pan); // saturation and pressure fprintf(TIMELOG,"%.5g %.5g %.5g ",awn,ans,aws); // interfacial areas fprintf(TIMELOG,"%.5g %.5g ",Jwn, Kwn); // curvature of wn interface From 72b01d926b10ca1248689a01bb6ae46cc25dd6fa Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 7 May 2018 21:14:01 -0400 Subject: [PATCH 008/124] some minor refactoring --- analysis/runAnalysis.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 46ed8772..bffbfdb8 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -423,6 +423,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep ) AnalysisType type = AnalysisType::AnalyzeNone; if ( timestep%d_analysis_interval + 8 == d_analysis_interval ) { // Copy the phase indicator field for the earlier timestep + printf("Copy phase indicator,timestep=%i\n",timestep); type |= AnalysisType::CopyPhaseIndicator; } if ( timestep%d_blobid_interval == 0 ) { @@ -439,11 +440,13 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep ) #endif */ if ( timestep%d_analysis_interval + 4 == d_analysis_interval ) { // Copy the averages to the CPU (and identify blobs) + printf("Copy sim state, timestep=%i \n",timestep); type |= AnalysisType::CopySimState; type |= AnalysisType::IdentifyBlobs; } if ( timestep%d_analysis_interval == 0 ) { // Run the analysis + printf("Compute averages, timestep=%i \n",timestep); type |= AnalysisType::ComputeAverages; } if (timestep%d_restart_interval == 0) { @@ -574,7 +577,9 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, } // Spawn threads to do the analysis work - if ( matches(type,AnalysisType::ComputeAverages) ) { + //if (timestep%d_restart_interval==0){ + if ( matches(type,AnalysisType::ComputeAverages) ) { + //if ( timestep%d_analysis_interval == 0 ) { auto work = new AnalysisWorkItem(type,timestep,Averages,d_last_index,d_last_id_map,d_beta); work->add_dependency(d_wait_blobID); work->add_dependency(d_wait_analysis); From f320f4c8bec284481f69dd8118cb363ebbcfebbb Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 12 May 2018 20:54:14 -0400 Subject: [PATCH 009/124] pre-processor allows multiple solid labels --- tests/lbpm_segmented_decomp.cpp | 2 +- tests/lbpm_segmented_pp.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index 05770f25..c612ee8c 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -274,7 +274,7 @@ int main(int argc, char **argv) n=k*nx*ny+j*nx+i; total++; Dm.id[n] = newIDs[n]; - if (Dm.id[n] == 0){ + if (!(Dm.id[n] > 0)){ count++; } } diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 3bdbf094..8024c903 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -235,7 +235,7 @@ int main(int argc, char **argv) for (i=0;i 0)) id[n] = 0; else id[n] = 1; } } From 0ceb0b51c7b0d652db122327afa4c07d48c2823c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 12 May 2018 21:07:30 -0400 Subject: [PATCH 010/124] pre-processor allows multiple solid labels --- tests/lbpm_segmented_pp.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 8024c903..e3d1d558 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -228,7 +228,6 @@ int main(int argc, char **argv) // DoubleArray Distance(nx,ny,nz); // DoubleArray Phase(nx,ny,nz); - int count = 0; // Solve for the position of the solid phase for (k=0;k 0)){ + count++; + } + } + } + } + MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); + float porosity = float(totalGlobal-countGlobal)/totalGlobal; + if (rank==0) printf("Porosity=%f\n",porosity); + + // Initialize the signed distance function for (k=0;k Date: Sat, 12 May 2018 21:46:48 -0400 Subject: [PATCH 011/124] fixing --- tests/lbpm_segmented_pp.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index e3d1d558..605fe331 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -243,7 +243,8 @@ int main(int argc, char **argv) int count = 0; int total = 0; - int totalGlobal = nx*ny*nz*nprocs; + int totalGlobal = 0; + int countGlobal = 0; for (k=1;k Date: Sat, 12 May 2018 21:52:37 -0400 Subject: [PATCH 012/124] removed print statement --- tests/lbpm_segmented_decomp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index c612ee8c..c6804988 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -307,7 +307,7 @@ int main(int argc, char **argv) } } float porosity = float(totalGlobal-countGlobal)/totalGlobal; - printf("Original Porosity=%f\n",porosity); + //printf("Original Porosity=%f\n",porosity); } count = 0; From d3e91bc8295534a9e4c1e536dc8a2b21cd395ffd Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Tue, 15 May 2018 10:01:14 -0400 Subject: [PATCH 013/124] Adding Database --- CMakeLists.txt | 13 +- IO/Writer.cpp | 13 +- analysis/{eikonal.hpp => eikonal.cpp} | 29 +- analysis/eikonal.h | 6 +- cmake/macros.cmake | 164 +------ common/Database.cpp | 597 ++++++++++++++++++++++++ common/Database.h | 288 ++++++++++++ common/Database.hpp | 173 +++++++ common/Domain.cpp | 631 +++++++++++++++++++++++--- common/Domain.h | 496 ++------------------ common/Units.cpp | 226 +++++++++ common/Units.h | 140 ++++++ tests/TestColorSquareTube.cpp | 131 ++---- tests/TestCommD3Q19.cpp | 120 ++--- tests/TestForceMoments.cpp | 82 +--- tests/TestInterfaceSpeed.cpp | 31 +- tests/TestPoiseuille.cpp | 118 ++--- tests/TestPressVel.cpp | 92 ++-- tests/TestSegDist.cpp | 56 ++- tests/TestTorus.cpp | 85 ++-- tests/TestWriter.cpp | 55 ++- tests/lbpm_captube_pp.cpp | 70 +-- tests/lbpm_dfh_simulator.cpp | 41 +- tests/lbpm_segmented_decomp.cpp | 4 +- tests/lbpm_segmented_pp.cpp | 26 +- 25 files changed, 2412 insertions(+), 1275 deletions(-) rename analysis/{eikonal.hpp => eikonal.cpp} (95%) create mode 100644 common/Database.cpp create mode 100644 common/Database.h create mode 100644 common/Database.hpp create mode 100644 common/Units.cpp create mode 100644 common/Units.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 88f76b88..3ec21350 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ SET( TEST_MAX_PROCS 16 ) # Initialize the project -PROJECT( ${PROJ} ) +PROJECT( ${PROJ} LANGUAGES CXX ) # Prevent users from building in place @@ -25,8 +25,13 @@ ENDIF() # Set the default C++ standard -IF ( NOT CXX_STD ) - SET( CXX_STD 11 ) +SET( CMAKE_CXX_EXTENSIONS OFF ) +IF ( NOT CMAKE_CXX_STANDARD ) + IF ( CXX_STD ) + MESSAGE( WARNING "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" ) + SET( CMAKE_CXX_STANDARD ${CXX_STD} ) + ENDIF() + SET( CMAKE_CXX_STANDARD 11 ) ENDIF() @@ -113,6 +118,8 @@ ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common vi # Check for CUDA +CHECK_ENABLE_FLAG( USE_CUDA 0 ) +NULL_USE( CMAKE_CUDA_FLAGS ) IF ( USE_CUDA ) ADD_DEFINITIONS( -DUSE_CUDA ) ENABLE_LANGUAGE( CUDA ) diff --git a/IO/Writer.cpp b/IO/Writer.cpp index 12bf07d0..bb522cf6 100644 --- a/IO/Writer.cpp +++ b/IO/Writer.cpp @@ -121,7 +121,7 @@ static std::vector writeMeshesOrigFormat( const std::vector N = { mesh.nx, mesh.ny, mesh.nz }; - std::array baseindex = { info.ix, info.jy, info.kz }; - const std::string meshname = database.domains[0].name; + auto meshname = database.domains[0].name; silo::writeUniformMesh<3>( fid, meshname, range, N ); silo::write( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } ); for (size_t i=0; i( var.type ); if ( var.precision == IO::DataType::Double ) { silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type ); @@ -336,10 +335,8 @@ static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename, const IO::MeshDataStruct& mesh, int format ) { - const int level = 0; - int rank = MPI_WORLD_RANK(); // Create the MeshDatabase - IO::MeshDatabase database = getDatabase( filename, mesh, format ); + auto database = getDatabase( filename, mesh, format ); if ( database.meshClass=="PointList" ) { writeSiloPointList( fid, mesh, database ); } else if ( database.meshClass=="TriMesh" ) { @@ -429,7 +426,7 @@ static std::vector writeMeshesSilo( sprintf(fullpath,"%s/%s",path.c_str(),filename); auto fid = silo::open( fullpath, silo::CREATE ); for (size_t i=0; i mesh = meshData[i].mesh; + auto mesh = meshData[i].mesh; meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) ); } silo::close( fid ); diff --git a/analysis/eikonal.hpp b/analysis/eikonal.cpp similarity index 95% rename from analysis/eikonal.hpp rename to analysis/eikonal.cpp index bd539ea1..82d46246 100644 --- a/analysis/eikonal.hpp +++ b/analysis/eikonal.cpp @@ -1,13 +1,10 @@ -#ifndef Eikonal_HPP_INC -#define Eikonal_HPP_INC - #include "analysis/eikonal.h" #include "analysis/imfilter.h" -inline float minmod(float &a, float &b) +static inline float minmod(float &a, float &b) { float value = a; if ( a*b < 0.0) @@ -18,7 +15,7 @@ inline float minmod(float &a, float &b) } -inline double minmod(double &a, double &b){ +static inline double minmod(double &a, double &b){ double value; @@ -33,9 +30,8 @@ inline double minmod(double &a, double &b){ /****************************************************************** * Solve the eikonal equation * ******************************************************************/ - - -inline double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ +double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps) +{ /* * This routine converts the data in the Distance array to a signed distance @@ -180,7 +176,7 @@ inline double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps return GlobalVar; } -inline float Eikonal3D( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps) +float Eikonal3D( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps) { PROFILE_START("Eikonal3D"); @@ -327,7 +323,7 @@ inline float Eikonal3D( Array &Distance, const Array &ID, const Dom /****************************************************************** * A fast distance calculation * ******************************************************************/ -inline bool CalcDist3DIteration( Array &Distance, const Domain &Dm ) +bool CalcDist3DIteration( Array &Distance, const Domain &Dm ) { const float sq2 = sqrt(2.0f); const float sq3 = sqrt(3.0f); @@ -367,7 +363,7 @@ inline bool CalcDist3DIteration( Array &Distance, const Domain &Dm ) } return changed; } -inline void CalcDist3D( Array &Distance, const Array &ID, const Domain &Dm ) +void CalcDist3D( Array &Distance, const Array &ID, const Domain &Dm ) { PROFILE_START("Calc Distance"); // Initialize the distance to be 0 fore the cells adjacent to the interface @@ -402,7 +398,7 @@ inline void CalcDist3D( Array &Distance, const Array &ID, const Dom /****************************************************************** * A fast distance calculation * ******************************************************************/ -inline void CalcDistMultiLevelHelper( Array &Distance, const Domain &Dm ) +void CalcDistMultiLevelHelper( Array &Distance, const Domain &Dm ) { size_t ratio = 4; std::function&)> coarsen = [ratio]( const Array& data ) @@ -440,7 +436,10 @@ inline void CalcDistMultiLevelHelper( Array &Distance, const Domain &Dm ) // Coarsen Array dist(Nx,Ny,Nz); fillData.copy(Distance,dist); - Domain Dm2(Nx/ratio,Ny/ratio,Nz/ratio,Dm.rank,Dm.nprocx,Dm.nprocy,Dm.nprocz,Dm.Lx,Dm.Ly,Dm.Lz,0); + auto db = Dm.getDatabase()->cloneDatabase(); + auto n = db->getVector( "n" ); + db->putVector( "n", { n[0]/ratio, n[1]/ratio, n[2]/ratio } ); + Domain Dm2(db); Dm2.CommInit(Dm.Comm); fillHalo fillData2(Dm2.Comm,Dm2.rank_info,Nx/ratio,Ny/ratio,Nz/ratio,1,1,1,0,1); auto dist2 = dist.coarsen( {ratio,ratio,ratio}, coarsen ); @@ -483,7 +482,7 @@ inline void CalcDistMultiLevelHelper( Array &Distance, const Domain &Dm ) } } } -inline void CalcDistMultiLevel( Array &Distance, const Array &ID, const Domain &Dm ) +void CalcDistMultiLevel( Array &Distance, const Array &ID, const Domain &Dm ) { PROFILE_START("Calc Distance Multilevel"); int Nx = Dm.Nx-2; @@ -515,5 +514,3 @@ inline void CalcDistMultiLevel( Array &Distance, const Array &ID, c Distance = imfilter::imfilter_separable( Distance, {H,H,H}, BC ); PROFILE_STOP("Calc Distance Multilevel"); } - -#endif diff --git a/analysis/eikonal.h b/analysis/eikonal.h index 6cc0896c..5157e1c5 100644 --- a/analysis/eikonal.h +++ b/analysis/eikonal.h @@ -1,6 +1,7 @@ #ifndef Eikonal_H_INC #define Eikonal_H_INC +#include "common/Domain.h" /*! @@ -21,7 +22,7 @@ * @param[in] timesteps Maximum number of timesteps to process * @return Returns the global variation */ -inline double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps); +double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps); float Eikonal3D( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps); @@ -52,7 +53,4 @@ void CalcDist3D( Array &Distance, const Array &ID, const Domain &Dm void CalcDistMultiLevel( Array &Distance, const Array &ID, const Domain &Dm ); - -#include "analysis/eikonal.hpp" - #endif diff --git a/cmake/macros.cmake b/cmake/macros.cmake index ddb8db85..374fcd66 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -35,11 +35,14 @@ IF ( ${CMAKE_BUILD_TYPE} STREQUAL "Release" AND ${CMAKE_VERSION} VERSION_GREATER CHECK_IPO_SUPPORTED(RESULT supported OUTPUT error) IF( supported ) MESSAGE(STATUS "IPO / LTO enabled") + SET( LTO TRUE ) SET( CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE ) ELSE() - MESSAGE(STATUS "IPO / LTO not supported: <${error}>") + SET( LTO FALSE ) + MESSAGE(STATUS "IPO / LTO not supported") ENDIF() ELSEIF( NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION ) + SET( LTO FALSE ) MESSAGE(STATUS "IPO / LTO disabled") ENDIF() @@ -320,7 +323,7 @@ MACRO( INSTALL_PROJ_LIB ) FOREACH ( tmp ${${PROJ}_LIBS} ) SET( tmp_link_list ${tmp_link_list} $ ) ENDFOREACH() - ADD_LIBRARY( ${${PROJ}_LIB} ${tmp_link_list} ) + ADD_LIBRARY( ${${PROJ}_LIB} ${LIB_TYPE} ${tmp_link_list} ) TARGET_LINK_EXTERNAL_LIBRARIES( ${${PROJ}_LIB} LINK_PUBLIC ) INSTALL( TARGETS ${${PROJ}_LIB} DESTINATION ${${PROJ}_INSTALL_DIR}/lib ) ENDMACRO() @@ -408,6 +411,9 @@ MACRO( IDENTIFY_COMPILER ) IF( CMAKE_COMPILER_IS_GNUG77 OR (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") ) SET( USING_GFORTRAN TRUE ) MESSAGE("Using gfortran") + IF ( NOT USING_GCC ) + LIST( REMOVE_ITEM CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES gcc ) + ENDIF() ELSEIF ( (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") ) SET(USING_IFORT TRUE) MESSAGE("Using ifort") @@ -477,76 +483,6 @@ MACRO( ADD_USER_FLAGS ) ENDMACRO() -# Macro to add user c++ std -MACRO( ADD_CXX_STD ) - IF ( NOT CXX_STD ) - MESSAGE( FATAL_ERROR "The desired c++ standard must be specified: CXX_STD=(98,11,14,NONE)") - ENDIF() - IF ( ${CXX_STD} STREQUAL "NONE" ) - # Do nothing - return() - ELSEIF ( (NOT ${CXX_STD} STREQUAL "98") AND (NOT ${CXX_STD} STREQUAL "11") AND (NOT ${CXX_STD} STREQUAL "14") ) - MESSAGE( FATAL_ERROR "Unknown c++ standard (98,11,14,NONE)" ) - ENDIF() - # Add the flags - SET( CMAKE_CXX_STANDARD ${CXX_STD} ) - MESSAGE( "C++ standard: ${CXX_STD}" ) - SET( CXX_STD_FLAG ) - IF ( USING_GCC ) - # GNU: -std= - IF ( ${CXX_STD} STREQUAL "98" ) - SET( CXX_STD_FLAG -std=c++98 ) - ELSEIF ( ${CXX_STD} STREQUAL "11" ) - SET( CXX_STD_FLAG -std=c++11 ) - ELSEIF ( ${CXX_STD} STREQUAL "14" ) - SET( CXX_STD_FLAG -std=c++1y ) - ELSE() - MESSAGE(FATAL_ERROR "Unknown standard") - ENDIF() - ELSEIF ( USING_MSVC ) - # Microsoft: Does not support this level of control - ELSEIF ( USING_ICC ) - # ICC: -std= - SET( CXX_STD_FLAG -std=c++${CXX_STD} ) - ELSEIF ( USING_CRAY ) - # Cray: Does not seem to support controlling the std? - ELSEIF ( USING_PGCC ) - # PGI: -std= - IF ( ${CXX_STD} STREQUAL "98" ) - SET( CXX_STD_FLAG --c++0x ) - ELSEIF ( ${CXX_STD} STREQUAL "11" ) - SET( CXX_STD_FLAG --c++11 ) - ELSEIF ( ${CXX_STD} STREQUAL "14" ) - MESSAGE( FATAL_ERROR "C++14 features are not availible yet for PGI" ) - ELSE() - MESSAGE(FATAL_ERROR "Unknown standard") - ENDIF() - ELSEIF ( USING_CLANG ) - # Clang: -std= - IF ( ( ${CXX_STD} STREQUAL "98") OR ( ${CXX_STD} STREQUAL "11" ) ) - SET( CXX_STD_FLAG -std=c++${CXX_STD} ) - ELSEIF ( ${CXX_STD} STREQUAL "14" ) - SET( CXX_STD_FLAG -std=c++1y ) - ELSE() - MESSAGE(FATAL_ERROR "Unknown standard") - ENDIF() - ELSEIF ( USING_XL ) - # XL: -std= - IF ( ( ${CXX_STD} STREQUAL "98") OR ( ${CXX_STD} STREQUAL "11" ) ) - SET( CXX_STD_FLAG -std=c++${CXX_STD} ) - ELSEIF ( ${CXX_STD} STREQUAL "14" ) - SET( CXX_STD_FLAG -std=c++1y ) - ELSE() - MESSAGE(FATAL_ERROR "Unknown standard") - ENDIF() - ELSEIF ( USING_DEFAULT ) - # Default: do nothing - ENDIF() - ADD_DEFINITIONS( -DCXX_STD=${CXX_STD} ) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_STD_FLAG}") -ENDMACRO() - - # Macro to set the compile/link flags MACRO( SET_COMPILER_FLAGS ) # Initilaize the compiler @@ -585,8 +521,6 @@ MACRO( SET_COMPILER_FLAGS ) SET( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Release flags" FORCE) # Add the user flags ADD_USER_FLAGS() - # Add the c++ standard flags - ADD_CXX_STD() # Set the warnings to use SET_WARNINGS() # Test the compile flags @@ -1098,71 +1032,19 @@ ENDMACRO() # Add a matlab mex file -FUNCTION( ADD_MATLAB_MEX MEXFILE ) - # Set the MEX compiler and default link flags - IF ( USING_MSVC ) - SET( MEX_LDFLAGS ${MEX_LDFLAGS} -L${CMAKE_CURRENT_BINARY_DIR}/.. - -L${CMAKE_CURRENT_BINARY_DIR}/../Debug -L${CMAKE_CURRENT_BINARY_DIR} ) - ENDIF() - SET( MEX_LDFLAGS ${MEX_LDFLAGS} -L${CMAKE_CURRENT_BINARY_DIR}/.. -L${CMAKE_CURRENT_BINARY_DIR}/../.. ${SYSTEM_LDFLAGS} ) - SET( MEX_LIBS ${MEX_LIBS} ${SYSTEM_LIBS} ) - IF ( NOT USING_MSVC ) - FOREACH( rpath ${CMAKE_INSTALL_RPATH} ) - SET( MEX_LDFLAGS ${MEX_LDFLAGS} "-Wl,-rpath,${rpath},--no-undefined" ) - ENDFOREACH() - ENDIF() - IF ( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" ) - SET( MEX_FLAGS ${MEX_FLAGS} -g ) - ELSEIF ( "${CMAKE_BUILD_TYPE}" STREQUAL "Release" ) - SET( MEX_FLAGS ${MEX_FLAGS} -O ) - ELSE() - MESSAGE( FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}" ) - ENDIF() - # Create the mex comamnd - STRING(REGEX REPLACE "[.]cpp" "" TARGET ${MEXFILE}) - STRING(REGEX REPLACE "[.]c" "" TARGET ${TARGET}) - FILE(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/temp/${TARGET}" ) - IF ( USING_MSVC ) - SET( MEX_BAT_FILE "${CMAKE_CURRENT_BINARY_DIR}/temp/${TARGET}/compile_mex.bat" ) - FILE( WRITE "${MEX_BAT_FILE}" "${MEX_EXE} \"${CMAKE_CURRENT_SOURCE_DIR}/${MEXFILE}\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${MEX_FLAGS}" "\"" "\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${COMPFLAGS}" "\"" "\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${MEX_INCLUDE}" "\"" "\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${MEX_LDFLAGS}" "\"" "\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${MEX_LIBS}" "\"" "\" " ) - APPEND_LIST( "${MEX_BAT_FILE}" "${COVERAGE_MATLAB_LIBS}" "\"" "\" " ) - SET( MEX_COMMAND "${MEX_BAT_FILE}" ) - ELSE() - STRING(REPLACE " " ";" MEX_CFLAGS "$$CFLAGS ${CMAKE_C_FLAGS}") - STRING(REPLACE " " ";" MEX_CXXFLAGS "$$CXXFLAGS ${CMAKE_CXX_FLAGS}") - SET( MEX_COMMAND ${MEX_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/${MEXFILE} - ${MEX_FLAGS} ${MEX_INCLUDE} - CFLAGS="${MEX_CFLAGS}" - CXXFLAGS="${MEX_CXXFLAGS}" - LDFLAGS="${MEX_LDFLAGS}" - ${MEX_LIBS} ${COVERAGE_MATLAB_LIBS} - ) - ENDIF() - ADD_CUSTOM_COMMAND( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.${MEX_EXTENSION} - COMMAND ${MEX_COMMAND} - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/temp/${TARGET}/${TARGET}.${MEX_EXTENSION}" "${CMAKE_CURRENT_BINARY_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/temp/${TARGET}/${TARGET}.${MEX_EXTENSION}" "${${PROJ}_INSTALL_DIR}/mex" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/temp/${TARGET} - DEPENDS ${${PROJ}_LIBS} ${MATLAB_TARGET} ${CMAKE_CURRENT_SOURCE_DIR}/${MEXFILE} +FUNCTION( ADD_MATLAB_MEX SOURCE ) + STRING( REGEX REPLACE "[.]cpp" "" TARGET ${SOURCE} ) + STRING( REGEX REPLACE "[.]c" "" TARGET ${TARGET} ) + MATLAB_ADD_MEX( + NAME ${TARGET} + SRC ${SOURCE} ) - ADD_CUSTOM_TARGET( ${TARGET} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.${MEX_EXTENSION} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${MEXFILE} - ) - IF ( MATLAB_TARGET ) - ADD_DEPENDENCIES( ${TARGET} ${MATLAB_TARGET} ) - ENDIF() - IF ( ${PROJ}_LIBS ) - ADD_DEPENDENCIES( ${TARGET} ${${PROJ}_LIBS} ) - ENDIF() + TARGET_LINK_LIBRARIES( ${TARGET} ${MATLAB_TARGET} ) + ADD_PROJ_EXE_DEP( ${TARGET} ) ADD_DEPENDENCIES( mex ${TARGET} ) - SET( MEX_FILES2 ${MEX_FILES} "${${PROJ}_INSTALL_DIR}/mex/${TARGET}.${MEX_EXTENSION}" ) + INSTALL( TARGETS ${TARGET} DESTINATION ${${PROJ}_INSTALL_DIR}/mex ) + ADD_DEPENDENCIES( mex ${TARGET} ) + SET( MEX_FILES2 ${MEX_FILES} "${${PROJ}_INSTALL_DIR}/mex/${TARGET}.${Matlab_MEX_EXTENSION}" ) LIST( REMOVE_DUPLICATES MEX_FILES2 ) SET( MEX_FILES ${MEX_FILES2} CACHE INTERNAL "" ) ENDFUNCTION() @@ -1178,7 +1060,7 @@ MACRO( ADD_MATLAB_TEST EXEFILE ${ARGN} ) IF ( USING_MSVC ) SET( MATLAB_OPTIONS "-logfile" "log_${EXEFILE}" ) ENDIF() - SET( MATLAB_COMMAND "try, ${EXEFILE}, catch ME, ME, clear all global, exit(1), end, disp('ALL TESTS PASSED'); exit(0)" ) + SET( MATLAB_COMMAND "try, ${EXEFILE}, catch ME, disp(getReport(ME)), clear all global, exit(1), end, disp('ALL TESTS PASSED'); exit(0)" ) SET( MATLAB_DEBUGGER_OPTIONS ) IF ( MATLAB_DEBUGGER ) SET( MATLAB_DEBUGGER_OPTIONS -D${MATLAB_DEBUGGER} ) @@ -1195,7 +1077,6 @@ FUNCTION( CREATE_MATLAB_WRAPPER ) SET( tmp_libs ${MEX_LIBCXX} ${MEX_FILES} ) STRING(REGEX REPLACE ";" ":" tmp_libs "${tmp_libs}") STRING(REGEX REPLACE ";" ":" tmp_path "${MATLABPATH}") - FIND_PROGRAM( MATLAB_EXE2 NAME matlab PATHS "${MATLAB_DIRECTORY}/bin" NO_DEFAULT_PATH ) IF ( USING_MSVC ) # Create a matlab wrapper for windows SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui.bat" ) @@ -1212,8 +1093,8 @@ FUNCTION( CREATE_MATLAB_WRAPPER ) SET( MATLAB_GUI "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-gui" ) SET( MATLAB_CMD "${CMAKE_CURRENT_BINARY_DIR}/tmp/matlab-cmd" ) SET( MATLAB_INSTALL_CMD "matlab-cmd" ) - FILE( WRITE "${MATLAB_GUI}" "LD_PRELOAD=\"${tmp_libs}\" MKL_NUM_THREADS=1 MATLABPATH=\"${tmp_path}\" \"${MATLAB_EXE2}\" -singleCompThread -nosplash \"$@\"\n") - FILE( WRITE "${MATLAB_CMD}" "LD_PRELOAD=\"${tmp_libs}\" MKL_NUM_THREADS=1 MATLABPATH=\"${tmp_path}\" \"${MATLAB_EXE2}\" -singleCompThread -nosplash -nodisplay -nojvm \"$@\"\n") + FILE( WRITE "${MATLAB_GUI}" "LD_PRELOAD=\"${tmp_libs}\" MKL_NUM_THREADS=1 MATLABPATH=\"${tmp_path}\" \"${Matlab_MAIN_PROGRAM}\" -singleCompThread -nosplash \"$@\"\n") + FILE( WRITE "${MATLAB_CMD}" "LD_PRELOAD=\"${tmp_libs}\" MKL_NUM_THREADS=1 MATLABPATH=\"${tmp_path}\" \"${Matlab_MAIN_PROGRAM}\" -singleCompThread -nosplash -nodisplay -nojvm \"$@\"\n") ENDIF() FILE( COPY "${MATLAB_GUI}" DESTINATION "${${PROJ}_INSTALL_DIR}/mex" FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) @@ -1294,6 +1175,7 @@ MACRO( ADD_DISTCLEAN ${ARGN} ) install_manifest.txt test matlab + Matlab mex tmp #tmp# diff --git a/common/Database.cpp b/common/Database.cpp new file mode 100644 index 00000000..d318cc03 --- /dev/null +++ b/common/Database.cpp @@ -0,0 +1,597 @@ +#include "common/Database.h" +#include "common/Utilities.h" + +#include +#include +#include +#include +#include +#include + + +/******************************************************************** + * Constructors/destructor * + ********************************************************************/ +Database::Database() = default; +Database::~Database() = default; +Database::Database( const Database& rhs ) : KeyData( rhs ) +{ + d_data.clear(); + for ( const auto& tmp : rhs.d_data ) + putData( tmp.first, tmp.second->clone() ); +} +Database& Database::operator=( const Database& rhs ) +{ + if ( this == &rhs ) + return *this; + d_data.clear(); + for ( const auto& tmp : rhs.d_data ) + putData( tmp.first, tmp.second->clone() ); + return *this; +} +Database::Database( Database&& rhs ) { std::swap( d_data, rhs.d_data ); } +Database& Database::operator=( Database&& rhs ) +{ + if ( this != &rhs ) + std::swap( d_data, rhs.d_data ); + return *this; +} + + +/******************************************************************** + * Clone the database * + ********************************************************************/ +std::shared_ptr Database::clone() const { return cloneDatabase(); } +std::shared_ptr Database::cloneDatabase() const +{ + auto db = std::make_shared(); + for ( const auto& tmp : d_data ) + db->putData( tmp.first, tmp.second->clone() ); + return db; +} + + +/******************************************************************** + * Get the data object * + ********************************************************************/ +bool Database::keyExists( const std::string& key ) const +{ + return d_data.find( key ) != d_data.end(); +} +std::shared_ptr Database::getData( const std::string& key ) +{ + auto it = d_data.find( key ); + if ( it == d_data.end() ) { + char msg[1000]; + sprintf( msg, "Variable %s was not found in database", key.c_str() ); + ERROR( msg ); + } + return it->second; +} +std::shared_ptr Database::getData( const std::string& key ) const +{ + return const_cast( this )->getData( key ); +} +bool Database::isDatabase( const std::string& key ) const +{ + auto ptr = getData( key ); + auto ptr2 = std::dynamic_pointer_cast( ptr ); + return ptr2 != nullptr; +} +std::shared_ptr Database::getDatabase( const std::string& key ) +{ + std::shared_ptr ptr = getData( key ); + std::shared_ptr ptr2 = std::dynamic_pointer_cast( ptr ); + if ( ptr2 == nullptr ) { + char msg[1000]; + sprintf( msg, "Variable %s is not a database", key.c_str() ); + ERROR( msg ); + } + return ptr2; +} +std::shared_ptr Database::getDatabase( const std::string& key ) const +{ + return const_cast( this )->getDatabase( key ); +} +std::vector Database::getAllKeys() const +{ + std::vector keys; + keys.reserve( d_data.size() ); + for ( const auto& it : d_data ) + keys.push_back( it.first ); + return keys; +} +void Database::putDatabase( const std::string& key, std::shared_ptr db ) +{ + d_data[key] = std::move( db ); +} +void Database::putData( const std::string& key, std::shared_ptr data ) +{ + d_data[key] = std::move( data ); +} + + +/******************************************************************** + * Is the data of the given type * + ********************************************************************/ +template<> +bool Database::isType( const std::string& key ) const +{ + auto type = getData( key )->type(); + return type == "double"; +} +template<> +bool Database::isType( const std::string& key ) const +{ + auto type = getData( key )->type(); + return type == "double"; +} +template<> +bool Database::isType( const std::string& key ) const +{ + bool pass = true; + auto type = getData( key )->type(); + if ( type == "double" ) { + auto data = getVector( key ); + for ( auto tmp : data ) + pass = pass && static_cast( static_cast( tmp ) ) == tmp; + } else { + pass = false; + } + return pass; +} +template<> +bool Database::isType( const std::string& key ) const +{ + auto type = getData( key )->type(); + return type == "string"; +} +template<> +bool Database::isType( const std::string& key ) const +{ + auto type = getData( key )->type(); + return type == "bool"; +} + + +/******************************************************************** + * Get a vector * + ********************************************************************/ +template<> +std::vector Database::getVector( + const std::string& key, const Units& ) const +{ + std::shared_ptr ptr = getData( key ); + if ( std::dynamic_pointer_cast( ptr ) ) + return std::vector(); + const auto* ptr2 = dynamic_cast( ptr.get() ); + if ( ptr2 == nullptr ) { + ERROR( "Key '" + key + "' is not a string" ); + } + return ptr2->d_data; +} +template<> +std::vector Database::getVector( const std::string& key, const Units& ) const +{ + std::shared_ptr ptr = getData( key ); + if ( std::dynamic_pointer_cast( ptr ) ) + return std::vector(); + const auto* ptr2 = dynamic_cast( ptr.get() ); + if ( ptr2 == nullptr ) { + ERROR( "Key '" + key + "' is not a bool" ); + } + return ptr2->d_data; +} +template +std::vector Database::getVector( const std::string& key, const Units& unit ) const +{ + std::shared_ptr ptr = getData( key ); + if ( std::dynamic_pointer_cast( ptr ) ) + return std::vector(); + std::vector data; + if ( std::dynamic_pointer_cast( ptr ) ) { + const auto* ptr2 = dynamic_cast( ptr.get() ); + const std::vector& data2 = ptr2->d_data; + double factor = 1; + if ( !unit.isNull() ) { + INSIST( !ptr2->d_unit.isNull(), "Field " + key + " must have units" ); + factor = ptr2->d_unit.convert( unit ); + INSIST( factor != 0, "Unit conversion failed" ); + } + data.resize( data2.size() ); + for ( size_t i = 0; i < data2.size(); i++ ) + data[i] = static_cast( factor * data2[i] ); + } else if ( std::dynamic_pointer_cast( ptr ) ) { + ERROR( "Converting std::string to another type" ); + } else if ( std::dynamic_pointer_cast( ptr ) ) { + ERROR( "Converting std::bool to another type" ); + } else { + ERROR( "Unable to convert data format" ); + } + return data; +} + + +/******************************************************************** + * Put a vector * + ********************************************************************/ +template<> +void Database::putVector( + const std::string& key, const std::vector& data, const Units& ) +{ + std::shared_ptr ptr( new KeyDataString() ); + ptr->d_data = data; + d_data[key] = ptr; +} +template<> +void Database::putVector( + const std::string& key, const std::vector& data, const Units& ) +{ + std::shared_ptr ptr( new KeyDataBool() ); + ptr->d_data = data; + d_data[key] = ptr; +} +template +void Database::putVector( const std::string& key, const std::vector& data, const Units& unit ) +{ + std::shared_ptr ptr( new KeyDataDouble() ); + ptr->d_unit = unit; + ptr->d_data.resize( data.size() ); + for ( size_t i = 0; i < data.size(); i++ ) + ptr->d_data[i] = static_cast( data[i] ); + d_data[key] = ptr; +} + + +/******************************************************************** + * Print the database * + ********************************************************************/ +void Database::print( std::ostream& os, const std::string& indent ) const +{ + for ( const auto& it : d_data ) { + os << indent << it.first; + if ( dynamic_cast( it.second.get() ) ) { + const auto* db = dynamic_cast( it.second.get() ); + os << " {\n"; + db->print( os, indent + " " ); + os << indent << "}\n"; + } else { + os << " = "; + it.second->print( os, "" ); + } + } +} +std::string Database::print( const std::string& indent ) const +{ + std::stringstream ss; + print( ss, indent ); + return ss.str(); +} + + +/******************************************************************** + * Read input database file * + ********************************************************************/ +Database::Database( const std::string& filename ) +{ + // Read the input file into memory + FILE* fid = fopen( filename.c_str(), "rb" ); + if ( fid == nullptr ) + ERROR( "Error opening file " + filename ); + fseek( fid, 0, SEEK_END ); + size_t bytes = ftell( fid ); + rewind( fid ); + auto* buffer = new char[bytes + 4]; + size_t result = fread( buffer, 1, bytes, fid ); + fclose( fid ); + if ( result != bytes ) + ERROR( "Error reading file " + filename ); + buffer[bytes + 0] = '\n'; + buffer[bytes + 1] = '}'; + buffer[bytes + 2] = '\n'; + buffer[bytes + 3] = 0; + // Create the database entries + loadDatabase( buffer, *this ); + // Free temporary memory + delete[] buffer; +} +std::shared_ptr Database::createFromString( const std::string& data ) +{ + std::shared_ptr db( new Database() ); + auto* buffer = new char[data.size() + 4]; + memcpy( buffer, data.data(), data.size() ); + buffer[data.size() + 0] = '\n'; + buffer[data.size() + 1] = '}'; + buffer[data.size() + 2] = '\n'; + buffer[data.size() + 3] = 0; + loadDatabase( buffer, *db ); + delete[] buffer; + return db; +} +enum class token_type { + newline, + line_comment, + block_start, + block_stop, + quote, + equal, + bracket, + end_bracket, + end +}; +inline size_t length( token_type type ) +{ + size_t len = 0; + if ( type == token_type::newline || type == token_type::quote || type == token_type::equal || + type == token_type::bracket || type == token_type::end_bracket || + type == token_type::end ) { + len = 1; + } else if ( type == token_type::line_comment || type == token_type::block_start || + type == token_type::block_stop ) { + len = 2; + } + return len; +} +inline std::tuple find_next_token( const char* buffer ) +{ + size_t i = 0; + while ( true ) { + if ( buffer[i] == '\n' || buffer[i] == '\r' ) { + return std::pair( i + 1, token_type::newline ); + } else if ( buffer[i] == 0 ) { + return std::pair( i + 1, token_type::end ); + } else if ( buffer[i] == '"' ) { + return std::pair( i + 1, token_type::quote ); + } else if ( buffer[i] == '=' ) { + return std::pair( i + 1, token_type::equal ); + } else if ( buffer[i] == '{' ) { + return std::pair( i + 1, token_type::bracket ); + } else if ( buffer[i] == '}' ) { + return std::pair( i + 1, token_type::end_bracket ); + } else if ( buffer[i] == '/' ) { + if ( buffer[i + 1] == '/' ) { + return std::pair( i + 2, token_type::line_comment ); + } else if ( buffer[i + 1] == '*' ) { + return std::pair( i + 2, token_type::block_start ); + } + } else if ( buffer[i] == '*' ) { + if ( buffer[i + 1] == '/' ) + return std::pair( i + 2, token_type::block_stop ); + } + i++; + } + return std::pair( 0, token_type::end ); +} +inline std::string deblank( const std::string& str ) +{ + size_t i1 = 0xFFFFFFF, i2 = 0; + for ( size_t i = 0; i < str.size(); i++ ) { + if ( str[i] != ' ' ) { + i1 = std::min( i1, i ); + i2 = std::max( i2, i ); + } + } + return i1 <= i2 ? str.substr( i1, i2 - i1 + 1 ) : std::string(); +} +size_t skip_comment( const char* buffer ) +{ + auto tmp = find_next_token( buffer ); + const token_type end_comment = ( std::get<1>( tmp ) == token_type::line_comment ) ? + token_type::newline : + token_type::block_stop; + size_t pos = 0; + while ( std::get<1>( tmp ) != end_comment ) { + if ( std::get<1>( tmp ) == token_type::end ) + ERROR( "Encountered end of file before block comment end" ); + pos += std::get<0>( tmp ); + tmp = find_next_token( &buffer[pos] ); + } + pos += std::get<0>( tmp ); + return pos; +} +inline std::string lower( const std::string& str ) +{ + std::string tmp( str ); + std::transform( tmp.begin(), tmp.end(), tmp.begin(), ::tolower ); + return tmp; +} +static std::tuple> read_value( + const char* buffer, const std::string& key ) +{ + // Get the value as a std::string + size_t pos = 0; + token_type type = token_type::end; + std::tie( pos, type ) = find_next_token( &buffer[pos] ); + size_t len = pos - length( type ); + while ( type != token_type::newline ) { + if ( type == token_type::quote ) { + size_t i = 0; + std::tie( i, type ) = find_next_token( &buffer[pos] ); + pos += i; + while ( type != token_type::quote ) { + ASSERT( type != token_type::end ); + std::tie( i, type ) = find_next_token( &buffer[pos] ); + pos += i; + } + } else if ( type == token_type::line_comment || type == token_type::block_start ) { + len = pos - length( type ); + pos += skip_comment( &buffer[pos - length( type )] ) - length( type ); + break; + } + size_t i = 0; + std::tie( i, type ) = find_next_token( &buffer[pos] ); + pos += i; + len = pos - length( type ); + } + const std::string value = deblank( std::string( buffer, len ) ); + // Split the value to an array of values + std::vector values; + size_t i0 = 0, i = 0, count = 0; + for ( ; i < value.size(); i++ ) { + if ( value[i] == '"' ) { + count++; + } else if ( value[i] == ',' && count % 2 == 0 ) { + values.push_back( deblank( value.substr( i0, i - i0 ) ) ); + i0 = i + 1; + } + } + values.push_back( deblank( value.substr( i0 ) ) ); + // Convert the string value to the database value + std::shared_ptr data; + if ( value.empty() ) { + data.reset( new EmptyKeyData() ); + } else if ( value.find( '"' ) != std::string::npos ) { + auto* data2 = new KeyDataString(); + data.reset( data2 ); + data2->d_data.resize( values.size() ); + for ( size_t i = 0; i < values.size(); i++ ) { + ASSERT( values[i].size() >= 2 ); + ASSERT( values[i][0] == '"' && values[i][values[i].size() - 1] == '"' ); + data2->d_data[i] = values[i].substr( 1, values[i].size() - 2 ); + } + } else if ( lower( value ) == "true" || lower( value ) == "false" ) { + auto* data2 = new KeyDataBool(); + data.reset( data2 ); + data2->d_data.resize( values.size() ); + for ( size_t i = 0; i < values.size(); i++ ) { + ASSERT( values[i].size() >= 2 ); + if ( lower( values[i] ) != "true" && lower( values[i] ) != "false" ) + ERROR( "Error converting " + key + " to logical array" ); + data2->d_data[i] = lower( values[i] ) == "true"; + } + } else { // if ( value.find('.')!=std::string::npos || value.find('e')!=std::string::npos ) { + auto* data2 = new KeyDataDouble(); + data.reset( data2 ); + data2->d_data.resize( values.size(), 0 ); + for ( size_t i = 0; i < values.size(); i++ ) { + Units unit; + std::tie( data2->d_data[i], unit ) = KeyDataDouble::read( values[i] ); + if ( !unit.isNull() ) + data2->d_unit = unit; + } + //} else { + // ERROR("Unable to determine data type: "+value); + } + return std::tuple>( pos, data ); +} +size_t Database::loadDatabase( const char* buffer, Database& db ) +{ + size_t pos = 0; + while ( true ) { + size_t i; + token_type type; + std::tie( i, type ) = find_next_token( &buffer[pos] ); + const std::string key = + deblank( std::string( &buffer[pos], std::max( i - length( type ), 1 ) - 1 ) ); + if ( type == token_type::line_comment || type == token_type::block_start ) { + // Comment + INSIST( key.empty(), "Key should be empty: " + key ); + pos += skip_comment( &buffer[pos] ); + } else if ( type == token_type::newline ) { + INSIST( key.empty(), "Key should be empty: " + key ); + pos += i; + } else if ( type == token_type::equal ) { + // Reading key/value pair + ASSERT( !key.empty() ); + pos += i; + std::shared_ptr data; + std::tie( i, data ) = read_value( &buffer[pos], key ); + ASSERT( data.get() != nullptr ); + db.d_data[key] = data; + pos += i; + } else if ( type == token_type::bracket ) { + // Read database + ASSERT( !key.empty() ); + pos += i; + std::shared_ptr database( new Database() ); + pos += loadDatabase( &buffer[pos], *database ); + db.d_data[key] = database; + } else if ( type == token_type::end_bracket ) { + // Finished with the database + pos += i; + break; + } else { + ERROR( "Error loading data" ); + } + } + return pos; +} + + +/******************************************************************** + * Data type helper functions * + ********************************************************************/ +void KeyDataDouble::print( std::ostream& os, const std::string& indent ) const +{ + os << indent; + for ( size_t i = 0; i < d_data.size(); i++ ) { + if ( i > 0 ) + os << ", "; + if ( d_data[i] != d_data[i] ) { + os << "nan"; + } else if ( d_data[i] == std::numeric_limits::infinity() ) { + os << "inf"; + } else if ( d_data[i] == -std::numeric_limits::infinity() ) { + os << "-inf"; + } else { + os << std::setprecision( 12 ) << d_data[i]; + } + } + if ( !d_unit.isNull() ) + os << " " << d_unit.str(); + os << std::endl; +} +std::tuple KeyDataDouble::read( const std::string& str ) +{ + std::string tmp = deblank( str ); + size_t index = tmp.find( " " ); + if ( index != std::string::npos ) { + return std::make_tuple( + readValue( tmp.substr( 0, index ) ), Units( tmp.substr( index + 1 ) ) ); + } else { + return std::make_tuple( readValue( tmp ), Units() ); + } +} +double KeyDataDouble::readValue( const std::string& str ) +{ + const std::string tmp = lower( str ); + double data = 0; + if ( tmp == "inf" || tmp == "infinity" ) { + data = std::numeric_limits::infinity(); + } else if ( tmp == "-inf" || tmp == "-infinity" ) { + data = -std::numeric_limits::infinity(); + } else if ( tmp == "nan" ) { + data = std::numeric_limits::quiet_NaN(); + } else if ( tmp.find( '/' ) != std::string::npos ) { + ERROR( "Error reading value" ); + } else { + char* pos = nullptr; + data = strtod( tmp.c_str(), &pos ); + if ( static_cast( pos - tmp.c_str() ) == tmp.size() + 1 ) + ERROR( "Error reading value" ); + } + return data; +} + + +/******************************************************************** + * Instantiations * + ********************************************************************/ +template std::vector Database::getVector( const std::string&, const Units& ) const; +template std::vector Database::getVector( const std::string&, const Units& ) const; +template std::vector Database::getVector( const std::string&, const Units& ) const; +template std::vector Database::getVector( const std::string&, const Units& ) const; +template std::vector Database::getVector( const std::string&, const Units& ) const; +template void Database::putVector( + const std::string&, const std::vector&, const Units& ); +template void Database::putVector( const std::string&, const std::vector&, const Units& ); +template void Database::putVector( + const std::string&, const std::vector&, const Units& ); +template void Database::putVector( + const std::string&, const std::vector&, const Units& ); +template void Database::putVector( + const std::string&, const std::vector&, const Units& ); +template bool Database::isType( const std::string& ) const; +template bool Database::isType( const std::string& ) const; +template bool Database::isType( const std::string& ) const; +template bool Database::isType( const std::string& ) const; diff --git a/common/Database.h b/common/Database.h new file mode 100644 index 00000000..4bf0bf61 --- /dev/null +++ b/common/Database.h @@ -0,0 +1,288 @@ +#ifndef included_Database +#define included_Database + +#include +#include +#include +#include +#include + +#include "common/Units.h" + + +//! Base class to hold data of a given type +class KeyData +{ +protected: + //! Empty constructor + KeyData() {} + +public: + //! Destructor + virtual ~KeyData() {} + //! Copy the data + virtual std::shared_ptr clone() const = 0; + //! Print the data to a stream + virtual void print( std::ostream& os, const std::string& indent = "" ) const = 0; + //! Return the native data type + virtual std::string type() const = 0; + +protected: + KeyData( const KeyData& ) {} + KeyData& operator=( const KeyData& ); +}; + + +//! Class to a database +class Database : public KeyData +{ +public: + //! Empty constructor + Database(); + + /** + * Open an database file. + * @param filename Name of input file to open + */ + explicit Database( const std::string& filename ); + + /** + * Create database from string + * @param data String containing the database data + */ + static std::shared_ptr createFromString( const std::string& data ); + + //! Copy constructor + Database( const Database& ); + + //! Assignment operator + Database& operator=( const Database& ); + + //! Move constructor + Database( Database&& rhs ); + + //! Move assignment operator + Database& operator=( Database&& rhs ); + + //! Destructor + virtual ~Database(); + + //! Copy the data + virtual std::shared_ptr clone() const override; + + //! Copy the data + std::shared_ptr cloneDatabase() const; + + + /** + * Return true if the specified key exists in the database and false + * otherwise. + * @param[in] key Key name to lookup. + */ + bool keyExists( const std::string& key ) const; + + + /** + * Return all keys in the database. + */ + std::vector getAllKeys() const; + + + //! Return the number of entries in the database + size_t size() const { return d_data.size(); } + + + /** + * Get the scalar entry from the database with the specified key + * name. If the specified key does not exist in the database or + * is not a scalar of the given type, then an error message is printed and + * the program exits. + * + * @param[in] key Key name in database. + * @param[in] unit Desired units + */ + template + inline TYPE getScalar( const std::string& key, const Units& unit = Units() ) const; + + + /// @copydoc Database::getScalar(const std::string&,const Units&) const + template + inline TYPE getScalar( const std::string& key, const std::string& unit ) const; + + + /** + * Get the scalar entry from the database with the specified key + * name. If the specified key does not exist in the database the + * the default value will be printed + * + * @param[in] key Key name in database + * @param[in] value Default value + * @param[in] unit Desired units + */ + template + inline TYPE getWithDefault( + const std::string& key, const TYPE& value, const Units& unit = Units() ) const; + + + /// @copydoc Database::getWithDefault(const std::string&,const TYPE&,const Units&) const + template + inline TYPE getWithDefault( + const std::string& key, const TYPE& value, const std::string& unit ) const; + + + /** + * Put the scalar entry into the database with the specified key name. + * @param key Key name in database. + * @param value Value to store + * @param unit Desired units + */ + template + inline void putScalar( const std::string& key, const TYPE& value, const Units& unit = Units() ); + + + /** + * Put the scalar entry into the database with the specified key name. + * @param key Key name in database. + * @param value Value to store + * @param unit Desired units + */ + template + inline void putScalar( const std::string& key, const TYPE& value, const std::string& unit ); + + + /** + * Get the vector entries from the database with the specified key + * name. If the specified key does not exist in the database or + * is not of the given type, then an error message is printed and + * the program exits. + * + * @param key Key name in database. + * @param unit Desired units + */ + template + std::vector getVector( const std::string& key, const Units& unit = Units() ) const; + + + /// @copydoc Database::getVector(const std::string&,const Units&) const + template + inline std::vector getVector( const std::string& key, const std::string& unit ) const; + + + /** + * Put the vector entries into the database with the specified key + * name. If the specified key does not exist in the database or + * is not of the given type, then an error message is printed and + * the program exits. + * + * @param key Key name in database. + * @param data Data to store + * @param unit Desired units + */ + template + void putVector( + const std::string& key, const std::vector& data, const Units& unit = Units() ); + + + /// @copydoc Database::putVector(const std::string&,const std::vector&,const Units&) + template + inline void putVector( + const std::string& key, const std::vector& data, const std::string& unit ); + + + /** + * Get the data for a key in the database. If the specified key + * does not exist in the database an error message is printed and + * the program exits. + * + * @param key Key name in database. + */ + std::shared_ptr getData( const std::string& key ); + + /** + * Get the data for a key in the database. If the specified key + * does not exist in the database an error message is printed and + * the program exits. + * + * @param key Key name in database. + */ + std::shared_ptr getData( const std::string& key ) const; + + + /** + * Put the data for a key in the database. + * + * @param key Key name in database. + * @param data Data to store + */ + void putData( const std::string& key, std::shared_ptr data ); + + + // Check if the key is a database object + bool isDatabase( const std::string& key ) const; + + + // Check if the entry can be stored as the given type + template + bool isType( const std::string& key ) const; + + + /** + * Get the database for a key in the database. If the specified key + * does not exist in the database an error message is printed and + * the program exits. + * + * @param key Key name in database. + */ + std::shared_ptr getDatabase( const std::string& key ); + + /** + * Get the database for a key in the database. If the specified key + * does not exist in the database an error message is printed and + * the program exits. + * + * @param key Key name in database. + */ + std::shared_ptr getDatabase( const std::string& key ) const; + + + /** + * Get the database for a key in the database. If the specified key + * does not exist in the database an error message is printed and + * the program exits. + * + * @param key Key name in database. + * @param db Database to store + */ + void putDatabase( const std::string& key, std::shared_ptr db ); + + + /** + * Print the data to a stream + * @param os Output stream + * @param indent Indenting to use before each line + */ + virtual void print( std::ostream& os, const std::string& indent = "" ) const override; + + + //! Print the type + virtual std::string type() const override { return "database"; } + + + /** + * Print the data to a string + * @return Output string + */ + std::string print( const std::string& indent = "" ) const; + + +protected: + std::map> d_data; + + // Function to load a database from a buffer + static size_t loadDatabase( const char* buffer, Database& db ); +}; + + +#include "common/Database.hpp" + +#endif diff --git a/common/Database.hpp b/common/Database.hpp new file mode 100644 index 00000000..37a89af0 --- /dev/null +++ b/common/Database.hpp @@ -0,0 +1,173 @@ +#ifndef included_Database_hpp +#define included_Database_hpp + +#include "common/Database.h" +#include "common/Utilities.h" + +#include + + +/******************************************************************** + * Basic classes for primative data types * + ********************************************************************/ +class EmptyKeyData : public KeyData +{ +public: + EmptyKeyData() {} + virtual ~EmptyKeyData() {} + virtual std::shared_ptr clone() const override + { + return std::make_shared(); + } + virtual void print( std::ostream& os, const std::string& = "" ) const override + { + os << std::endl; + } + virtual std::string type() const override { return ""; } +}; +class KeyDataDouble : public KeyData +{ +public: + KeyDataDouble() {} + explicit KeyDataDouble( const std::vector& data, const Units& unit ) + : d_data( data ), d_unit( unit ) + { + } + virtual ~KeyDataDouble() {} + virtual std::shared_ptr clone() const override + { + return std::make_shared( d_data, d_unit ); + } + virtual void print( std::ostream& os, const std::string& indent = "" ) const override; + virtual std::string type() const override { return "double"; } + + static std::tuple read( const std::string& ); + static double readValue( const std::string& ); + +public: + std::vector d_data; + Units d_unit; +}; +class KeyDataBool : public KeyData +{ +public: + KeyDataBool() {} + explicit KeyDataBool( const std::vector& data ) : d_data( data ) {} + virtual ~KeyDataBool() {} + virtual std::shared_ptr clone() const override + { + return std::make_shared( d_data ); + } + virtual void print( std::ostream& os, const std::string& indent = "" ) const override + { + os << indent; + for ( size_t i = 0; i < d_data.size(); i++ ) { + if ( i > 0 ) { + os << ", "; + } + if ( d_data[i] ) { + os << "true"; + } else { + os << "false"; + } + } + os << std::endl; + } + virtual std::string type() const override { return "bool"; } + std::vector d_data; +}; +class KeyDataString : public KeyData +{ +public: + KeyDataString() {} + explicit KeyDataString( const std::vector& data ) : d_data( data ) {} + virtual ~KeyDataString() {} + virtual std::shared_ptr clone() const override + { + return std::make_shared( d_data ); + } + virtual void print( std::ostream& os, const std::string& indent = "" ) const override + { + os << indent; + for ( size_t i = 0; i < d_data.size(); i++ ) { + if ( i > 0 ) { + os << ", "; + } + os << '"' << d_data[i] << '"'; + } + os << std::endl; + } + virtual std::string type() const override { return "string"; } + std::vector d_data; +}; + + +/******************************************************************** + * Get a vector * + ********************************************************************/ +template +inline std::vector Database::getVector( + const std::string& key, const std::string& unit ) const +{ + return getVector( key, Units( unit ) ); +} +template +inline void Database::putVector( + const std::string& key, const std::vector& data, const std::string& unit ) +{ + putVector( key, data, Units( unit ) ); +} + + +/******************************************************************** + * Get a scalar * + ********************************************************************/ +template +inline TYPE Database::getScalar( const std::string& key, const Units& unit ) const +{ + const std::vector& data = getVector( key, unit ); + if ( data.size() != 1 ) { + char msg[1000]; + sprintf( msg, "Variable %s is not a scalar", key.c_str() ); + ERROR( msg ); + } + return data[0]; +} +template +inline TYPE Database::getWithDefault( + const std::string& key, const TYPE& value, const Units& unit ) const +{ + if ( !keyExists( key ) ) + return value; + return getScalar( key, unit ); +} +template +inline void Database::putScalar( const std::string& key, const TYPE& data, const Units& unit ) +{ + putVector( key, std::vector( 1, data ), unit ); +} +template +inline TYPE Database::getScalar( const std::string& key, const std::string& unit ) const +{ + return getScalar( key, Units( unit ) ); +} +template +inline TYPE Database::getWithDefault( + const std::string& key, const TYPE& value, const std::string& unit ) const +{ + return getWithDefault( key, value, Units( unit ) ); +} +template +inline void Database::putScalar( const std::string& key, const TYPE& data, const std::string& unit ) +{ + putScalar( key, data, Units( unit ) ); +} +template +inline void putVector( + const std::string& key, const std::vector& data, const std::string& unit ) +{ + putVector( key, data, Units( unit ) ); +} + + +#endif diff --git a/common/Domain.cpp b/common/Domain.cpp index 4c59e2fa..20d8f1fd 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -15,92 +15,131 @@ #include "common/MPI_Helpers.h" #include "common/Communication.h" -using namespace std; -// Reading the domain information file -void read_domain( int rank, int nprocs, MPI_Comm comm, - int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz, - int& nspheres, double& Lx, double& Ly, double& Lz ) + + +// Inline function to read line without a return argument +static inline void fgetl( char * str, int num, FILE * stream ) { - if (rank==0){ - ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> nx; - domain >> ny; - domain >> nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - - } - MPI_Barrier(comm); - // Computational domain - //................................................. - MPI_Bcast(&nx,1,MPI_INT,0,comm); - MPI_Bcast(&ny,1,MPI_INT,0,comm); - MPI_Bcast(&nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - MPI_Barrier(comm); + char* ptr = fgets( str, num, stream ); + if ( 0 ) {char *temp = (char *)&ptr; temp++;} } + /******************************************************** - * Constructor/Destructor * + * Constructors/Destructor * ********************************************************/ -Domain::Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz, - double lx, double ly, double lz, int BC): - Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), - Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), - Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), - rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), - rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), - rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), - rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), - sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), - sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), - sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), - sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), - sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), - sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), - sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), - sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), - sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), - recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), - recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), - recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), - recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), - recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), - recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), - recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), - recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), - recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), - sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), - sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), - sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), - recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), - recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), - recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), - id(NULL) +Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, + double lx, double ly, double lz, int BC): + Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), + Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), + Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), + rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), + rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), + rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), + rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), + sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), + sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), + sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), + sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), + sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), + sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), + sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), + sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), + sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), + recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), + recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), + recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), + recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), + recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), + recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), + recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), + recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), + recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), + sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), + sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), + sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), + recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), + recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), + recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), + id(NULL) { - Volume = nx*ny*nx*npx*npy*npz*1.0; - Nx = nx+2; Ny = ny+2; Nz = nz+2; - Lx = lx, Ly = ly, Lz = lz; - rank = rnk; - nprocx=npx; nprocy=npy; nprocz=npz; + auto db = std::make_shared( ); + db->putScalar( "BC", BC ); + db->putVector( "nproc", { npx, npx, npx } ); + db->putVector( "n", { nx, ny, nz } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { lx, ly, lz } ); + initialize( db ); +} +Domain::Domain( std::shared_ptr db ): + Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), + Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), + Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), + rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), + rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), + rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), + rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), + sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), + sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), + sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), + sendList_x(NULL), sendList_y(NULL), sendList_z(NULL), sendList_X(NULL), sendList_Y(NULL), sendList_Z(NULL), + sendList_xy(NULL), sendList_yz(NULL), sendList_xz(NULL), sendList_Xy(NULL), sendList_Yz(NULL), sendList_xZ(NULL), + sendList_xY(NULL), sendList_yZ(NULL), sendList_Xz(NULL), sendList_XY(NULL), sendList_YZ(NULL), sendList_XZ(NULL), + sendBuf_x(NULL), sendBuf_y(NULL), sendBuf_z(NULL), sendBuf_X(NULL), sendBuf_Y(NULL), sendBuf_Z(NULL), + sendBuf_xy(NULL), sendBuf_yz(NULL), sendBuf_xz(NULL), sendBuf_Xy(NULL), sendBuf_Yz(NULL), sendBuf_xZ(NULL), + sendBuf_xY(NULL), sendBuf_yZ(NULL), sendBuf_Xz(NULL), sendBuf_XY(NULL), sendBuf_YZ(NULL), sendBuf_XZ(NULL), + recvCount_x(0), recvCount_y(0), recvCount_z(0), recvCount_X(0), recvCount_Y(0), recvCount_Z(0), + recvCount_xy(0), recvCount_yz(0), recvCount_xz(0), recvCount_Xy(0), recvCount_Yz(0), recvCount_xZ(0), + recvCount_xY(0), recvCount_yZ(0), recvCount_Xz(0), recvCount_XY(0), recvCount_YZ(0), recvCount_XZ(0), + recvList_x(NULL), recvList_y(NULL), recvList_z(NULL), recvList_X(NULL), recvList_Y(NULL), recvList_Z(NULL), + recvList_xy(NULL), recvList_yz(NULL), recvList_xz(NULL), recvList_Xy(NULL), recvList_Yz(NULL), recvList_xZ(NULL), + recvList_xY(NULL), recvList_yZ(NULL), recvList_Xz(NULL), recvList_XY(NULL), recvList_YZ(NULL), recvList_XZ(NULL), + recvBuf_x(NULL), recvBuf_y(NULL), recvBuf_z(NULL), recvBuf_X(NULL), recvBuf_Y(NULL), recvBuf_Z(NULL), + recvBuf_xy(NULL), recvBuf_yz(NULL), recvBuf_xz(NULL), recvBuf_Xy(NULL), recvBuf_Yz(NULL), recvBuf_xZ(NULL), + recvBuf_xY(NULL), recvBuf_yZ(NULL), recvBuf_Xz(NULL), recvBuf_XY(NULL), recvBuf_YZ(NULL), recvBuf_XZ(NULL), + sendData_x(NULL), sendData_y(NULL), sendData_z(NULL), sendData_X(NULL), sendData_Y(NULL), sendData_Z(NULL), + sendData_xy(NULL), sendData_yz(NULL), sendData_xz(NULL), sendData_Xy(NULL), sendData_Yz(NULL), sendData_xZ(NULL), + sendData_xY(NULL), sendData_yZ(NULL), sendData_Xz(NULL), sendData_XY(NULL), sendData_YZ(NULL), sendData_XZ(NULL), + recvData_x(NULL), recvData_y(NULL), recvData_z(NULL), recvData_X(NULL), recvData_Y(NULL), recvData_Z(NULL), + recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), + recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), + id(NULL) +{ + initialize( db ); +} +void Domain::initialize( std::shared_ptr db ) +{ + d_db = db; + auto nproc = d_db->getVector("nproc"); + auto n = d_db->getVector("n"); + auto L = d_db->getVector("L"); + ASSERT( n.size() == 3u ); + ASSERT( L.size() == 3u ); + ASSERT( nproc.size() == 3u ); + int nx = n[0]; + int ny = n[1]; + int nz = n[2]; + Lx = L[0]; + Ly = L[1]; + Lz = L[2]; + Nx = nx+2; + Ny = ny+2; + Nz = nz+2; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; N = Nx*Ny*Nz; + Volume = nx*ny*nx*nprocx*nprocy*nprocz*1.0; id = new char[N]; memset(id,0,N); - BoundaryCondition = BC; + BoundaryCondition = d_db->getScalar("BC"); + MPI_Comm_rank( MPI_COMM_WORLD, &rank ); rank_info=RankInfoStruct(rank,nprocx,nprocy,nprocz); + int nprocs; + MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); + INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); } Domain::~Domain() { @@ -1240,3 +1279,447 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh) UnpackMeshData(recvList_YZ, recvCount_YZ ,recvData_YZ, MeshData); } + + + + +double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ + /* + * This routine converts the data in the Distance array to a signed distance + * by solving the equation df/dt = sign(1-|grad f|), where Distance provides + * the values of f on the mesh associated with domain Dm + * It has been tested with segmented data initialized to values [-1,1] + * and will converge toward the signed distance to the surface bounding the associated phases + */ + + int Q=26; + int q,i,j,k; + double dt=0.1; + int in,jn,kn; + double Dqx,Dqy,Dqz,Dx,Dy,Dz,W; + double nx,ny,nz,Cqx,Cqy,Cqz,sign,norm; + double TotalVariation=0.0; + + const static int D3Q27[26][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, + {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, + {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1},{1,1,1},{-1,-1,-1},{1,1,-1},{-1,-1,1}, + {-1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1}}; + + double weights[26]; + // Compute the weights from the finite differences + for (q=0; q fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + + int count = 0; + while (count < timesteps){ + + // Communicate the halo of values + fillData.fill(Distance); + + TotalVariation=0.0; + // Execute the next timestep + for (k=1;k 0.0 ){ + for (q=0; q<26; q++){ + Cqx = 1.0*D3Q27[q][0]; + Cqy = 1.0*D3Q27[q][1]; + Cqz = 1.0*D3Q27[q][2]; + // get the associated neighbor + in = i + D3Q27[q][0]; + jn = j + D3Q27[q][1]; + kn = k + D3Q27[q][2]; + + // make sure the neighbor is in the domain (periodic BC) + /* if (in < 0 ) in +=Nx; + * don't need this in parallel since MPI handles the halos + if (jn < 0 ) jn +=Ny; + if (kn < 0 ) kn +=Nz; + if (!(in < Nx) ) in -=Nx; + if (!(jn < Ny) ) jn -=Ny; + if (!(kn < Nz) ) kn -=Nz; + // symmetric boundary + if (in < 0 ) in = i; + if (jn < 0 ) jn = j; + if (kn < 0 ) kn = k; + if (!(in < Nx) ) in = i; + if (!(jn < Ny) ) jn = k; + if (!(kn < Nz) ) kn = k; + */ + + // Compute the gradient using upwind finite differences + Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx; + Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy; + Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz; + + // Only include upwind derivatives + if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){ + + Dx += Dqx; + Dy += Dqy; + Dz += Dqz; + W += weights[q]; + } + } + // Normalize by the weight to get the approximation to the gradient + if (fabs(W) > 0.0){ + Dx /= W; + Dy /= W; + Dz /= W; + } + norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz); + } + else{ + norm = 0.0; + } + Distance(i,j,k) += dt*sign*(1.0 - norm); + TotalVariation += dt*sign*(1.0 - norm); + // Disallow any change in phase + // if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k); + } + } + } + TotalVariation /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2); + count++; + } + + return TotalVariation; +} + + +void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad) +{ + // Read in the full sphere pack + //...... READ IN THE SPHERES................................... + cout << "Reading the packing file..." << endl; + FILE *fid = fopen("pack.out","rb"); + INSIST(fid!=NULL,"Error opening pack.out"); + //.........Trash the header lines.......... + char line[100]; + fgetl(line, 100, fid); + fgetl(line, 100, fid); + fgetl(line, 100, fid); + fgetl(line, 100, fid); + fgetl(line, 100, fid); + //........read the spheres.................. + // We will read until a blank like or end-of-file is reached + int count = 0; + while ( !feof(fid) && fgets(line,100,fid)!=NULL ) { + char* line2 = line; + List_cx[count] = strtod(line2,&line2); + List_cy[count] = strtod(line2,&line2); + List_cz[count] = strtod(line2,&line2); + List_rad[count] = strtod(line2,&line2); + count++; + } + cout << "Number of spheres extracted is: " << count << endl; + INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" ); + // ............................................................. +} + +void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) +{ + // Use sphere lists to determine which nodes are in porespace + // Write out binary file for nodes + char value; + int N = Nx*Ny*Nz; // Domain size, including the halo + double hx,hy,hz; + double x,y,z; + double cx,cy,cz,r; + int imin,imax,jmin,jmax,kmin,kmax; + int p,i,j,k,n; + //............................................ + double min_x,min_y,min_z; +// double max_x,max_y,max_z; + //............................................ + // Lattice spacing for the entire domain + // It should generally be true that hx=hy=hz + // Otherwise, you will end up with ellipsoids + hx = Lx/(Nx*nprocx-1); + hy = Ly/(Ny*nprocy-1); + hz = Lz/(Nz*nprocz-1); + //............................................ + // Get maximum and minimum for this domain + // Halo is included ! + min_x = double(iproc*Nx-1)*hx; + min_y = double(jproc*Ny-1)*hy; + min_z = double(kproc*Nz-1)*hz; +// max_x = ((iproc+1)*Nx+1)*hx; +// max_y = ((jproc+1)*Ny+1)*hy; +// max_z = ((kproc+1)*Nz+1)*hz; + //............................................ + + //............................................ + // Pre-initialize local ID + for (n=0;nNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;iNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;i #include #include #include #include #include -#include // std::exception +#include #include #include "common/Array.h" #include "common/Utilities.h" #include "common/MPI_Helpers.h" #include "common/Communication.h" +#include "common/Database.h" -using namespace std; //! Read the domain information file -void read_domain( int rank, int nprocs, MPI_Comm comm, - int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz, - int& nspheres, double& Lx, double& Ly, double& Lz ); +std::shared_ptr read_domain( ); //! Class to hold domain info -struct Domain{ - // Default constructor - Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz, - double lx, double ly, double lz, int BC); +class Domain{ +public: + //! Default constructor + Domain( std::shared_ptr db ); - // Destructor + //! Obsolete constructor + Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, + double lx, double ly, double lz, int BC); + + //! Empty constructor + Domain() = delete; + + //! Copy constructor + Domain( const Domain& ) = delete; + + //! Assignment operator + Domain& operator=( const Domain& ) = delete; + + //! Destructor ~Domain(); + //! Get the database + inline std::shared_ptr getDatabase() const { return d_db; } + +private: + + void initialize( std::shared_ptr db ); + + std::shared_ptr d_db; + +public: + // Basic domain information int Nx,Ny,Nz,N; int iproc,jproc,kproc; @@ -114,454 +135,27 @@ private: }; +double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps); -// Inline function to read line without a return argument -static inline void fgetl( char * str, int num, FILE * stream ) -{ - char* ptr = fgets( str, num, stream ); - if ( 0 ) {char *temp = (char *)&ptr; temp++;} -} +void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad); +void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); +void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); -inline double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ - /* - * This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - */ +void WriteLocalSolidID(char *FILENAME, char *ID, int N); - int Q=26; - int q,i,j,k; - double dt=0.1; - int in,jn,kn; - double Dqx,Dqy,Dqz,Dx,Dy,Dz,W; - double nx,ny,nz,Cqx,Cqy,Cqz,sign,norm; - double TotalVariation=0.0; +void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N); - const static int D3Q27[26][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, - {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, - {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1},{1,1,1},{-1,-1,-1},{1,1,-1},{-1,-1,1}, - {-1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1}}; +void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np); - double weights[26]; - // Compute the weights from the finite differences - for (q=0; q fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); - - int count = 0; - while (count < timesteps){ - - // Communicate the halo of values - fillData.fill(Distance); - - TotalVariation=0.0; - // Execute the next timestep - for (k=1;k 0.0 ){ - for (q=0; q<26; q++){ - Cqx = 1.0*D3Q27[q][0]; - Cqy = 1.0*D3Q27[q][1]; - Cqz = 1.0*D3Q27[q][2]; - // get the associated neighbor - in = i + D3Q27[q][0]; - jn = j + D3Q27[q][1]; - kn = k + D3Q27[q][2]; - - // make sure the neighbor is in the domain (periodic BC) - /* if (in < 0 ) in +=Nx; - * don't need this in parallel since MPI handles the halos - if (jn < 0 ) jn +=Ny; - if (kn < 0 ) kn +=Nz; - if (!(in < Nx) ) in -=Nx; - if (!(jn < Ny) ) jn -=Ny; - if (!(kn < Nz) ) kn -=Nz; - // symmetric boundary - if (in < 0 ) in = i; - if (jn < 0 ) jn = j; - if (kn < 0 ) kn = k; - if (!(in < Nx) ) in = i; - if (!(jn < Ny) ) jn = k; - if (!(kn < Nz) ) kn = k; - */ - - // Compute the gradient using upwind finite differences - Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx; - Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy; - Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz; - - // Only include upwind derivatives - if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){ - - Dx += Dqx; - Dy += Dqy; - Dz += Dqz; - W += weights[q]; - } - } - // Normalize by the weight to get the approximation to the gradient - if (fabs(W) > 0.0){ - Dx /= W; - Dy /= W; - Dz /= W; - } - norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz); - } - else{ - norm = 0.0; - } - Distance(i,j,k) += dt*sign*(1.0 - norm); - TotalVariation += dt*sign*(1.0 - norm); - // Disallow any change in phase - // if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k); - } - } - } - TotalVariation /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2); - count++; - } - - return TotalVariation; -} - - -inline void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad) -{ - // Read in the full sphere pack - //...... READ IN THE SPHERES................................... - cout << "Reading the packing file..." << endl; - FILE *fid = fopen("pack.out","rb"); - INSIST(fid!=NULL,"Error opening pack.out"); - //.........Trash the header lines.......... - char line[100]; - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - //........read the spheres.................. - // We will read until a blank like or end-of-file is reached - int count = 0; - while ( !feof(fid) && fgets(line,100,fid)!=NULL ) { - char* line2 = line; - List_cx[count] = strtod(line2,&line2); - List_cy[count] = strtod(line2,&line2); - List_cz[count] = strtod(line2,&line2); - List_rad[count] = strtod(line2,&line2); - count++; - } - cout << "Number of spheres extracted is: " << count << endl; - INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" ); - // ............................................................. -} - -inline void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, - double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, - int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) -{ - // Use sphere lists to determine which nodes are in porespace - // Write out binary file for nodes - char value; - int N = Nx*Ny*Nz; // Domain size, including the halo - double hx,hy,hz; - double x,y,z; - double cx,cy,cz,r; - int imin,imax,jmin,jmax,kmin,kmax; - int p,i,j,k,n; - //............................................ - double min_x,min_y,min_z; -// double max_x,max_y,max_z; - //............................................ - // Lattice spacing for the entire domain - // It should generally be true that hx=hy=hz - // Otherwise, you will end up with ellipsoids - hx = Lx/(Nx*nprocx-1); - hy = Ly/(Ny*nprocy-1); - hz = Lz/(Nz*nprocz-1); - //............................................ - // Get maximum and minimum for this domain - // Halo is included ! - min_x = double(iproc*Nx-1)*hx; - min_y = double(jproc*Ny-1)*hy; - min_z = double(kproc*Nz-1)*hz; -// max_x = ((iproc+1)*Nx+1)*hx; -// max_y = ((jproc+1)*Ny+1)*hy; -// max_z = ((kproc+1)*Nz+1)*hz; - //............................................ - - //............................................ - // Pre-initialize local ID - for (n=0;nNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;iNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;i +#include +#include + + +constexpr double Units::d_pow10[22]; +constexpr char Units::d_prefixSymbol[]; + + +/******************************************************************** + * Constructors * + ********************************************************************/ +Units::Units() : d_prefix( UnitPrefix::unknown ), d_unit( UnitValue::unknown ) {} +Units::Units( UnitPrefix p, UnitValue u ) : d_prefix( p ), d_unit( u ) {} +Units::Units( const std::string& unit ) + : d_prefix( UnitPrefix::unknown ), d_unit( UnitValue::unknown ) +{ + // Parse the string to get it into a more friendly format + auto tmp = unit; + tmp.erase( std::remove( tmp.begin(), tmp.end(), ' ' ), tmp.end() ); + // Check if the character '-' is present indicating a seperation between the prefix and unit + size_t index = tmp.find( '-' ); + if ( index != std::string::npos ) { + d_prefix = getUnitPrefix( tmp.substr( 0, index ) ); + d_unit = getUnitValue( tmp.substr( index + 1 ) ); + } else { + if ( tmp.size() <= 1 ) { + d_prefix = UnitPrefix::none; + d_unit = getUnitValue( tmp ); + } else if ( tmp.substr( 0, 2 ) == "da" ) { + d_prefix = UnitPrefix::deca; + d_unit = getUnitValue( tmp.substr( 2 ) ); + } else { + d_prefix = getUnitPrefix( tmp.substr( 0, 1 ) ); + d_unit = getUnitValue( tmp.substr( 1 ) ); + if ( d_prefix == UnitPrefix::unknown || d_unit == UnitValue::unknown ) { + d_prefix = UnitPrefix::none; + d_unit = getUnitValue( tmp ); + } + } + } +} + + +/******************************************************************** + * Get prefix * + ********************************************************************/ +Units::UnitPrefix Units::getUnitPrefix( const std::string& str ) noexcept +{ + Units::UnitPrefix value = UnitPrefix::unknown; + if ( str.empty() ) { + value = UnitPrefix::none; + } else if ( str == "yotta" || str == "Y" ) { + value = UnitPrefix::yotta; + } else if ( str == "zetta" || str == "Z" ) { + value = UnitPrefix::zetta; + } else if ( str == "exa" || str == "E" ) { + value = UnitPrefix::exa; + } else if ( str == "peta" || str == "P" ) { + value = UnitPrefix::peta; + } else if ( str == "tera" || str == "T" ) { + value = UnitPrefix::tera; + } else if ( str == "giga" || str == "G" ) { + value = UnitPrefix::giga; + } else if ( str == "mega" || str == "M" ) { + value = UnitPrefix::mega; + } else if ( str == "kilo" || str == "k" ) { + value = UnitPrefix::kilo; + } else if ( str == "hecto" || str == "h" ) { + value = UnitPrefix::hecto; + } else if ( str == "deca" || str == "da" ) { + value = UnitPrefix::deca; + } else if ( str == "deci" || str == "d" ) { + value = UnitPrefix::deci; + } else if ( str == "centi" || str == "c" ) { + value = UnitPrefix::centi; + } else if ( str == "milli" || str == "m" ) { + value = UnitPrefix::milli; + } else if ( str == "micro" || str == "u" ) { + value = UnitPrefix::micro; + } else if ( str == "nano" || str == "n" ) { + value = UnitPrefix::nano; + } else if ( str == "pico" || str == "p" ) { + value = UnitPrefix::pico; + } else if ( str == "femto" || str == "f" ) { + value = UnitPrefix::femto; + } else if ( str == "atto" || str == "a" ) { + value = UnitPrefix::atto; + } else if ( str == "zepto" || str == "z" ) { + value = UnitPrefix::zepto; + } else if ( str == "yocto" || str == "y" ) { + value = UnitPrefix::yocto; + } + return value; +} + + +/******************************************************************** + * Get unit value * + ********************************************************************/ +Units::UnitValue Units::getUnitValue( const std::string& str ) noexcept +{ + Units::UnitValue value = UnitValue::unknown; + if ( str == "meter" || str == "m" ) { + value = UnitValue::meter; + } else if ( str == "gram" || str == "g" ) { + value = UnitValue::gram; + } else if ( str == "second" || str == "s" ) { + value = UnitValue::second; + } else if ( str == "ampere" || str == "A" ) { + value = UnitValue::ampere; + } else if ( str == "kelvin" || str == "K" ) { + value = UnitValue::kelvin; + } else if ( str == "joule" || str == "J" ) { + value = UnitValue::joule; + } else if ( str == "ergs" || str == "erg" ) { + value = UnitValue::erg; + } else if ( str == "degree" || str == "degrees" ) { + value = UnitValue::degree; + } else if ( str == "radian" || str == "radians" ) { + value = UnitValue::radian; + } + return value; +} + + +/******************************************************************** + * Get unit type * + ********************************************************************/ +Units::UnitType Units::getUnitType( UnitValue u ) noexcept +{ + switch ( u ) { + case UnitValue::meter: + return UnitType::length; + case UnitValue::gram: + return UnitType::mass; + case UnitValue::second: + return UnitType::time; + case UnitValue::ampere: + return UnitType::current; + case UnitValue::kelvin: + return UnitType::temperature; + case UnitValue::joule: + case UnitValue::erg: + return UnitType::energy; + case UnitValue::degree: + case UnitValue::radian: + return UnitType::angle; + default: + return UnitType::unknown; + } +} + + +/******************************************************************** + * Convert to another unit system * + ********************************************************************/ +double Units::convert( const Units& rhs ) const noexcept +{ + if ( this->operator==( rhs ) ) + return 1; + // Convert the prefix + double cp = convert( d_prefix ) / convert( rhs.d_prefix ); + if ( d_unit == rhs.d_unit ) + return cp; // Only need to convert prefix + // Convert the unit + if ( getUnitType( d_unit ) != getUnitType( rhs.d_unit ) ) + return 0; // Invalid conversion + double cu = 0; + if ( d_unit == UnitValue::joule && rhs.d_unit == UnitValue::erg ) + cu = 1e7; + else if ( d_unit == UnitValue::erg && rhs.d_unit == UnitValue::joule ) + cu = 1e-7; + else if ( d_unit == UnitValue::degree && rhs.d_unit == UnitValue::radian ) + cu = 0.017453292519943; + else if ( d_unit == UnitValue::radian && rhs.d_unit == UnitValue::degree ) + cu = 57.295779513082323; + // Return the total conversion + return cp * cu; +} + + +/******************************************************************** + * Write a string for the units * + ********************************************************************/ +std::string Units::str() const +{ + ASSERT( !isNull() ); + return std::string( str( d_prefix ).data() ) + str( d_unit ); +} +std::array Units::str( UnitPrefix p ) noexcept +{ + std::array str; + str[0] = d_prefixSymbol[static_cast( p )]; + str[1] = 0; + str[2] = 0; + if ( p == UnitPrefix::deca ) + str[1] = 'a'; + return str; +} +std::string Units::str( UnitValue u ) +{ + if ( u == UnitValue::meter ) { + return "m"; + } else if ( u == UnitValue::gram ) { + return "g"; + } else if ( u == UnitValue::second ) { + return "s"; + } else if ( u == UnitValue::ampere ) { + return "A"; + } else if ( u == UnitValue::kelvin ) { + return "K"; + } else if ( u == UnitValue::joule ) { + return "J"; + } else if ( u == UnitValue::erg ) { + return "erg"; + } else if ( u == UnitValue::degree ) { + return "degree"; + } else if ( u == UnitValue::radian ) { + return "radian"; + } + return "unknown"; +} diff --git a/common/Units.h b/common/Units.h new file mode 100644 index 00000000..a3496ee1 --- /dev/null +++ b/common/Units.h @@ -0,0 +1,140 @@ +#ifndef included_Units +#define included_Units + +#include +#include +#include +#include +#include +#include + + +//! Unit system class +class Units final +{ +public: + //! Enum to hold prefix + enum class UnitPrefix : int8_t { + yocto = 0, + zepto = 1, + atto = 2, + femto = 3, + pico = 4, + nano = 5, + micro = 6, + milli = 7, + centi = 8, + deci = 9, + none = 10, + deca = 11, + hecto = 12, + kilo = 13, + mega = 14, + giga = 15, + tera = 16, + peta = 17, + exa = 18, + zetta = 19, + yotta = 20, + unknown = 21 + }; + + //! Enum to unit type + enum class UnitType : uint8_t { + length, + mass, + time, + current, + temperature, + energy, + angle, + unknown + }; + + //! Enum to hold unit + enum class UnitValue : uint8_t { + meter, + gram, + second, + ampere, + kelvin, + joule, + erg, + degree, + radian, + unknown + }; + + +public: + //! Constructor + Units(); + + //! Constructor + explicit Units( const std::string& unit ); + + //! Constructor + explicit Units( UnitPrefix, UnitValue ); + + //! Get the prefix + inline UnitPrefix getPrefix() const noexcept { return d_prefix; } + + //! Get the unit + inline UnitValue getUnit() const noexcept { return d_unit; } + + //! Get the unit + inline UnitType getUnitType() const noexcept { return getUnitType( d_unit ); } + + //! Get the unit + static UnitType getUnitType( UnitValue ) noexcept; + + //! Get the prefix from a string + static UnitPrefix getUnitPrefix( const std::string& ) noexcept; + + //! Get the unit value from a string + static UnitValue getUnitValue( const std::string& ) noexcept; + + //! Convert to the given unit system + double convert( const Units& ) const noexcept; + + //! Convert a prefix to a scalar + static inline double convert( UnitPrefix x ) noexcept + { + return d_pow10[static_cast( x )]; + } + + //! Get a string representation of the units + std::string str() const; + + //! Get a string representation for the prefix + static std::array str( UnitPrefix ) noexcept; + + //! Get a string representation for the unit value + static std::string str( UnitValue ); + + //! Operator == + inline bool operator==( const Units& rhs ) const noexcept + { + return d_prefix == rhs.d_prefix && d_unit == rhs.d_unit; + } + + //! Operator != + inline bool operator!=( const Units& rhs ) const noexcept + { + return d_prefix != rhs.d_prefix || d_unit != rhs.d_unit; + } + + //! Check if unit is null + bool isNull() const { return d_prefix == UnitPrefix::unknown || d_unit == UnitValue::unknown; } + +protected: + UnitPrefix d_prefix; + UnitValue d_unit; + +private: + constexpr static double d_pow10[22] = { 1e-24, 1e-21, 1e-18, 1e-15, 1e-12, 1e-9, 1e-6, 1e-3, + 1e-2, 0.1, 1, 10, 100, 1000, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24, 0 }; + constexpr static char d_prefixSymbol[] = "yzafpnumcd\0dhkMGTPEZYu"; +}; + +#endif diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index 39d16e59..c219e9b4 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -9,7 +9,35 @@ #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" -using namespace std; + +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared( "Domain.in" ); + const int dim = 50; + db->putScalar( "BC", 0 ); + if ( nprocs == 1 ){ + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 3, 1, 1 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if ( nprocs == 2 ) { + db->putVector( "nproc", { 2, 1, 1 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if ( nprocs == 4 ) { + db->putVector( "nproc", { 2, 2, 1 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if (nprocs==8){ + db->putVector( "nproc", { 2, 2, 2 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } + return db; +} //*************************************************************************************** @@ -26,11 +54,6 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); int check; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); printf("Running Color Model: TestColor \n"); @@ -43,11 +66,8 @@ int main(int argc, char **argv) int timestepMax, interval; double Fx,Fy,Fz,tol; // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; - int dim = 50; + //if (rank == 0) printf("dim=%d\n",dim); int timestep = 1; int timesteps = 100; @@ -78,90 +98,25 @@ int main(int argc, char **argv) flux = 10.f; dout=1.f; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=3; Ny = 1; - Nz = 1; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=2; nprocy=1; - nprocz=1; - Nx=Ny=Nz=dim; - Nx = dim; Ny = dim; Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); printf("********************************************************\n"); } - MPI_Barrier(comm); double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + + Domain Dm(db); Nx += 2; Ny += 2; @@ -186,9 +141,9 @@ int main(int argc, char **argv) } } - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; + int kproc = rank/(nprocx*nprocy); + int jproc = (rank-nprocx*nprocy*kproc)/nprocx; + int iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; printf("rank=%i, %i,%i,%i \n",rank,iproc,jproc,kproc); // Initialize a square tube for (k=1;k loadInputs( int nprocs ) +{ + auto db = std::make_shared( "Domain.in" ); + const int dim = 50; + db->putScalar( "BC", 0 ); + if ( nprocs == 1 ){ + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 3, 1, 1 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if ( nprocs == 2 ) { + db->putVector( "nproc", { 2, 1, 1 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if ( nprocs == 4 ) { + db->putVector( "nproc", { 2, 2, 1 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if (nprocs==8){ + db->putVector( "nproc", { 2, 2, 2 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } + return db; +} + + extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int Nx, int Ny, int Nz, int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) { @@ -176,88 +206,13 @@ int main(int argc, char **argv) int timestepMax, interval; double tau,Fx,Fy,Fz,tol; // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=50; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=nprocy=1; - nprocz=2; - Nx=Ny=Nz=50; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=50; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=50; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nBlocks,1,MPI_UNSIGNED,0,comm); - MPI_Bcast(&nthreads,1,MPI_UNSIGNED,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; if (rank==0){ printf("********************************************************\n"); @@ -272,8 +227,9 @@ int main(int argc, char **argv) iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + + Domain Dm(db); + InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index d07a7213..fe842dc7 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -30,6 +30,22 @@ extern void PrintNeighborList(int * neighborList, int Np, int rank) { } } + +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared( "Domain.in" ); + const int dim = 50; + db->putScalar( "BC", 0 ); + if ( nprocs == 1 ){ + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 3, 3, 3 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } + return db; +} + + //*************************************************************************************** int main(int argc, char **argv) { @@ -45,7 +61,6 @@ int main(int argc, char **argv) int check; { // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; int iproc,jproc,kproc; @@ -61,9 +76,6 @@ int main(int argc, char **argv) int timestepMax, interval; double tau,Fx,Fy,Fz,tol; // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; int dim = 3; if (rank == 0) printf("dim=%d\n",dim); int timestep = 0; @@ -76,58 +88,15 @@ int main(int argc, char **argv) Fx = Fy = 1.0; Fz = 1.0; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - printf("domain.good == true \n"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=3; Ny = 3; - Nz = 3; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); @@ -150,9 +119,8 @@ int main(int argc, char **argv) } double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Domain Dm(db); Nx += 2; Ny += 2; diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 9ace6822..53cc55ae 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -15,6 +15,20 @@ #define SPEED -1 #define PI 3.14159 + +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared( ); + const int dim = 50; + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { N, N, N } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} + + int main (int argc, char *argv[]) { // Initialize MPI @@ -24,16 +38,11 @@ int main (int argc, char *argv[]) MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int npx,npy,npz; int i,j,k,n; - int Nx,Ny,Nz; - double Lx,Ly,Lz; - Nx=Ny=Nz=N; - npx=npy=npz=1; - Lx=Ly=Lz=1.0; - int BC=0; // periodic boundary condition + // Load inputs + auto db = loadInputs( nprocs ); - Domain Dm(Nx,Ny,Nz,rank,npx,npy,npz,Lx,Ly,Lz,BC); + Domain Dm(db); for (i=0; i loadInputs( int nprocs ) +{ + const int dim = 12; + std::shared_ptr db; + if ( exists( "Domain.in" ) ) { + db = std::make_shared( "Domain.in" ); + } else if (nprocs==1) { + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { dim, dim, dim } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } else if (nprocs==2) { + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { dim/2, dim/2, dim/2 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } + db->putScalar( "BC", 0 ); + return db; +} + //*************************************************************************************** int main(int argc, char **argv) @@ -25,11 +53,6 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); int check; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); printf("Running Unit Test: TestPoiseuille \n"); @@ -42,11 +65,7 @@ int main(int argc, char **argv) int timestepMax, interval; double tau,Fx,Fy,Fz,tol; // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; - int dim = 12; if (rank == 0) printf("dim=%d\n",dim); int timestep = 0; @@ -56,68 +75,16 @@ int main(int argc, char **argv) double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); Fx = 0; Fy = 0; Fz = 1e-3; //1.f; // 1e-3; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - printf("domain.good == true \n"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - printf("domain.good == false - using predefined parameters \n"); - nprocx=nprocy=nprocz=1; - Nx=dim; Ny = dim; - Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - printf("domain.good == false - using predefined parameters \n"); - nprocx=2; - nprocy=nprocz=1; - Nx=0.5*dim; Ny = 0.5*dim; - Nz = 0.5*dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + if (rank==0){ printf("********************************************************\n"); @@ -126,9 +93,9 @@ int main(int argc, char **argv) } MPI_Barrier(comm); - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; + int kproc = rank/(nprocx*nprocy); + int jproc = (rank-nprocx*nprocy*kproc)/nprocx; + int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; if (rank == 0) { printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); @@ -140,9 +107,8 @@ int main(int argc, char **argv) } double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Domain Dm(db); Nx += 2; Ny += 2; diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index 233f185d..bdd67aec 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -9,7 +9,21 @@ #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" -using namespace std; + +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared( "Domain.in" ); + const int dim = 50; + db->putScalar( "BC", 0 ); + if ( nprocs == 1 ){ + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 4, 4, 4 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + } + return db; +} + //*************************************************************************************** int main(int argc, char **argv) @@ -25,11 +39,6 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); int check; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); printf("Running Unit Test: TestPressVel \n"); @@ -42,67 +51,21 @@ int main(int argc, char **argv) int timestepMax, interval; double tau,Fx,Fy,Fz,tol; // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; - int dim = 50; if (rank == 0) printf("dim=%d\n",dim); double rlx_setA=1.0; double rlx_setB=1.0; Fx = Fy = 0.f; Fz = 1.0e-4; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=4; Ny = 4; - Nz = 4; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); @@ -111,9 +74,9 @@ int main(int argc, char **argv) } MPI_Barrier(comm); - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; + int kproc = rank/(nprocx*nprocy); + int jproc = (rank-nprocx*nprocy*kproc)/nprocx; + int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; if (rank == 0) { printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); @@ -125,9 +88,8 @@ int main(int argc, char **argv) } double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Domain Dm(db); Nx += 2; diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index 11e8c2a7..be7d00a4 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -14,6 +14,20 @@ #include "analysis/eikonal.h" +std::shared_ptr loadInputs( int nprocs ) +{ + INSIST(nprocs==8, "TestSegDist: Number of MPI processes must be equal to 8"); + auto db = std::make_shared( ); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 2, 2, 2 } ); + db->putVector( "n", { 50, 50, 50 } ); + db->putScalar( "nspheres", 0 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} + + +//*************************************************************************************** int main(int argc, char **argv) { // Initialize MPI @@ -25,39 +39,35 @@ int main(int argc, char **argv) { int i,j,k,n,nn; int iproc,jproc,kproc; - int nx,ny,nz; - int Nx, Ny, Nz, N; - int nprocx, nprocy, nprocz, nspheres; - double Lx, Ly, Lz; - Nx = Ny = Nz = 50; - nx = ny = nz = 50; - N = Nx*Ny*Nz; - nprocx=nprocy=nprocz=2; - Lx = Ly = Lz = 1.0; - int BC=0; - if (nprocs != 8){ - ERROR("TestSegDist: Number of MPI processes must be equal to 8"); - } - if (nprocx !=2 || nprocz !=2 || nprocy !=2 ){ - ERROR("TestSegDist: MPI process grid must be 2x2x2"); - } + + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + // Get the rank info - Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - for (k=0;k loadInputs( int nprocs ) +{ + auto db = std::make_shared( "Domain.in" ); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 100, 100, 100 } ); + db->putScalar( "nspheres", 1 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} int main(int argc, char **argv) @@ -34,65 +41,33 @@ int main(int argc, char **argv) //....................................................................... // Reading the domain information file //....................................................................... - int nprocx, nprocy, nprocz, nx, ny, nz, nspheres; - double Lx, Ly, Lz; - int Nx,Ny,Nz; int i,j,k,n; - if (rank==0){ - /* ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> nx; - domain >> ny; - domain >> nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - */ - // Set the domain for single processor test - nprocx=nprocy=nprocz=1; - nx=ny=nz=100; - nspheres=1; - Lx=Ly=Lz=1; - } - MPI_Barrier(comm); - // Computational domain - MPI_Bcast(&nx,1,MPI_INT,0,comm); - MPI_Bcast(&ny,1,MPI_INT,0,comm); - MPI_Bcast(&nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + + if (rank==0){ + printf("********************************************************\n"); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("********************************************************\n"); + } + - // Check that the number of processors >= the number of ranks - if ( rank==0 ) { - printf("Number of MPI ranks required: %i \n", nprocx*nprocy*nprocz); - printf("Number of MPI ranks used: %i \n", nprocs); - printf("Full domain size: %i x %i x %i \n",nx*nprocx,ny*nprocy,nz*nprocz); - } - if ( nprocs < nprocx*nprocy*nprocz ) - ERROR("Insufficient number of processors"); - // Set up the domain - int BC=0; // Get the rank info - Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + Domain Dm(db); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); TwoPhase Averages(Dm); - int N = (nx+2)*(ny+2)*(nz+2); - Nx = nx+2; - Ny = ny+2; - Nz = nz+2; - if (rank == 0) cout << "Domain set." << endl; + Nx += 2; + Ny += 2; + Nz += 2; + int N = Nx*Ny*Nz; //....................................................................... for ( k=1;k(1e-7*fabs(A+B),1e-20); } @@ -85,7 +85,7 @@ void testWriter( const std::string& format, std::vector& mes // Get a list of the timesteps PROFILE_START(format+"-read-timesteps"); - std::vector timesteps = IO::readTimesteps( path + "/" + summary_name ); + auto timesteps = IO::readTimesteps( path + "/" + summary_name ); PROFILE_STOP(format+"-read-timesteps"); if ( timesteps.size()==2 ) ut.passes(format+": Corrent number of timesteps"); @@ -117,7 +117,7 @@ void testWriter( const std::string& format, std::vector& mes pass = true; for (size_t k=0; k mesh = IO::getMesh(path,timestep,database,k); + auto mesh = IO::getMesh(path,timestep,database,k); PROFILE_STOP(format+"-read-getMesh"); if ( mesh.get()==NULL ) { printf("Failed to load %s\n",database.name.c_str()); @@ -126,7 +126,7 @@ void testWriter( const std::string& format, std::vector& mes } if ( database.name=="pointmesh" ) { // Check the pointmesh - std::shared_ptr pmesh = IO::getPointList(mesh); + auto pmesh = IO::getPointList(mesh); if ( pmesh.get()==NULL ) { pass = false; break; @@ -138,8 +138,8 @@ void testWriter( const std::string& format, std::vector& mes } if ( database.name=="trimesh" || database.name=="trilist" ) { // Check the trimesh/trilist - std::shared_ptr mesh1 = IO::getTriMesh(mesh); - std::shared_ptr mesh2 = IO::getTriList(mesh); + auto mesh1 = IO::getTriMesh(mesh); + auto mesh2 = IO::getTriList(mesh); if ( mesh1.get()==NULL || mesh2.get()==NULL ) { pass = false; break; @@ -198,8 +198,7 @@ void testWriter( const std::string& format, std::vector& mes auto mesh = IO::getMesh(path,timestep,database,k); for (size_t v=0; vvars.size(); v++) { PROFILE_START(format+"-read-getVariable"); - std::shared_ptr variable = - IO::getVariable(path,timestep,database,k,mesh0->vars[v]->name); + auto variable = IO::getVariable(path,timestep,database,k,mesh0->vars[v]->name); if ( format=="new" ) IO::reformatVariable( *mesh, *variable ); PROFILE_STOP(format+"-read-getVariable"); @@ -251,19 +250,19 @@ int main(int argc, char **argv) }; // Create the meshes - std::shared_ptr set1( new IO::PointList(N_points) ); + auto set1 = std::make_shared(N_points); for (int i=0; ipoints[i].x = x[i]; set1->points[i].y = y[i]; set1->points[i].z = z[i]; } - std::shared_ptr trimesh( new IO::TriMesh(N_tri,set1) ); + auto trimesh = std::make_shared(N_tri,set1); for (int i=0; iA[i] = tri[i][0]; trimesh->B[i] = tri[i][1]; trimesh->C[i] = tri[i][2]; } - std::shared_ptr trilist( new IO::TriList(*trimesh) ); + auto trilist = std::make_shared(*trimesh); for (int i=0; i domain( new IO::DomainMesh(rank_data,6,7,8,1.0,1.0,1.0) ); + auto domain = std::make_shared(rank_data,6,7,8,1.0,1.0,1.0); // Create the variables - const auto NodeVar = IO::VariableType::NodeVariable; - const auto VolVar = IO::VariableType::VolumeVariable; - std::shared_ptr set_node_mag( new IO::Variable(1,NodeVar,"Node_set_mag") ); - std::shared_ptr set_node_vec( new IO::Variable(3,NodeVar,"Node_set_vec") ); - std::shared_ptr list_node_mag( new IO::Variable(1,NodeVar,"Node_list_mag") ); - std::shared_ptr list_node_vec( new IO::Variable(3,NodeVar,"Node_list_vec") ); - std::shared_ptr point_node_mag( new IO::Variable(1,NodeVar,"Node_point_mag") ); - std::shared_ptr point_node_vec( new IO::Variable(3,NodeVar,"Node_point_vec") ); - std::shared_ptr domain_node_mag( new IO::Variable(1,NodeVar,"Node_domain_mag") ); - std::shared_ptr domain_node_vec( new IO::Variable(3,NodeVar,"Node_domain_vec") ); - std::shared_ptr set_cell_mag( new IO::Variable(1,VolVar,"Cell_set_mag") ); - std::shared_ptr set_cell_vec( new IO::Variable(3,VolVar,"Cell_set_vec") ); - std::shared_ptr list_cell_mag( new IO::Variable(1,VolVar,"Cell_list_mag") ); - std::shared_ptr list_cell_vec( new IO::Variable(3,VolVar,"Cell_list_vec") ); - std::shared_ptr domain_cell_mag( new IO::Variable(1,VolVar,"Cell_domain_mag") ); - std::shared_ptr domain_cell_vec( new IO::Variable(3,VolVar,"Cell_domain_vec") ); + const auto NodeVar = IO::VariableType::NodeVariable; + const auto VolVar = IO::VariableType::VolumeVariable; + auto set_node_mag = std::make_shared(1,NodeVar,"Node_set_mag"); + auto set_node_vec = std::make_shared(3,NodeVar,"Node_set_vec"); + auto list_node_mag = std::make_shared(1,NodeVar,"Node_list_mag"); + auto list_node_vec = std::make_shared(3,NodeVar,"Node_list_vec"); + auto point_node_mag = std::make_shared(1,NodeVar,"Node_point_mag"); + auto point_node_vec = std::make_shared(3,NodeVar,"Node_point_vec"); + auto domain_node_mag = std::make_shared(1,NodeVar,"Node_domain_mag"); + auto domain_node_vec = std::make_shared(3,NodeVar,"Node_domain_vec"); + auto set_cell_mag = std::make_shared(1,VolVar,"Cell_set_mag"); + auto set_cell_vec = std::make_shared(3,VolVar,"Cell_set_vec"); + auto list_cell_mag = std::make_shared(1,VolVar,"Cell_list_mag"); + auto list_cell_vec = std::make_shared(3,VolVar,"Cell_list_vec"); + auto domain_cell_mag = std::make_shared(1,VolVar,"Cell_domain_mag"); + auto domain_cell_vec = std::make_shared(3,VolVar,"Cell_domain_vec"); point_node_mag->data.resize( N_points ); point_node_vec->data.resize( N_points, 3 ); for (int i=0; i loadInputs( ) +{ + auto db = std::make_shared( "Domain.in" ); + const int dim = 50; + db->putScalar( "BC", 0 ); + return db; +} + + +//*************************************************************************************** int main(int argc, char **argv) { //***************************************** @@ -24,8 +35,6 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); { // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; int sendtag,recvtag; //***************************************** // MPI ranks for all 18 neighbors @@ -54,55 +63,24 @@ int main(int argc, char **argv) // Variables that specify the computational domain string FILENAME; - int Nx,Ny,Nz; // local sub-domain size - int nspheres; // number of spheres in the packing - double Lx,Ly,Lz; // Domain length int i,j,k,n; // pmmc threshold values - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); // ************************************************************** - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - + // Load inputs + auto db = loadInputs( ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + int kproc = rank/(nprocx*nprocy); + int jproc = (rank-nprocx*nprocy*kproc)/nprocx; + int iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; + if (rank==0){ printf("********************************************************\n"); printf("Sub-domain size = %i x %i x %i\n",Nz,Nz,Nz); @@ -111,7 +89,7 @@ int main(int argc, char **argv) } // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + Domain Dm(db); Dm.CommInit(comm); TwoPhase Averages(Dm); diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index 1ad4c245..d5568b13 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -96,7 +96,7 @@ int main(int argc, char **argv) int RESTART_INTERVAL=20000; //int ANALYSIS_)INTERVAL=1000; int BLOB_ANALYSIS_INTERVAL=1000; - int timestep = 0; + int timestep = 6; if (rank==0){ //............................................................. @@ -492,7 +492,7 @@ int main(int argc, char **argv) ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np); ScaLBL_DeviceBarrier(); delete [] TmpMap; - + // Compute the solid interaction potential and copy result to device if (rank==0) printf("Computing solid interaction potential \n"); double *Tmp; @@ -523,14 +523,14 @@ int main(int argc, char **argv) for (int kk=0; kk<5; kk++){ for (int jj=0; jj<5; jj++){ for (int ii=0; ii<5; ii++){ - + int index = kk*25+jj*5+ii; double distval= Dst[index]; int idi=i+ii-2; int idj=j+jj-2; int idk=k+kk-2; - + if (idi < 0) idi=0; if (idj < 0) idj=0; if (idk < 0) idk=0; @@ -540,16 +540,16 @@ int main(int argc, char **argv) int nn = idk*Nx*Ny + idj*Nx + idi; if (!(Mask.id[nn] > 0)){ - double vec_x = double(ii-2); - double vec_y = double(jj-2); - double vec_z = double(kk-2); - - double ALPHA=PhaseLabel[nn]; - double GAMMA=-2.f; - if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance - phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; - phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; - phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; + double vec_x = double(ii-2); + double vec_y = double(jj-2); + double vec_z = double(kk-2); + + double ALPHA=PhaseLabel[nn]; + double GAMMA=-2.f; + if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance + phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; + phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; + phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; } } } @@ -557,17 +557,17 @@ int main(int argc, char **argv) Tmp[idx] = phi_x; Tmp[idx+Np] = phi_y; Tmp[idx+2*Np] = phi_z; - + /* double d = Averages->SDs(n); double dx = Averages->SDs_x(n); double dy = Averages->SDs_y(n); double dz = Averages->SDs_z(n); double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d)); - + Tmp[idx] = value*dx; Tmp[idx+Np] = value*dy; Tmp[idx+2*Np] = value*dz; - */ + */ } } } @@ -642,7 +642,7 @@ int main(int argc, char **argv) // Copy the phase from the GPU -> CPU //........................................................................... ScaLBL_DeviceBarrier(); - ScaLBL_Comm.RegularLayout(Map,Phi,Averages->Phase); + ScaLBL_CopyToHost(Averages->Phase.data(),Phi,Np*sizeof(double)); ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press); ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x); ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y); @@ -751,7 +751,7 @@ int main(int argc, char **argv) PROFILE_STOP("Update"); // Run the analysis - analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); + // analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); } analysis.finish(); @@ -782,8 +782,7 @@ int main(int argc, char **argv) FILE *OUTFILE; sprintf(LocalRankFilename,"Phase.%05i.raw",rank); OUTFILE = fopen(LocalRankFilename,"wb"); - //fwrite(PhaseField.data(),8,N,OUTFILE); - fwrite(Averages->Phase.data(),8,N,OUTFILE); + fwrite(PhaseField.data(),8,N,OUTFILE); fclose(OUTFILE); PROFILE_STOP("Main"); diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index c6804988..05770f25 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -274,7 +274,7 @@ int main(int argc, char **argv) n=k*nx*ny+j*nx+i; total++; Dm.id[n] = newIDs[n]; - if (!(Dm.id[n] > 0)){ + if (Dm.id[n] == 0){ count++; } } @@ -307,7 +307,7 @@ int main(int argc, char **argv) } } float porosity = float(totalGlobal-countGlobal)/totalGlobal; - //printf("Original Porosity=%f\n",porosity); + printf("Original Porosity=%f\n",porosity); } count = 0; diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 605fe331..3bdbf094 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -228,40 +228,18 @@ int main(int argc, char **argv) // DoubleArray Distance(nx,ny,nz); // DoubleArray Phase(nx,ny,nz); + int count = 0; // Solve for the position of the solid phase for (k=0;k 0)) id[n] = 0; + if (Dm.id[n] == 0) id[n] = 0; else id[n] = 1; } } } - - - int count = 0; - int total = 0; - int totalGlobal = 0; - int countGlobal = 0; - for (k=1;k 0)){ - count++; - } - } - } - } - MPI_Allreduce(&total,&totalGlobal,1,MPI_INT,MPI_SUM,comm); - MPI_Allreduce(&count,&countGlobal,1,MPI_INT,MPI_SUM,comm); - float poros = float(totalGlobal-countGlobal)/totalGlobal; - if (rank==0) printf("Porosity=%f\n",poros); - - // Initialize the signed distance function for (k=0;k Date: Tue, 15 May 2018 11:30:12 -0400 Subject: [PATCH 014/124] Working on moving to Database --- common/Database.h | 8 ++ example/Bubble/Color.in | 6 -- example/Bubble/Domain.in | 3 - example/Bubble/input.in | 28 +++++++ tests/TestBubble.cpp | 157 ++++++++++++--------------------------- tests/TestPoiseuille.cpp | 6 -- 6 files changed, 83 insertions(+), 125 deletions(-) delete mode 100644 example/Bubble/Color.in delete mode 100644 example/Bubble/Domain.in create mode 100644 example/Bubble/input.in diff --git a/common/Database.h b/common/Database.h index 4bf0bf61..dc2f846c 100644 --- a/common/Database.h +++ b/common/Database.h @@ -1,6 +1,7 @@ #ifndef included_Database #define included_Database +#include #include #include #include @@ -10,6 +11,13 @@ #include "common/Units.h" +inline bool exists( const std::string& filename ) +{ + std::ifstream domain( filename ); + return domain.good(); +} + + //! Base class to hold data of a given type class KeyData { diff --git a/example/Bubble/Color.in b/example/Bubble/Color.in deleted file mode 100644 index 257e789c..00000000 --- a/example/Bubble/Color.in +++ /dev/null @@ -1,6 +0,0 @@ -1.0 -1.0e-2 0.95 0.8 -0.7 -0.0 0.0 0.0 -0 0 1.0 1.0 -200 1000 1e-5 diff --git a/example/Bubble/Domain.in b/example/Bubble/Domain.in deleted file mode 100644 index 97770beb..00000000 --- a/example/Bubble/Domain.in +++ /dev/null @@ -1,3 +0,0 @@ -1 1 1 -80 80 80 -1.0 1.0 1.0 diff --git a/example/Bubble/input.in b/example/Bubble/input.in new file mode 100644 index 00000000..ae2764cb --- /dev/null +++ b/example/Bubble/input.in @@ -0,0 +1,28 @@ +Color { + tau = 1.0; + alpha = 1e-2; + beta = 0.95; + phi_s = 0.8; + wp_saturation = 0.7 + F = 0, 0, 0 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 200 + interval = 1000 + tol = 1e-5; + das = 0.1 + dbs = 0.9 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) +} + +Analysis { + +} diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index 92b1a46c..84aaddcd 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -16,6 +16,17 @@ using namespace std; +std::shared_ptr readInput( const std::string& file ) +{ + if ( exists( file ) + return std::make_shared( file ); + auto db = std::make_shared(); + db.putData( "Color", std::make_shared() ); + db.putData( "Domain", std::make_shared() ); + return db; +} + + int main(int argc, char **argv) { // Initialize MPI @@ -54,13 +65,6 @@ int main(int argc, char **argv) int nspheres; double Lx,Ly,Lz; // Color Model parameters - int timestepMax, interval; - double tau,Fx,Fy,Fz,tol; - double alpha, beta; - double das, dbs, phi_s; - double din,dout; - double wp_saturation; - bool pBC,Restart; int i,j,k,n; // pmmc threshold values @@ -71,109 +75,43 @@ int main(int argc, char **argv) nthreads = 128; int RESTART_INTERVAL=1000; - - if (rank==0){ - //............................................................. - // READ SIMULATION PARMAETERS FROM INPUT FILE - //............................................................. - ifstream input("Color.in"); - if (input.good()){ - input >> tau; // Viscosity parameter - input >> alpha; // Surface Tension parameter - input >> beta; // Width of the interface - input >> phi_s; // value of phi at the solid surface - input >> wp_saturation; - input >> Fx; - input >> Fy; - input >> Fz; - input >> Restart; - input >> pBC; - input >> din; - input >> dout; - input >> timestepMax; // max no. of timesteps - input >> interval; // restart interval - input >> tol; // error tolerance - } - else { - printf("WARNING: No valid Color.in file, using hard-coded values \n"); - tau = 1.0; - alpha = 1e-3; - beta = 0.95; - phi_s = 0.0; - wp_saturation=Fx=Fy=Fz=0; - Restart = 0; - pBC = 0; - din=dout = 1.0; - timestepMax = 3; - interval = 100; - tol = 1e-6; - das = 0.1; dbs = 0.9; // hard coded for density initialization - // should be OK to remove these parameters - // they should have no impact with the - // current boundary condition - } - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else { - //....................................................................... - // Set the domain for single processor test - printf("WARNING: No valid Domain.in file, using hard-coded values \n"); - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=50; - nspheres=1; - Lx=Ly=Lz=1; - } - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&pBC,1,MPI_LOGICAL,0,comm); - MPI_Bcast(&Restart,1,MPI_LOGICAL,0,comm); - MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - MPI_Bcast(&interval,1,MPI_INT,0,comm); - MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - // MPI_Bcast(&nBlocks,1,MPI_INT,0,comm); - // MPI_Bcast(&nthreads,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. + auto db = readInput( "input.in" ); + + // Read variables from Color + auto color_db = db->getDatabase( "Color" ); + auto tau = color_db->getScalarWithDefault( "tau", 1.0 ); + auto alpha = color_db->getScalarWithDefault( "alpha", 1e-3 ); + auto beta = color_db->getScalarWithDefault( "beta", 0.95 ); + auto phi_s = color_db->getScalarWithDefault( "phi_s", 0.0 ); + auto Fx = color_db->getVectorWithDefault( "F", { 0, 0, 0 } )[0]; + auto Fy = color_db->getScalarWithDefault( "F", { 0, 0, 0 } )[1]; + auto Fz = color_db->getScalarWithDefault( "F", { 0, 0, 0 } )[2]; + auto Restart = color_db->getScalarWithDefault( "Restart", 0 ); + auto pBC = color_db->getScalarWithDefault( "pBC", 0 ); + auto din = color_db->getScalarWithDefault( "din", 1.0 ); + auto dout = color_db->getScalarWithDefault( "dout", 1.0 ); + auto timestepMax = color_db->getScalarWithDefault( "timestepMax", 3 ); + auto interval = color_db->getScalarWithDefault( "interval", 100 ); + auto tol = color_db->getScalarWithDefault( "tol", 1e-6 ); + auto das = color_db->getScalarWithDefault( "das", 0.1 ); + auto dbs = color_db->getScalarWithDefault( "dab", 0.9 ); + + // Read variables from Domain + auto domain_db = db->getDatabase( "Domain" ); + auto n = domain_db->getVectorWithDefault( "n", { 50, 50, 50 } ); + auto L = domain_db->getVectorWithDefault( "L", { 1.0, 1.0, 1.0 } ); + auto nproc = domain_db->getVectorWithDefault( "nproc", { 1, 1, 1 } ); + auto nspheres = domain_db->getScalarWithDefault( "nspheres", 1 ); + int Nx = n[0]; + int Ny = n[1]; + int Nz = n[2]; + int nprocx = nproc[0]; + int nprocy = nproc[1]; + int nprocz = nproc[2]; + double Lx = L[0]; + double Ly = L[1]; + double Lz = L[2]; // Get the rank info const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -227,7 +165,6 @@ int main(int argc, char **argv) MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; - //Nx = Ny = Nz; // Cubic domain int N = Nx*Ny*Nz; int dist_mem_size = N*sizeof(double); diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index ad8801d1..96debec5 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -10,12 +10,6 @@ #include "common/MPI_Helpers.h" -bool exists( const std::string& filename ) -{ - ifstream domain( filename ); - return domain.good(); -} - std::shared_ptr loadInputs( int nprocs ) { From 6ab72207c855513cc28756416748f9e45394c31f Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Tue, 15 May 2018 11:59:01 -0400 Subject: [PATCH 015/124] Moving runAnalysis to database input --- analysis/runAnalysis.cpp | 45 +++++++++++++++++----------- analysis/runAnalysis.h | 8 ++--- example/Bubble/input.in | 10 ++++++- tests/TestBubbleDFH.cpp | 4 +-- tests/lbpm_color_macro_simulator.cpp | 4 +-- tests/lbpm_dfh_simulator.cpp | 19 ++---------- 6 files changed, 46 insertions(+), 44 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index bffbfdb8..45581aae 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -281,30 +281,35 @@ runAnalysis::commWrapper runAnalysis::getComm( ) /****************************************************************** * Constructor/Destructors * ******************************************************************/ -runAnalysis::runAnalysis( int restart_interval, int analysis_interval, - int blobid_interval, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm, - int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err, - IntArray Map, const std::string& LocalRestartFile ): +runAnalysis::runAnalysis( std::shared_ptr db, + const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm, + int Np, bool pBC, double beta, double err, + IntArray Map ): d_Np( Np ), - d_restart_interval( restart_interval ), - d_analysis_interval( analysis_interval ), - d_blobid_interval( blobid_interval ), d_beta( beta ), d_ScaLBL_Comm( ScaLBL_Comm ), d_rank_info( rank_info ), d_Map( Map ), - d_fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1), - d_restartFile( LocalRestartFile ) + d_fillData(Dm.Comm,Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,1,1,1,0,1) { - d_N[0] = Nx; - d_N[1] = Ny; - d_N[2] = Nz; + INSIST( db, "Input database is empty" ); + char rankString[20]; + sprintf(rankString,"%05d",Dm.rank); + d_N[0] = Dm.Nx; + d_N[1] = Dm.Ny; + d_N[2] = Dm.Nz; + d_restart_interval = db->getScalar( "restart_interval" ); + d_analysis_interval = db->getScalar( "analysis_interval" ); + d_blobid_interval = db->getScalar( "blobid_interval" ); + d_visualization_interval = db->getScalar( "visualization_interval" ); + auto restart_file = db->getScalar( "restart_file" ); + d_restartFile = restart_file + "." + rankString; d_rank = MPI_WORLD_RANK(); writeIDMap(ID_map_struct(),0,id_map_filename); // Create the MeshDataStruct d_meshData.resize(1); d_meshData[0].meshName = "domain"; - d_meshData[0].mesh = std::make_shared( Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz ); + d_meshData[0].mesh = std::make_shared( Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.Lx,Dm.Ly,Dm.Lz ); auto PhaseVar = std::make_shared(); auto PressVar = std::make_shared(); auto SignDistVar = std::make_shared(); @@ -312,22 +317,22 @@ runAnalysis::runAnalysis( int restart_interval, int analysis_interval, PhaseVar->name = "phase"; PhaseVar->type = IO::VariableType::VolumeVariable; PhaseVar->dim = 1; - PhaseVar->data.resize(Nx-2,Ny-2,Nz-2); + PhaseVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); d_meshData[0].vars.push_back(PhaseVar); PressVar->name = "Pressure"; PressVar->type = IO::VariableType::VolumeVariable; PressVar->dim = 1; - PressVar->data.resize(Nx-2,Ny-2,Nz-2); + PressVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); d_meshData[0].vars.push_back(PressVar); SignDistVar->name = "SignDist"; SignDistVar->type = IO::VariableType::VolumeVariable; SignDistVar->dim = 1; - SignDistVar->data.resize(Nx-2,Ny-2,Nz-2); + SignDistVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); d_meshData[0].vars.push_back(SignDistVar); BlobIDVar->name = "BlobID"; BlobIDVar->type = IO::VariableType::VolumeVariable; BlobIDVar->dim = 1; - BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2); + BlobIDVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); d_meshData[0].vars.push_back(BlobIDVar); // Initialize the comms MPI_Comm_dup(MPI_COMM_WORLD,&d_comm); @@ -335,6 +340,10 @@ runAnalysis::runAnalysis( int restart_interval, int analysis_interval, d_comms[i] = MPI_COMM_NULL; d_comm_used[i] = false; } + // Initialize the threads + int N_threads = db->getWithDefault( "N_threads", 4 ); + auto method = db->getWithDefault( "load_balance", "default" ); + createThreads( method, N_threads ); } runAnalysis::~runAnalysis( ) { @@ -453,7 +462,7 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep ) // Write the restart file type |= AnalysisType::CreateRestart; } - if (timestep%d_restart_interval == 0) { + if (timestep%d_visualization_interval == 0) { // Write the visualization data type |= AnalysisType::WriteVis; type |= AnalysisType::CopySimState; diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index 9bd7aa65..c8dc1260 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -23,10 +23,10 @@ class runAnalysis public: //! Constructor - runAnalysis( int restart_interval, int analysis_interval, int blobid_interval, + runAnalysis( std::shared_ptr db, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm, - int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err, - IntArray Map, const std::string& LocalRestartFile ); + int Np, bool pBC, double beta, double err, + IntArray Map ); //! Destructor ~runAnalysis(); @@ -85,7 +85,7 @@ private: int d_N[3]; int d_Np; int d_rank; - int d_restart_interval, d_analysis_interval, d_blobid_interval; + int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval; double d_beta; ThreadPool d_tpool; ScaLBL_Communicator d_ScaLBL_Comm; diff --git a/example/Bubble/input.in b/example/Bubble/input.in index ae2764cb..5bee6d4d 100644 --- a/example/Bubble/input.in +++ b/example/Bubble/input.in @@ -24,5 +24,13 @@ Domain { } Analysis { - + blobid_interval = 1000 // Frequency to perform blob identification + analysis_interval = 1000 // Frequency to perform analysis + restart_interval = 20000 // Frequency to write restart data + vis_interval = 20000 // Frequency to write visualization data + restart_file = "Restart" // Filename to use for restart file (will append rank) + N_threads = 4 // Number of threads to use + load_balance = "independent" // Load balance method to use: "none", "default", "independent" } + + diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index bf7f3b2c..f5a730d4 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -503,8 +503,8 @@ int main(int argc, char **argv) //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL, - rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile ); + std::shared_ptr analysis_db; + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index f2a1a870..401dd100 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -550,8 +550,8 @@ int main(int argc, char **argv) //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL, - rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile ); + std::shared_ptr analysis_db; + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index d5568b13..c22b6702 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -60,16 +60,6 @@ int main(int argc, char **argv) PROFILE_START("Main"); Utilities::setErrorHandlers(); - int ANALYSIS_INTERVAL = 1000; - int BLOBID_INTERVAL = 1000; - std::string analysis_method = "independent"; - if (argc >= 3) { - ANALYSIS_INTERVAL = atoi(argv[1]); - BLOBID_INTERVAL = atoi(argv[2]); - } - if (argc >= 4) - analysis_method = std::string(argv[3]); - // Variables that specify the computational domain string FILENAME; int Nx,Ny,Nz,Np; // local sub-domain size @@ -94,7 +84,6 @@ int main(int argc, char **argv) dout=1.f; int RESTART_INTERVAL=20000; - //int ANALYSIS_)INTERVAL=1000; int BLOB_ANALYSIS_INTERVAL=1000; int timestep = 6; @@ -662,15 +651,11 @@ int main(int argc, char **argv) err = 1.0; double sat_w_previous = 1.01; // slightly impossible value! if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol); - if (rank==0){ - printf("Analysis intervals: (restart) %i, (TCAT) %i, (blobtracking) %i \n",RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL); - } //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - runAnalysis analysis( RESTART_INTERVAL,ANALYSIS_INTERVAL,BLOBID_INTERVAL, - rank_info, ScaLBL_Comm, Dm, Np, Nx, Ny, Nz, Lx, Ly, Lz, pBC, beta, err, Map, LocalRestartFile ); - analysis.createThreads( analysis_method, 4 ); + std::shared_ptr analysis_db; + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } PROFILE_START("Update"); From c4b084c306b48ad9fa8679e06266d612ee9cc1c6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 14:04:48 -0400 Subject: [PATCH 016/124] Udating unit tests and examples to read from database --- example/Bubble/input.in | 1 + example/Piston/input.db | 37 +++++++++++++++++++++++++++++++++++ example/Sph125/input.db | 37 +++++++++++++++++++++++++++++++++++ example/Sph1896/input.db | 37 +++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 +- tests/TestPoiseuille.cpp | 42 +++++++++++----------------------------- 6 files changed, 124 insertions(+), 32 deletions(-) create mode 100644 example/Piston/input.db create mode 100644 example/Sph125/input.db create mode 100644 example/Sph1896/input.db diff --git a/example/Bubble/input.in b/example/Bubble/input.in index 5bee6d4d..3e8edefa 100644 --- a/example/Bubble/input.in +++ b/example/Bubble/input.in @@ -21,6 +21,7 @@ Domain { n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) n_spheres = 1 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type } Analysis { diff --git a/example/Piston/input.db b/example/Piston/input.db new file mode 100644 index 00000000..3e8edefa --- /dev/null +++ b/example/Piston/input.db @@ -0,0 +1,37 @@ +Color { + tau = 1.0; + alpha = 1e-2; + beta = 0.95; + phi_s = 0.8; + wp_saturation = 0.7 + F = 0, 0, 0 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 200 + interval = 1000 + tol = 1e-5; + das = 0.1 + dbs = 0.9 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type +} + +Analysis { + blobid_interval = 1000 // Frequency to perform blob identification + analysis_interval = 1000 // Frequency to perform analysis + restart_interval = 20000 // Frequency to write restart data + vis_interval = 20000 // Frequency to write visualization data + restart_file = "Restart" // Filename to use for restart file (will append rank) + N_threads = 4 // Number of threads to use + load_balance = "independent" // Load balance method to use: "none", "default", "independent" +} + + diff --git a/example/Sph125/input.db b/example/Sph125/input.db new file mode 100644 index 00000000..3e8edefa --- /dev/null +++ b/example/Sph125/input.db @@ -0,0 +1,37 @@ +Color { + tau = 1.0; + alpha = 1e-2; + beta = 0.95; + phi_s = 0.8; + wp_saturation = 0.7 + F = 0, 0, 0 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 200 + interval = 1000 + tol = 1e-5; + das = 0.1 + dbs = 0.9 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type +} + +Analysis { + blobid_interval = 1000 // Frequency to perform blob identification + analysis_interval = 1000 // Frequency to perform analysis + restart_interval = 20000 // Frequency to write restart data + vis_interval = 20000 // Frequency to write visualization data + restart_file = "Restart" // Filename to use for restart file (will append rank) + N_threads = 4 // Number of threads to use + load_balance = "independent" // Load balance method to use: "none", "default", "independent" +} + + diff --git a/example/Sph1896/input.db b/example/Sph1896/input.db new file mode 100644 index 00000000..3e8edefa --- /dev/null +++ b/example/Sph1896/input.db @@ -0,0 +1,37 @@ +Color { + tau = 1.0; + alpha = 1e-2; + beta = 0.95; + phi_s = 0.8; + wp_saturation = 0.7 + F = 0, 0, 0 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 200 + interval = 1000 + tol = 1e-5; + das = 0.1 + dbs = 0.9 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type +} + +Analysis { + blobid_interval = 1000 // Frequency to perform blob identification + analysis_interval = 1000 // Frequency to perform analysis + restart_interval = 20000 // Frequency to write restart data + vis_interval = 20000 // Frequency to write visualization data + restart_file = "Restart" // Filename to use for restart file (will append rank) + N_threads = 4 // Number of threads to use + load_balance = "independent" // Load balance method to use: "none", "default", "independent" +} + + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 92bfcfe0..2121a88b 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,7 +47,7 @@ ADD_LBPM_TEST( TestBubbleDFH ) ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestColorMassBounceback ) ADD_LBPM_TEST( TestPressVel ) -ADD_LBPM_TEST( TestPoiseuille ) +ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) ADD_LBPM_TEST( TestForceMoments ) ADD_LBPM_TEST( TestForceD3Q19 ) ADD_LBPM_TEST( TestMomentsD3Q19 ) diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 96debec5..7e6a4407 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -11,26 +11,6 @@ -std::shared_ptr loadInputs( int nprocs ) -{ - const int dim = 12; - std::shared_ptr db; - if ( exists( "Domain.in" ) ) { - db = std::make_shared( "Domain.in" ); - } else if (nprocs==1) { - db->putVector( "nproc", { 1, 1, 1 } ); - db->putVector( "n", { dim, dim, dim } ); - db->putScalar( "nspheres", 0 ); - db->putVector( "L", { 1, 1, 1 } ); - } else if (nprocs==2) { - db->putVector( "nproc", { 1, 1, 1 } ); - db->putVector( "n", { dim/2, dim/2, dim/2 } ); - db->putScalar( "nspheres", 0 ); - db->putVector( "L", { 1, 1, 1 } ); - } - db->putScalar( "BC", 0 ); - return db; -} //*************************************************************************************** @@ -54,7 +34,7 @@ int main(int argc, char **argv) } // BGK Model parameters - string FILENAME; + string FILENAME = argv[1]; unsigned int nBlocks, nthreads; int timestepMax, interval; double tau,Fx,Fy,Fz,tol; @@ -71,15 +51,15 @@ int main(int argc, char **argv) Fz = 1e-3; //1.f; // 1e-3; // Load inputs - auto db = loadInputs( nprocs ); - int Nx = db->getVector( "n" )[0]; - int Ny = db->getVector( "n" )[1]; - int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; - - + if (rank==0) printf("Loading input database \n"); + auto db = std::make_shared(FILENAME); + auto domain_db= db-> getDatabase("Domain"); + int Nx = domain_db->getVector( "n" )[0]; + int Ny = domain_db->getVector( "n" )[1]; + int Nz = domain_db->getVector( "n" )[2]; + int nprocx = domain_db->getVector( "nproc" )[0]; + int nprocy = domain_db->getVector( "nproc" )[1]; + int nprocz = domain_db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); @@ -102,7 +82,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(db); + Domain Dm(domain_db); Nx += 2; Ny += 2; From dfb6d1e5c79d9c23b92e8da49798eda1cc538bc1 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Tue, 15 May 2018 14:06:53 -0400 Subject: [PATCH 017/124] Updating lbpm_dfh_simulator --- analysis/pmmc.h | 52 +- analysis/runAnalysis.cpp | 8 +- analysis/runAnalysis.h | 6 +- example/Bubble/input.in | 1 + tests/TestBubbleDFH.cpp | 2 +- tests/lbpm_color_macro_simulator.cpp | 2 +- tests/lbpm_dfh_simulator.cpp | 1318 ++++++++++++-------------- 7 files changed, 664 insertions(+), 725 deletions(-) diff --git a/analysis/pmmc.h b/analysis/pmmc.h index 64c6254e..b56a2796 100644 --- a/analysis/pmmc.h +++ b/analysis/pmmc.h @@ -745,12 +745,20 @@ inline Point VertexInterp(const Point &p1, const Point &p2, double valp1, double inline void SolidMarchingCubes(DoubleArray &A, const double &v, DoubleArray &B, const double &isovalue, int i,int j,int k, int m, int n, int o, DTMutableList &cellvertices, int &lengthvertices, IntArray &Triangles, int &nTris, - DoubleArray &values){ + DoubleArray &values) +{ + + NULL_USE( isovalue ); + NULL_USE( m ); + NULL_USE( n ); + NULL_USE( o ); // THIS SUBROUTINE COMPUTES THE VERTICES FOR THE SOLID PHASE IN // A PARTICULAR GRID CELL, THEN ARRANGES THEM INTO TRIANGLES // ALSO ORGANIZES THE LIST OF VALUES TO CORRESPOND WITH VERTEX LIST + NULL_USE( v ); + //int N = 0; Point P,Q; Point PlaceHolder; @@ -1079,7 +1087,12 @@ inline void SolidMarchingCubes(DoubleArray &A, const double &v, DoubleArray &B, inline void SOL_SURF(DoubleArray &A, const double &v, DoubleArray &B, const double &isovalue, int i,int j,int k, int m, int n, int o, DTMutableList &cellvertices, int &lengthvertices, IntArray &Tlist, int &nTris, - DoubleArray &values){ + DoubleArray &values) +{ + NULL_USE( isovalue ); + NULL_USE( m ); + NULL_USE( n ); + NULL_USE( o ); // THIS SUBROUTINE COMPUTES THE VERTICES FOR THE SOLID PHASE IN // A PARTICULAR GRID CELL, THEN ARRANGES THEM INTO TRIANGLES @@ -2540,6 +2553,11 @@ inline void EDGE(DoubleArray &A, double &v, DoubleArray &solid, int &i, int &j, DTMutableList &nw_pts, int &n_nw_pts, IntArray &nw_tris, int &n_nw_tris, DTMutableList &local_nws_pts, int &n_local_nws_pts) { + + NULL_USE( m ); + NULL_USE( n ); + NULL_USE( o ); + // FIND THE POINTS ON THE nw SURFACE THAT ARE ON THE EDGE (COMMON LINE WITH SOLID PHASE) // function A is the fluid data padded (so that it has values inside the solid phase) @@ -3319,6 +3337,8 @@ inline void pmmc_MeshCurvature(DoubleArray &f, DoubleArray &MeanCurvature, Doubl //-------------------------------------------------------------------------------------------------------- inline int pmmc_CubeListFromMesh(IntArray &cubeList, int ncubes, int Nx, int Ny, int Nz) { + NULL_USE( ncubes ); + int i,j,k,nc; nc=0; //........................................................................... @@ -3974,6 +3994,8 @@ inline double pmmc_CommonCurveSpeed(DoubleArray &CubeValues, DoubleArray &dPdt, DoubleArray &Sx, DoubleArray &Sy, DoubleArray &Sz, DTMutableList &Points, int i, int j, int k, int npts) { + NULL_USE( CubeValues ); + int p; double s,lwns,norm; double zeta; @@ -4125,7 +4147,12 @@ inline void pmmc_CurveCurvature(DoubleArray &f, DoubleArray &s, DoubleArray &f_x, DoubleArray &f_y, DoubleArray &f_z, DoubleArray &s_x, DoubleArray &s_y, DoubleArray &s_z, DoubleArray &KN, DoubleArray &KG, - double &KNavg, double &KGavg, DTMutableList &Points, int npts, int ic, int jc, int kc){ + double &KNavg, double &KGavg, DTMutableList &Points, int npts, int ic, int jc, int kc) +{ + NULL_USE( f ); + NULL_USE( s ); + NULL_USE( KN ); + NULL_USE( KG ); int p,i,j,k; double length; @@ -4324,6 +4351,10 @@ inline double pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArr DoubleArray &SurfaceVector, DoubleArray &AvgSpeed, DoubleArray &AvgVel, int i, int j, int k, int npts, int ntris) { + NULL_USE( CubeValues ); + NULL_USE( SurfaceVector ); + NULL_USE( npts ); + Point A,B,C,P; double x,y,z; double s,s1,s2,s3,area; @@ -4378,7 +4409,15 @@ inline double pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArr } //-------------------------------------------------------------------------------------------------------- inline double geomavg_EulerCharacteristic(DTMutableList &Points, IntArray &Triangles, - int &npts, int &ntris, int &i, int &j, int &k){ + int &npts, int &ntris, int &i, int &j, int &k) +{ + NULL_USE( Points ); + NULL_USE( Triangles ); + NULL_USE( npts ); + NULL_USE( ntris ); + NULL_USE( i ); + NULL_USE( j ); + NULL_USE( k ); /* REFERENCE * Huang, Liu, Lee, Yang, Tsang @@ -4506,7 +4545,10 @@ inline double mink_phase_epc6(IntArray &PhaseID, DoubleArray &CubeValues, int Ph } inline double mink_EulerCharacteristic(DTMutableList &Points, IntArray &Triangles, - DoubleArray &CubeValues, int &npts, int &ntris, int &i, int &j, int &k){ + DoubleArray &CubeValues, int &npts, int &ntris, int &i, int &j, int &k) +{ + + NULL_USE( CubeValues ); // Compute the Euler characteristic for triangles in a cube // Exclude edges and vertices shared with between multiple cubes diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 45581aae..8f9c05e8 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -237,14 +237,14 @@ private: * MPI comm wrapper for use with analysis * ******************************************************************/ runAnalysis::commWrapper::commWrapper( int tag_, MPI_Comm comm_, runAnalysis* analysis_ ): - tag(tag_), comm(comm_), + tag(tag_), analysis(analysis_) { } runAnalysis::commWrapper::commWrapper( commWrapper &&rhs ): - tag(rhs.tag), comm(rhs.comm), + tag(rhs.tag), analysis(rhs.analysis) { rhs.tag = -1; @@ -283,8 +283,7 @@ runAnalysis::commWrapper runAnalysis::getComm( ) ******************************************************************/ runAnalysis::runAnalysis( std::shared_ptr db, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm, - int Np, bool pBC, double beta, double err, - IntArray Map ): + int Np, bool pBC, double beta, IntArray Map ): d_Np( Np ), d_beta( beta ), d_ScaLBL_Comm( ScaLBL_Comm ), @@ -292,6 +291,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, d_Map( Map ), d_fillData(Dm.Comm,Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,1,1,1,0,1) { + NULL_USE( pBC ); INSIST( db, "Input database is empty" ); char rankString[20]; sprintf(rankString,"%05d",Dm.rank); diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index c8dc1260..b3bd83b5 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -23,10 +23,8 @@ class runAnalysis public: //! Constructor - runAnalysis( std::shared_ptr db, - const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm, - int Np, bool pBC, double beta, double err, - IntArray Map ); + runAnalysis( std::shared_ptr db, const RankInfoStruct& rank_info, + const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm, int Np, bool pBC, double beta, IntArray Map ); //! Destructor ~runAnalysis(); diff --git a/example/Bubble/input.in b/example/Bubble/input.in index 5bee6d4d..91d748a0 100644 --- a/example/Bubble/input.in +++ b/example/Bubble/input.in @@ -21,6 +21,7 @@ Domain { n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) n_spheres = 1 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition } Analysis { diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index f5a730d4..1075247b 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -504,7 +504,7 @@ int main(int argc, char **argv) //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); std::shared_ptr analysis_db; - runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index 401dd100..0ef35bfb 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -551,7 +551,7 @@ int main(int argc, char **argv) //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); std::shared_ptr analysis_db; - runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index c22b6702..fbf5d554 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -24,759 +24,657 @@ using namespace std; //************************************************************************* // Implementation of Two-Phase Immiscible LBM //************************************************************************* - int main(int argc, char **argv) { - // Initialize MPI - int provided_thread_support = -1; - MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD,&comm); - int rank = comm_rank(comm); - int nprocs = comm_size(comm); - { // Limit scope so variables that contain communicators will free before MPI_Finialize + // Initialize MPI + int provided_thread_support = -1; + MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided_thread_support); + MPI_Comm comm; + MPI_Comm_dup(MPI_COMM_WORLD,&comm); + int rank = comm_rank(comm); + int nprocs = comm_size(comm); + { // Limit scope so variables that contain communicators will free before MPI_Finialize - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; + if ( argc < 2 ) { + std::cerr << "Invalid number of arguments, no input file specified\n"; + return -1; + } + auto filename = argv[1]; + + auto db = std::make_shared( filename ); + auto domain_db = db->getDatabase( "Domain" ); + auto color_db = db->getDatabase( "Color" ); + auto analysis_db = db->getDatabase( "Analysis" ); - MPI_Request req1[18],req2[18]; - MPI_Status stat1[18],stat2[18]; + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Color LBM \n"); + printf("********************************************************\n"); + } + // Initialize compute device + // int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); - if (rank == 0){ - printf("********************************************************\n"); - printf("Running Color LBM \n"); - printf("********************************************************\n"); - } - // Initialize compute device - // int device=ScaLBL_SetDevice(rank); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); - PROFILE_ENABLE(1); - //PROFILE_ENABLE_TRACE(); - //PROFILE_ENABLE_MEMORY(); - PROFILE_SYNCHRONIZE(); - PROFILE_START("Main"); - Utilities::setErrorHandlers(); + // Variables that specify the computational domain + string FILENAME; - // Variables that specify the computational domain - string FILENAME; - int Nx,Ny,Nz,Np; // local sub-domain size - double Lx,Ly,Lz; // Domain length - double D = 1.0; // reference length for non-dimensionalization - // Color Model parameters - int timestepMax; - double tauA, tauB, rhoA,rhoB; - double Fx,Fy,Fz,tol,err; - double alpha, beta; - int BoundaryCondition; - int InitialCondition; - // bool pBC,Restart; - int i,j,k,n; - double din, dout, flux; - double inletA,inletB,outletA,outletB; - inletA=1.f; - inletB=0.f; - outletA=0.f; - outletB=1.f; - flux = 10.f; - dout=1.f; + // Color Model parameters + int timestepMax = domain_db->getScalar( "timestepMax" ); + double tauA = domain_db->getScalar( "tauA" ); + double tauB = domain_db->getScalar( "tauB" ); + double rhoA = domain_db->getScalar( "rhoA" ); + double rhoB = domain_db->getScalar( "rhoB" ); + double Fx = domain_db->getVector( "F" )[0]; + double Fy = domain_db->getVector( "F" )[1]; + double Fz = domain_db->getVector( "F" )[2]; + double alpha = domain_db->getScalar( "alpha" ); + double beta = domain_db->getScalar( "beta" ); + bool Restart = domain_db->getScalar( "Restart" ); + double din = domain_db->getScalar( "din" ); + double dout = domain_db->getScalar( "dout" );; + double inletA=1.f; + double inletB=0.f; + double outletA=0.f; + double outletB=1.f; + double flux = 10.f; - int RESTART_INTERVAL=20000; - int BLOB_ANALYSIS_INTERVAL=1000; - int timestep = 6; + // Read domain values + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + int BoundaryCondition = domain_db->getScalar( "BC" ); + int Nx = size[0]; + int Ny = size[1]; + int Nz = size[2]; + int Lx = L[0]; + int Ly = L[1]; + int Lz = L[2]; + int nprocx = nproc[0]; + int nprocy = nproc[1]; + int nprocz = nproc[2]; - if (rank==0){ - //............................................................. - // READ SIMULATION PARMAETERS FROM INPUT FILE - //............................................................. - ifstream input("Color.in"); - if (input.is_open()){ - // Line 1: model parameters (tau, alpha, beta, das, dbs) - input >> tauA; // Viscosity non-wetting - input >> tauB; // Viscosity wetting - input >> rhoA; // density non-wetting - input >> rhoB; // density wetting - input >> alpha; // Surface Tension parameter - input >> beta; // Width of the interface - // Line 2: External force components (Fx,Fy, Fz) - input >> Fx; - input >> Fy; - input >> Fz; - // Line 4: Pressure Boundary conditions - input >> InitialCondition; - input >> BoundaryCondition; - input >> din; - input >> dout; - // Line 5: time-stepping criteria - input >> timestepMax; // max no. of timesteps - input >> RESTART_INTERVAL; // restart interval - input >> tol; // error tolerance - // Line 6: Analysis options - input >> BLOB_ANALYSIS_INTERVAL; // interval to analyze blob states - //............................................................. - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Color.in is missing)! Default parameters will be used. \n"); - tauA = tauB = 1.0; - rhoA = rhoB = 1.0; - alpha=0.005; - beta= 0.9; - Fx = Fy = Fz = 0.0; - InitialCondition=0; - BoundaryCondition=0; - din=dout=1.0; - timestepMax=0; - } + int timestep = 6; - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (input.is_open()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Domain.in is missing)! Default parameters will be used. \n"); - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=10; - Lx=Ly=Lz=1.0; - } - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm); - MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm); - MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm); - MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. + flux = 0.f; + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details - flux = 0.f; - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details + // Get the rank info + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - // Get the rank info - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + MPI_Barrier(comm); - MPI_Barrier(comm); + if (nprocs != nprocx*nprocy*nprocz){ + printf("nprocx = %i \n",nprocx); + printf("nprocy = %i \n",nprocy); + printf("nprocz = %i \n",nprocz); + INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); + } - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + if (rank==0){ + printf("********************************************************\n"); + printf("tau (non-wetting) = %f \n", tauA); + printf("tau (wetting) = %f \n", tauB); + printf("density (non-wetting) = %f \n", rhoA); + printf("density (wetting) = %f \n", rhoB); + printf("alpha = %f \n", alpha); + printf("beta = %f \n", beta); + printf("gamma_{wn} = %f \n", 5.796*alpha); + printf("Force(x) = %f \n", Fx); + printf("Force(y) = %f \n", Fy); + printf("Force(z) = %f \n", Fz); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); + if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); + if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); + if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); + if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); + if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n"); + if (!Restart) printf("Initial conditions assigned from phase ID file \n"); + if (Restart) printf("Initial conditions assigned from restart file \n"); + printf("********************************************************\n"); + } - if (rank==0){ - printf("********************************************************\n"); - printf("tau (non-wetting) = %f \n", tauA); - printf("tau (wetting) = %f \n", tauB); - printf("density (non-wetting) = %f \n", rhoA); - printf("density (wetting) = %f \n", rhoB); - printf("alpha = %f \n", alpha); - printf("beta = %f \n", beta); - printf("gamma_{wn} = %f \n", 5.796*alpha); - printf("Force(x) = %f \n", Fx); - printf("Force(y) = %f \n", Fy); - printf("Force(z) = %f \n", Fz); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); - if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); - if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); - if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); - if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); - if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n"); - if (InitialCondition==0) printf("Initial conditions assigned from phase ID file \n"); - if (InitialCondition==1) printf("Initial conditions assigned from restart file \n"); - printf("********************************************************\n"); - } + // Initialized domain and averaging framework for Two-Phase Flow + bool pBC; + if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4) + pBC=true; + else + pBC=false; - // Initialized domain and averaging framework for Two-Phase Flow - bool pBC,velBC; - if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4) - pBC=true; - else pBC=false; - if (BoundaryCondition==2) velBC=true; - else velBC=false; + // Full domain used for averaging (do not use mask for analysis) + Domain Dm(domain_db); + for (int i=0; i Averages( new TwoPhase(Dm) ); + // TwoPhase Averages(Dm); + Dm.CommInit(comm); - bool Restart; - if (InitialCondition==1) Restart=true; - else Restart=false; - NULL_USE(pBC); NULL_USE(velBC); + // Mask that excludes the solid phase + Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + MPI_Barrier(comm); - // Full domain used for averaging (do not use mask for analysis) - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - for (i=0; i Averages( new TwoPhase(Dm) ); - // TwoPhase Averages(Dm); - Dm.CommInit(comm); + Nx+=2; Ny+=2; Nz += 2; + int N = Nx*Ny*Nz; + //....................................................................... + if (rank == 0) printf("Read input media... \n"); + //....................................................................... - // Mask that excludes the solid phase - Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - MPI_Barrier(comm); + //....................................................................... + // Filenames used + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - Nx+=2; Ny+=2; Nz += 2; - int N = Nx*Ny*Nz; - //....................................................................... - if (rank == 0) printf("Read input media... \n"); - //....................................................................... + // printf("Local File Name = %s \n",LocalRankFilename); + // .......... READ THE INPUT FILE ....................................... + // char value; + char *id; + id = new char[N]; + double sum, sum_local; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + //........................................................................... + if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; - //....................................................................... - // Filenames used - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - char tmpstr[10]; - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + //....................................................................... + // Read the signed distance + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); + ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); + MPI_Barrier(comm); + if (rank == 0) cout << "Domain set." << endl; - // printf("Local File Name = %s \n",LocalRankFilename); - // .......... READ THE INPUT FILE ....................................... - // char value; - char *id; - id = new char[N]; - double sum, sum_local; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); - double porosity, pore_vol; - //........................................................................... - if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; + //....................................................................... + // Assign the phase ID field based on the signed distance + //....................................................................... + for (int k=0;kSDs(n) > 0.0){ + id[n] = 2; + } + // compute the porosity (actual interface location used) + if (Averages->SDs(n) > 0.0){ + sum++; + } + } + } + } - //....................................................................... - // Read the signed distance - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); - ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); - MPI_Barrier(comm); - if (rank == 0) cout << "Domain set." << endl; + if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); + sprintf(LocalRankFilename,"ID.%05i",rank); + size_t readID; + FILE *IDFILE = fopen(LocalRankFilename,"rb"); + if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: ID.xxxxx"); + readID=fread(id,1,N,IDFILE); + if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading ID (rank=%i) \n",rank); + fclose(IDFILE); + + // Read id from restart + if (Restart == true){ + if (rank==0){ + printf("Reading restart file! \n"); + ifstream restart("Restart.txt"); + if (restart.is_open()){ + restart >> timestep; + printf("Restarting from timestep =%i \n",timestep); + } + else{ + printf("WARNING:No Restart.txt file, setting timestep=0 \n"); + timestep=0; + } + } + MPI_Bcast(×tep,1,MPI_INT,0,comm); + FILE *RESTART = fopen(LocalRestartFile,"rb"); + if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); + readID=fread(id,1,N,RESTART); + if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading Restart (rank=%i) \n",rank); + fclose(RESTART); + /* + // Read in the restart file to CPU buffers + double *cDen = new double[2*Np]; + double *cfq = new double[19*Np]; + ReadCheckpoint(LocalRestartFile, cDen, cfq, Np); + // Copy the restart data to the GPU + ScaLBL_CopyToDevice(fq,cfq,19*Np*sizeof(double)); + ScaLBL_CopyToDevice(Den,cDen,2*Np*sizeof(double)); + ScaLBL_DeviceBarrier(); + delete [] cDen; + delete [] cfq; + */ + MPI_Barrier(comm); + } + + + //....................................................................... + // Compute the media porosity, assign phase labels and solid composition + //....................................................................... + sum_local=0.0; + int Np=0; // number of local pore nodes + //....................................................................... + for (int k=1;k 0){ + sum_local+=1.0; + Np++; + } + } + } + } + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); + double porosity = sum*iVol_global; + if (rank==0) printf("Media porosity = %f \n",porosity); + //......................................................... + // If external boundary conditions are applied remove solid + if (BoundaryCondition > 0 && Dm.kproc == 0){ + for (int k=0; k<3; k++){ + for (int j=0;jSDs(n) = max(Averages->SDs(n),1.0*(2.5-k)); + } + } + } + } + if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){ + for (int k=Nz-3; kSDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5)); + } + } + } + } + //......................................................... + // don't perform computations at the eight corners + id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + //......................................................... - //....................................................................... - // Assign the phase ID field based on the signed distance - //....................................................................... - for (k=0;kSDs(n) > 0.0){ - id[n] = 2; - } - // compute the porosity (actual interface location used) - if (Averages->SDs(n) > 0.0){ - sum++; - } - } - } - } + // Initialize communication structures in averaging domain + for (int i=0; i> timestep; - printf("Restarting from timestep =%i \n",timestep); - } - else{ - printf("WARNING:No Restart.txt file, setting timestep=0 \n"); - timestep=0; - } - } - MPI_Bcast(×tep,1,MPI_INT,0,comm); - FILE *RESTART = fopen(LocalRestartFile,"rb"); - if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); - readID=fread(id,1,N,RESTART); - if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading Restart (rank=%i) \n",rank); - fclose(RESTART); - /* - // Read in the restart file to CPU buffers - double *cDen = new double[2*Np]; - double *cfq = new double[19*Np]; - ReadCheckpoint(LocalRestartFile, cDen, cfq, Np); - // Copy the restart data to the GPU - ScaLBL_CopyToDevice(fq,cfq,19*Np*sizeof(double)); - ScaLBL_CopyToDevice(Den,cDen,2*Np*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] cDen; - delete [] cfq; - */ - MPI_Barrier(comm); - } - - - //....................................................................... - // Compute the media porosity, assign phase labels and solid composition - //....................................................................... - sum_local=0.0; - Np=0; // number of local pore nodes - //....................................................................... - for (k=1;k 0){ - sum_local+=1.0; - Np++; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = sum*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - //......................................................... - // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm.kproc == 0){ - for (k=0; k<3; k++){ - for (j=0;jSDs(n) = max(Averages->SDs(n),1.0*(2.5-k)); - } - } - } - } - if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){ - for (k=Nz-3; kSDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5)); - } - } - } - } - //......................................................... - // don't perform computations at the eight corners - id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - //......................................................... + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); - // Initialize communication structures in averaging domain - for (i=0; iUpdateMeshValues(); // this computes the gradient of distance field (among other things) + // Create the distance stencil + // Compute solid forces based on mean field approximation + double *Dst; + Dst = new double [5*5*5]; + for (int kk=0; kk<5; kk++){ + for (int jj=0; jj<5; jj++){ + for (int ii=0; ii<5; ii++){ + int index = kk*25+jj*5+ii; + Dst[index] = sqrt(double(ii-2)*double(ii-2) + double(jj-2)*double(jj-2)+ double(kk-2)*double(kk-2)); + } + } + } + for (int k=1; kUpdateMeshValues(); // this computes the gradient of distance field (among other things) - // Create the distance stencil - // Compute solid forces based on mean field approximation - double *Dst; - Dst = new double [5*5*5]; - for (int kk=0; kk<5; kk++){ - for (int jj=0; jj<5; jj++){ - for (int ii=0; ii<5; ii++){ - int index = kk*25+jj*5+ii; - Dst[index] = sqrt(double(ii-2)*double(ii-2) + double(jj-2)*double(jj-2)+ double(kk-2)*double(kk-2)); - } - } - } - for (k=1; k 0)){ + double vec_x = double(ii-2); + double vec_y = double(jj-2); + double vec_z = double(kk-2); + + double ALPHA=PhaseLabel[nn]; + double GAMMA=-2.f; + if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance + phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; + phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; + phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; + } + } + } + } + Tmp[idx] = phi_x; + Tmp[idx+Np] = phi_y; + Tmp[idx+2*Np] = phi_z; + + /* double d = Averages->SDs(n); + double dx = Averages->SDs_x(n); + double dy = Averages->SDs_y(n); + double dz = Averages->SDs_z(n); + double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d)); + + Tmp[idx] = value*dx; + Tmp[idx+Np] = value*dy; + Tmp[idx+2*Np] = value*dz; + */ + } + } + } + } + ScaLBL_CopyToDevice(SolidPotential, Tmp, 3*sizeof(double)*Np); + ScaLBL_DeviceBarrier(); + delete [] Tmp; + delete [] Dst; + + DoubleArray Psx(Nx,Ny,Nz); + DoubleArray Psy(Nx,Ny,Nz); + DoubleArray Psz(Nx,Ny,Nz); + DoubleArray Psnorm(Nx,Ny,Nz); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[0],Psx); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[Np],Psy); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[2*Np],Psz); + for (int n=0; n 0)){ - double vec_x = double(ii-2); - double vec_y = double(jj-2); - double vec_z = double(kk-2); - - double ALPHA=PhaseLabel[nn]; - double GAMMA=-2.f; - if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance - phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; - phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; - phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; - } - } - } - } - Tmp[idx] = phi_x; - Tmp[idx+Np] = phi_y; - Tmp[idx+2*Np] = phi_z; - - /* double d = Averages->SDs(n); - double dx = Averages->SDs_x(n); - double dy = Averages->SDs_y(n); - double dz = Averages->SDs_z(n); - double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d)); - - Tmp[idx] = value*dx; - Tmp[idx+Np] = value*dy; - Tmp[idx+2*Np] = value*dz; - */ - } - } - } - } - ScaLBL_CopyToDevice(SolidPotential, Tmp, 3*sizeof(double)*Np); - ScaLBL_DeviceBarrier(); - delete [] Tmp; - delete [] Dst; - - DoubleArray Psx(Nx,Ny,Nz); - DoubleArray Psy(Nx,Ny,Nz); - DoubleArray Psz(Nx,Ny,Nz); - DoubleArray Psnorm(Nx,Ny,Nz); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[0],Psx); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[Np],Psy); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[2*Np],Psz); - for (n=0; nUpdateSolid(); + //....................................................................... + //ScaLBL_D3Q19_Pressure(fq,Pressure,Np); + //ScaLBL_D3Q19_Momentum(fq,Velocity,Np); + //........................................................................... + // Copy the phase indicator field for the earlier timestep + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages->Phase_tplus.data(),Phi,Np*sizeof(double)); + //........................................................................... + // Copy the data for for the analysis timestep + //........................................................................... + // Copy the phase from the GPU -> CPU + //........................................................................... + ScaLBL_DeviceBarrier(); + ScaLBL_CopyToHost(Averages->Phase.data(),Phi,Np*sizeof(double)); + ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press); + ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x); + ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y); + ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); + //........................................................................... - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("No. of timesteps: %i \n", timestepMax); - //....................................................................... - // Once phase has been initialized, map solid to account for 'smeared' interface - //for (i=0; iUpdateSolid(); - //....................................................................... - //ScaLBL_D3Q19_Pressure(fq,Pressure,Np); - //ScaLBL_D3Q19_Momentum(fq,Velocity,Np); - //........................................................................... - // Copy the phase indicator field for the earlier timestep - ScaLBL_DeviceBarrier(); - ScaLBL_CopyToHost(Averages->Phase_tplus.data(),Phi,Np*sizeof(double)); - //........................................................................... - // Copy the data for for the analysis timestep - //........................................................................... - // Copy the phase from the GPU -> CPU - //........................................................................... - ScaLBL_DeviceBarrier(); - ScaLBL_CopyToHost(Averages->Phase.data(),Phi,Np*sizeof(double)); - ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press); - ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x); - ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y); - ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); - //........................................................................... + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + starttime = MPI_Wtime(); + //......................................... - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("No. of timesteps: %i \n", timestepMax); + //************ MAIN ITERATION LOOP ***************************************/ + PROFILE_START("Loop"); + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); + while (timestep < timestepMax ) { + //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } + PROFILE_START("Update"); + // *************ODD TIMESTEP************* + timestep++; + // Compute the Phase indicator field + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); + + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set BCs + if (BoundaryCondition > 0){ + ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + if (BoundaryCondition == 3){ + ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + if (BoundaryCondition == 4){ + din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... + // *************EVEN TIMESTEP************* + timestep++; + // Compute the Phase indicator field + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); - err = 1.0; - double sat_w_previous = 1.01; // slightly impossible value! - if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol); + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition > 0){ + ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + if (BoundaryCondition == 3){ + ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************ + MPI_Barrier(comm); + PROFILE_STOP("Update"); - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - std::shared_ptr analysis_db; - runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, err, Map ); - while (timestep < timestepMax && err > tol ) { - //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } - PROFILE_START("Update"); - // *************ODD TIMESTEP************* - timestep++; - // Compute the Phase indicator field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Set BCs - if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + // Run the analysis + analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); - // *************EVEN TIMESTEP************* - timestep++; - // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Set boundary conditions - if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - else if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - //************************************************************************ - MPI_Barrier(comm); - PROFILE_STOP("Update"); - - // Run the analysis - // analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); - - } + } analysis.finish(); - PROFILE_STOP("Loop"); - PROFILE_SAVE("lbpm_color_simulator",1); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Np)/cputime/1000000; + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_color_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); - // ************************************************************************ - - // Copy back final phase indicator field and convert to regular layout - DoubleArray PhaseField(Nx,Ny,Nz); + // ************************************************************************ + + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); ScaLBL_Comm.RegularLayout(Map,Phi,PhaseField); - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PhaseField.data(),8,N,OUTFILE); - fclose(OUTFILE); - - PROFILE_STOP("Main"); - PROFILE_SAVE("lbpm_color_simulator",1); - // **************************************************** - MPI_Barrier(comm); - } // Limit scope so variables that contain communicators will free before MPI_Finialize - MPI_Comm_free(&comm); - MPI_Finalize(); + FILE *OUTFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); + + PROFILE_STOP("Main"); + PROFILE_SAVE("lbpm_color_simulator",1); + // **************************************************** + MPI_Barrier(comm); + } // Limit scope so variables that contain communicators will free before MPI_Finialize + MPI_Comm_free(&comm); + MPI_Finalize(); } From 1ba427456cc34b8f594470ad7a837cbc36934a32 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 14:34:32 -0400 Subject: [PATCH 018/124] refactor TestPressVel --- tests/TestPressVel.cpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index bdd67aec..7e352c99 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -155,25 +155,20 @@ int main(int argc, char **argv) printf("Reduced domain size = %i \n",Np); } - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - - int neighborSize=18*Np*sizeof(int); + if (rank==0) printf ("Set up the neighborlist \n"); + int Npad=Np+32; + int neighborSize=18*Npad*sizeof(int); int *neighborList; IntArray Map(Nx,Ny,Nz); - - neighborList= new int[18*Np]; - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); - // ScaLBL_Comm.MemoryDenseLayoutFull(Map,neighborList,Dm.id,Np); // this was how I tested for correctness - + neighborList= new int[18*Npad]; + Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); MPI_Barrier(comm); //......................device distributions................................. + if (rank==0) printf ("Allocating distributions \n"); int dist_mem_size = Np*sizeof(double); - int *NeighborList; - // double *f_even,*f_odd; double * dist; double * Velocity; //........................................................................... @@ -187,7 +182,6 @@ int main(int argc, char **argv) * AA Algorithm begins here * */ - //ScaLBL_D3Q19_Init(ID, dist, &dist[10*Np], Np, 1, 1); double *DIST; DIST = new double [19*Np]; double VALUE=0.1; @@ -215,7 +209,6 @@ int main(int argc, char **argv) DIST[17*Np + n] = 1.0; DIST[18*Np + n] = 1.0; } - ScaLBL_CopyToDevice(dist, DIST, 19*Np*sizeof(double)); From 2288dd7bd98db3debc6114b1eb244970ec70e860 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 14:40:38 -0400 Subject: [PATCH 019/124] refactor TestPressVel --- tests/TestPressVel.cpp | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index 7e352c99..db19ba2c 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -10,21 +10,6 @@ #include "common/MPI_Helpers.h" -std::shared_ptr loadInputs( int nprocs ) -{ - auto db = std::make_shared( "Domain.in" ); - const int dim = 50; - db->putScalar( "BC", 0 ); - if ( nprocs == 1 ){ - db->putVector( "nproc", { 1, 1, 1 } ); - db->putVector( "n", { 4, 4, 4 } ); - db->putScalar( "nspheres", 0 ); - db->putVector( "L", { 1, 1, 1 } ); - } - return db; -} - - //*************************************************************************************** int main(int argc, char **argv) { @@ -59,14 +44,15 @@ int main(int argc, char **argv) Fz = 1.0e-4; // Load inputs - auto db = loadInputs( nprocs ); - int Nx = db->getVector( "n" )[0]; - int Ny = db->getVector( "n" )[1]; - int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; - + if (rank==0) printf("Loading input database \n"); + auto db = std::make_shared(FILENAME); + auto domain_db= db-> getDatabase("Domain"); + int Nx = domain_db->getVector( "n" )[0]; + int Ny = domain_db->getVector( "n" )[1]; + int Nz = domain_db->getVector( "n" )[2]; + int nprocx = domain_db->getVector( "nproc" )[0]; + int nprocy = domain_db->getVector( "nproc" )[1]; + int nprocz = domain_db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); @@ -91,7 +77,6 @@ int main(int argc, char **argv) Domain Dm(db); - Nx += 2; Ny += 2; Nz += 2; @@ -104,12 +89,6 @@ int main(int argc, char **argv) sprintf(LocalRankString,"%05d",rank); char LocalRankFilename[40]; sprintf(LocalRankFilename,"ID.%05i",rank); - /* - FILE *IDFILE = fopen(LocalRankFilename,"rb"); - if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - fread(Dm.id,1,N,IDFILE); - fclose(IDFILE); - */ Dm.CommInit(comm); @@ -211,7 +190,6 @@ int main(int argc, char **argv) } ScaLBL_CopyToDevice(dist, DIST, 19*Np*sizeof(double)); - double *Vz; Vz= new double [Np]; size_t SIZE=Np*sizeof(double); From 820a173a48bd6483bcaa384698d82f37cfcd3dca Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 15:00:26 -0400 Subject: [PATCH 020/124] refatoring tests --- tests/CMakeLists.txt | 6 +++--- tests/TestForceMoments.cpp | 24 +++++++++++------------- tests/TestInterfaceSpeed.cpp | 21 ++++----------------- tests/TestPoiseuille.cpp | 3 +-- tests/TestPressVel.cpp | 4 ++-- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2121a88b..e43d7ee1 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,13 +46,13 @@ ADD_LBPM_TEST( TestColorGrad ) ADD_LBPM_TEST( TestBubbleDFH ) ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestColorMassBounceback ) -ADD_LBPM_TEST( TestPressVel ) +ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) -ADD_LBPM_TEST( TestForceMoments ) +ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db) ADD_LBPM_TEST( TestForceD3Q19 ) ADD_LBPM_TEST( TestMomentsD3Q19 ) -ADD_LBPM_TEST( TestInterfaceSpeed ) +ADD_LBPM_TEST( TestInterfaceSpeed ../example/Piston/input.db) ADD_LBPM_TEST( TestSphereCurvature ) #ADD_LBPM_TEST_1_2_4( TestTwoPhase ) ADD_LBPM_TEST_1_2_4( TestBlobIdentify ) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index fe842dc7..ed917730 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -11,8 +11,6 @@ using namespace std; - - extern void PrintNeighborList(int * neighborList, int Np, int rank) { if (rank == 0) { int n; @@ -63,7 +61,6 @@ int main(int argc, char **argv) // parallel domain size (# of sub-domains) int iproc,jproc,kproc; - if (rank == 0){ printf("********************************************************\n"); printf("Running Unit Test: TestForceMoments \n"); @@ -71,7 +68,6 @@ int main(int argc, char **argv) } // BGK Model parameters - string FILENAME; unsigned int nBlocks, nthreads; int timestepMax, interval; double tau,Fx,Fy,Fz,tol; @@ -88,16 +84,18 @@ int main(int argc, char **argv) Fx = Fy = 1.0; Fz = 1.0; - // Load inputs - auto db = loadInputs( nprocs ); - int Nx = db->getVector( "n" )[0]; - int Ny = db->getVector( "n" )[1]; - int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; - + string FILENAME = argv[1]; + // Load inputs + if (rank==0) printf("Loading input database \n"); + auto db = std::make_shared(FILENAME); + auto domain_db= db-> getDatabase("Domain"); + int Nx = domain_db->getVector( "n" )[0]; + int Ny = domain_db->getVector( "n" )[1]; + int Nz = domain_db->getVector( "n" )[2]; + int nprocx = domain_db->getVector( "nproc" )[0]; + int nprocy = domain_db->getVector( "nproc" )[1]; + int nprocz = domain_db->getVector( "nproc" )[2]; if (rank==0){ printf("********************************************************\n"); printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 53cc55ae..4bf47688 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -15,20 +15,6 @@ #define SPEED -1 #define PI 3.14159 - -std::shared_ptr loadInputs( int nprocs ) -{ - auto db = std::make_shared( ); - const int dim = 50; - db->putScalar( "BC", 0 ); - db->putVector( "nproc", { 1, 1, 1 } ); - db->putVector( "n", { N, N, N } ); - db->putScalar( "nspheres", 0 ); - db->putVector( "L", { 1, 1, 1 } ); - return db; -} - - int main (int argc, char *argv[]) { // Initialize MPI @@ -39,10 +25,11 @@ int main (int argc, char *argv[]) MPI_Comm_size(comm,&nprocs); int i,j,k,n; - // Load inputs - auto db = loadInputs( nprocs ); + // Load inputs + string FILENAME = argv[1]; + auto db = loadInputs( FILENAME ); - Domain Dm(db); + Domain Dm(db); for (i=0; i(FILENAME); @@ -159,7 +159,6 @@ int main(int argc, char **argv) printf("Reduced domain size = %i \n",Np); } - // LBM variables if (rank==0) printf ("Allocating distributions \n"); if (rank==0) printf ("Set up the neighborlist \n"); diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index db19ba2c..164e498f 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -29,9 +29,8 @@ int main(int argc, char **argv) printf("Running Unit Test: TestPressVel \n"); printf("********************************************************\n"); } - + // BGK Model parameters - string FILENAME; unsigned int nBlocks, nthreads; int timestepMax, interval; double tau,Fx,Fy,Fz,tol; @@ -44,6 +43,7 @@ int main(int argc, char **argv) Fz = 1.0e-4; // Load inputs + string FILENAME = argv[1]; if (rank==0) printf("Loading input database \n"); auto db = std::make_shared(FILENAME); auto domain_db= db-> getDatabase("Domain"); From 7528a3497205285e9356f26956570e83fe8ca170 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 15:06:28 -0400 Subject: [PATCH 021/124] refactr --- tests/TestInterfaceSpeed.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 4bf47688..92082ab6 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -27,8 +27,16 @@ int main (int argc, char *argv[]) int i,j,k,n; // Load inputs string FILENAME = argv[1]; - auto db = loadInputs( FILENAME ); - + // Load inputs + if (rank==0) printf("Loading input database \n"); + auto db = std::make_shared(FILENAME); + auto domain_db= db-> getDatabase("Domain"); + int Nx = domain_db->getVector( "n" )[0]; + int Ny = domain_db->getVector( "n" )[1]; + int Nz = domain_db->getVector( "n" )[2]; + int nprocx = domain_db->getVector( "nproc" )[0]; + int nprocy = domain_db->getVector( "nproc" )[1]; + int nprocz = domain_db->getVector( "nproc" )[2]; Domain Dm(db); for (i=0; i Date: Tue, 15 May 2018 15:14:13 -0400 Subject: [PATCH 022/124] Cleaning up Domain --- common/Domain.cpp | 660 ++-------------------------------------------- common/Domain.h | 190 +++++++++---- 2 files changed, 170 insertions(+), 680 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 20d8f1fd..6005dd41 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -32,13 +32,9 @@ static inline void fgetl( char * str, int num, FILE * stream ) ********************************************************/ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC): - Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), + Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), - Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), - rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), - rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), - rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), - rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), + Comm(MPI_COMM_NULL), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), @@ -74,13 +70,9 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, initialize( db ); } Domain::Domain( std::shared_ptr db ): - Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), nprocx(0), nprocy(0), nprocz(0), + Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), - Group(MPI_GROUP_NULL), Comm(MPI_COMM_NULL), - rank_x(0), rank_y(0), rank_z(0), rank_X(0), rank_Y(0), rank_Z(0), - rank_xy(0), rank_XY(0), rank_xY(0), rank_Xy(0), - rank_xz(0), rank_XZ(0), rank_xZ(0), rank_Xz(0), - rank_yz(0), rank_YZ(0), rank_yZ(0), rank_Yz(0), + Comm(MPI_COMM_NULL), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), @@ -115,6 +107,7 @@ void Domain::initialize( std::shared_ptr db ) auto nproc = d_db->getVector("nproc"); auto n = d_db->getVector("n"); auto L = d_db->getVector("L"); + //nspheres = d_db->getScalar("nspheres"); ASSERT( n.size() == 3u ); ASSERT( L.size() == 3u ); ASSERT( nproc.size() == 3u ); @@ -127,17 +120,17 @@ void Domain::initialize( std::shared_ptr db ) Nx = nx+2; Ny = ny+2; Nz = nz+2; - nprocx = nproc[0]; - nprocy = nproc[1]; - nprocz = nproc[2]; + // Initialize ranks + int myrank; + MPI_Comm_rank( MPI_COMM_WORLD, &myrank ); + rank_info = RankInfoStruct(myrank,nproc[0],nproc[1],nproc[2]); + // Fill remaining variables N = Nx*Ny*Nz; Volume = nx*ny*nx*nprocx*nprocy*nprocz*1.0; id = new char[N]; memset(id,0,N); BoundaryCondition = d_db->getScalar("BC"); - MPI_Comm_rank( MPI_COMM_WORLD, &rank ); - rank_info=RankInfoStruct(rank,nprocx,nprocy,nprocz); - int nprocs; + int nprocs; MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); } @@ -188,91 +181,27 @@ Domain::~Domain() // Free id delete [] id; // Free the communicator - if ( Group!=MPI_GROUP_NULL ) { - MPI_Group_free(&Group); + if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) { MPI_Comm_free(&Comm); } } -void Domain::InitializeRanks() +/******************************************************** + * Initialize communication * + ********************************************************/ +void Domain::CommInit(MPI_Comm Communicator) { - int i,j,k; - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; - - // set up the neighbor ranks - i = iproc; - j = jproc; - k = kproc; - - rank_X = getRankForBlock(i+1,j,k); - rank_x = getRankForBlock(i-1,j,k); - rank_Y = getRankForBlock(i,j+1,k); - rank_y = getRankForBlock(i,j-1,k); - rank_Z = getRankForBlock(i,j,k+1); - rank_z = getRankForBlock(i,j,k-1); - rank_XY = getRankForBlock(i+1,j+1,k); - rank_xy = getRankForBlock(i-1,j-1,k); - rank_Xy = getRankForBlock(i+1,j-1,k); - rank_xY = getRankForBlock(i-1,j+1,k); - rank_XZ = getRankForBlock(i+1,j,k+1); - rank_xz = getRankForBlock(i-1,j,k-1); - rank_Xz = getRankForBlock(i+1,j,k-1); - rank_xZ = getRankForBlock(i-1,j,k+1); - rank_YZ = getRankForBlock(i,j+1,k+1); - rank_yz = getRankForBlock(i,j-1,k-1); - rank_Yz = getRankForBlock(i,j+1,k-1); - rank_yZ = getRankForBlock(i,j-1,k+1); -} - - -void Domain::CommInit(MPI_Comm Communicator){ int i,j,k,n; int sendtag = 21; int recvtag = 21; - //...................................................................................... - //Get the ranks of each process and it's neighbors - // map the rank to the block index - //iproc = rank%nprocx; - //jproc = (rank/nprocx)%nprocy; - //kproc = rank/(nprocx*nprocy); + MPI_Comm_dup(Communicator,&Comm); + + // set up the neighbor ranks + rank_info = RankInfoStruct( rank, nprocx, nprocy, nprocz ); MPI_Barrier(Communicator); - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; - - // set up the neighbor ranks - i = iproc; - j = jproc; - k = kproc; - - rank_X = getRankForBlock(i+1,j,k); - rank_x = getRankForBlock(i-1,j,k); - rank_Y = getRankForBlock(i,j+1,k); - rank_y = getRankForBlock(i,j-1,k); - rank_Z = getRankForBlock(i,j,k+1); - rank_z = getRankForBlock(i,j,k-1); - rank_XY = getRankForBlock(i+1,j+1,k); - rank_xy = getRankForBlock(i-1,j-1,k); - rank_Xy = getRankForBlock(i+1,j-1,k); - rank_xY = getRankForBlock(i-1,j+1,k); - rank_XZ = getRankForBlock(i+1,j,k+1); - rank_xz = getRankForBlock(i-1,j,k-1); - rank_Xz = getRankForBlock(i+1,j,k-1); - rank_xZ = getRankForBlock(i-1,j,k+1); - rank_YZ = getRankForBlock(i,j+1,k+1); - rank_yz = getRankForBlock(i,j-1,k-1); - rank_Yz = getRankForBlock(i,j+1,k-1); - rank_yZ = getRankForBlock(i,j-1,k+1); - //...................................................................................... - - //MPI_Comm_group(Communicator,&Group); - //MPI_Comm_create(Communicator,Group,&Comm); - MPI_Comm_dup(MPI_COMM_WORLD,&Comm); //...................................................................................... MPI_Request req1[18], req2[18]; @@ -572,6 +501,9 @@ void Domain::CommInit(MPI_Comm Communicator){ } +/******************************************************** + * AssignComponentLabels * + ********************************************************/ void Domain::AssignComponentLabels(double *phase) { int NLABELS=0; @@ -660,543 +592,6 @@ void Domain::AssignComponentLabels(double *phase) } -void Domain::TestCommInit(MPI_Comm Communicator){ - int i,j,k,n; - int sendtag = 21; - int recvtag = 21; - - //...................................................................................... - //Get the ranks of each process and it's neighbors - // map the rank to the block index - iproc = rank%nprocx; - jproc = (rank/nprocx)%nprocy; - kproc = rank/(nprocx*nprocy); - // set up the neighbor ranks - i = iproc; - j = jproc; - k = kproc; - - MPI_Barrier(MPI_COMM_WORLD); - kproc = rank/(nprocx*nprocy); - jproc = (rank-nprocx*nprocy*kproc)/nprocx; - iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; - - if (rank == 0) { - printf("* In Domain::CommInit...\n"); - printf("* i,j,k proc=%d %d %d \n",i,j,k); - } - MPI_Barrier(MPI_COMM_WORLD); - if (rank == 1){ - printf("* i,j,k proc=%d %d %d \n",i,j,k); - printf("\n\n"); - } - - - - if(rank == 0) { printf("* Setting up ranks for each processor...\n"); - } - - MPI_Barrier(MPI_COMM_WORLD); - - rank_X = getRankForBlock(i+1,j,k); - rank_x = getRankForBlock(i-1,j,k); - - - if (rank ==0 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x); - if (rank ==1 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x); - if (rank ==2 ) printf("rank = %d: rank_X = %d, rank_x = %d \n", rank, rank_X, rank_x); - - rank_Y = getRankForBlock(i,j+1,k); - rank_y = getRankForBlock(i,j-1,k); - rank_Z = getRankForBlock(i,j,k+1); - rank_z = getRankForBlock(i,j,k-1); - rank_XY = getRankForBlock(i+1,j+1,k); - rank_xy = getRankForBlock(i-1,j-1,k); - rank_Xy = getRankForBlock(i+1,j-1,k); - rank_xY = getRankForBlock(i-1,j+1,k); - rank_XZ = getRankForBlock(i+1,j,k+1); - rank_xz = getRankForBlock(i-1,j,k-1); - rank_Xz = getRankForBlock(i+1,j,k-1); - rank_xZ = getRankForBlock(i-1,j,k+1); - rank_YZ = getRankForBlock(i,j+1,k+1); - rank_yz = getRankForBlock(i,j-1,k-1); - rank_Yz = getRankForBlock(i,j+1,k-1); - rank_yZ = getRankForBlock(i,j-1,k+1); - //...................................................................................... - - MPI_Comm_group(Communicator,&Group); - MPI_Comm_create(Communicator,Group,&Comm); - - //...................................................................................... - MPI_Request req1[18], req2[18]; - MPI_Status stat1[18],stat2[18]; - //...................................................................................... - - for (k=1; k> Isending and Ireceiving count data across processors...\n"); - - - - MPI_Barrier(MPI_COMM_WORLD); - - MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]); - MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]); - MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]); - MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]); - - - - - MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]); - MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]); - MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]); - MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]); - MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]); - MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]); - MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]); - MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]); - MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]); - MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]); - MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]); - MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]); - MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]); - MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]); - MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]); - MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]); - MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]); - MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]); - MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]); - MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]); - MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]); - MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]); - MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]); - MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]); - MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]); - MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]); - MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]); - MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]); - MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]); - MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]); - MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]); - MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]); - MPI_Waitall(18,req1,stat1); - MPI_Waitall(18,req2,stat2); - MPI_Barrier(Communicator); - //...................................................................................... - - if (rank == 0) printf("* recvList_# has been allocated through construction of Dm but sizes not determined. Creating arrays with size recvCount_#... *\n"); - - MPI_Barrier(MPI_COMM_WORLD); - - // recv buffers - recvList_x = new int [recvCount_x]; - recvList_y = new int [recvCount_y]; - recvList_z = new int [recvCount_z]; - recvList_X = new int [recvCount_X]; - recvList_Y = new int [recvCount_Y]; - recvList_Z = new int [recvCount_Z]; - recvList_xy = new int [recvCount_xy]; - recvList_yz = new int [recvCount_yz]; - recvList_xz = new int [recvCount_xz]; - recvList_Xy = new int [recvCount_Xy]; - recvList_Yz = new int [recvCount_Yz]; - recvList_xZ = new int [recvCount_xZ]; - recvList_xY = new int [recvCount_xY]; - recvList_yZ = new int [recvCount_yZ]; - recvList_Xz = new int [recvCount_Xz]; - recvList_XY = new int [recvCount_XY]; - recvList_YZ = new int [recvCount_YZ]; - recvList_XZ = new int [recvCount_XZ]; - //...................................................................................... - - if (rank == 0) printf("* >> Isending and Ireceiving list data (of size sendCount_# and recvCount_#) across processors...\n"); - - MPI_Barrier(MPI_COMM_WORLD); - - MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]); - MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]); - MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]); - MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]); - MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]); - MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]); - MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]); - MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]); - MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]); - MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]); - MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]); - MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]); - MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]); - MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]); - MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]); - MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]); - MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]); - MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]); - MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]); - MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]); - MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]); - MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]); - MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]); - MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]); - MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]); - MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]); - MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]); - MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]); - MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]); - MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]); - MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]); - MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]); - MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]); - MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]); - MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]); - MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]); - MPI_Waitall(18,req1,stat1); - MPI_Waitall(18,req2,stat2); - - - if (rank == 0) { - - printf("* recvList_x: %d %d %d %d \n",recvList_x[0],recvList_x[1],recvList_x[2],recvList_x[3]); - - printf("* recvList_X: %d %d %d %d \n",recvList_X[0],recvList_X[1],recvList_X[2],recvList_X[3]); - - printf("\n"); - } - - MPI_Barrier(MPI_COMM_WORLD); - - if (rank == 1) { - - printf("* recvList_x: %d %d %d %d \n",recvList_x[0],recvList_x[1],recvList_x[2],recvList_x[3]); - - printf("* recvList_X: %d %d %d %d \n",recvList_X[0],recvList_X[1],recvList_X[2],recvList_X[3]); - - printf("\n\n"); - } - - - - //...................................................................................... - for (int idx=0; idx #include @@ -19,9 +17,47 @@ #include "common/Database.h" +class Domain; +template class PatchData; -//! Read the domain information file -std::shared_ptr read_domain( ); + +//! Class to hold information about a box +class Box { +public: + int ifirst[3]; + int ilast[3]; +}; + + +enum class DataLocation { CPU, DEVICE }; + + +//! Class to hold information about a patch +class Patch { +public: + + //! Empty constructor + Patch() = delete; + + //! Copy constructor + Patch( const Patch& ) = delete; + + //! Assignment operator + Patch& operator=( const Patch& ) = delete; + + //! Return the box for the patch + inline const Box& getBox() const { return d_box; } + + //! Create patch data + template + std::shared_ptr> createPatchData( DataLocation location ) const; + +private: + Box d_box; + int d_owner; + Domain *d_domain; + +}; //! Class to hold domain info @@ -49,32 +85,67 @@ public: //! Get the database inline std::shared_ptr getDatabase() const { return d_db; } + //! Get the domain box + inline const Box& getBox() const { return d_box; } + + //! Get local patch + inline const Patch& getLocalPatch() const { return *d_localPatch; } + + //! Get all patches + inline const std::vector& getAllPatch() const { return d_patches; } + + private: void initialize( std::shared_ptr db ); std::shared_ptr d_db; + Box d_box; + Patch *d_localPatch; + std::vector d_patches; -public: - // Basic domain information - int Nx,Ny,Nz,N; - int iproc,jproc,kproc; - int nprocx,nprocy,nprocz; +public: // Public variables (need to create accessors instead) + double Lx,Ly,Lz,Volume; - int rank; - int BoundaryCondition; + int Nx,Ny,Nz,N; RankInfoStruct rank_info; - MPI_Group Group; // Group of processors associated with this domain + MPI_Comm Comm; // MPI Communicator for this domain + int BoundaryCondition; + + MPI_Group Group; // Group of processors associated with this domain + //********************************** // MPI ranks for all 18 neighbors //********************************** - int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; - int rank_xy,rank_XY,rank_xY,rank_Xy; - int rank_xz,rank_XZ,rank_xZ,rank_Xz; - int rank_yz,rank_YZ,rank_yZ,rank_Yz; + const int& iproc = rank_info.ix; + const int& jproc = rank_info.jy; + const int& kproc = rank_info.kz; + const int& nprocx = rank_info.nx; + const int& nprocy = rank_info.ny; + const int& nprocz = rank_info.nz; + const int& rank = rank_info.rank[1][1][1]; + const int& rank_X = rank_info.rank[2][1][1]; + const int& rank_x = rank_info.rank[0][1][1]; + const int& rank_Y = rank_info.rank[1][2][1]; + const int& rank_y = rank_info.rank[1][0][1]; + const int& rank_Z = rank_info.rank[1][1][2]; + const int& rank_z = rank_info.rank[1][1][0]; + const int& rank_XY = rank_info.rank[2][2][1]; + const int& rank_xy = rank_info.rank[0][0][1]; + const int& rank_Xy = rank_info.rank[2][0][1]; + const int& rank_xY = rank_info.rank[0][2][1]; + const int& rank_XZ = rank_info.rank[2][1][2]; + const int& rank_xz = rank_info.rank[0][1][0]; + const int& rank_Xz = rank_info.rank[2][1][0]; + const int& rank_xZ = rank_info.rank[0][1][2]; + const int& rank_YZ = rank_info.rank[1][2][2]; + const int& rank_yz = rank_info.rank[1][0][0]; + const int& rank_Yz = rank_info.rank[1][2][0]; + const int& rank_yZ = rank_info.rank[1][0][2]; + //********************************** //...................................................................................... // Get the actual D3Q19 communication counts (based on location of solid phase) @@ -88,10 +159,6 @@ public: int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ; int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ; //...................................................................................... - int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; - int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; - int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; - //...................................................................................... int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; @@ -99,53 +166,78 @@ public: int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ; - //...................................................................................... - int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; - int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; - int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; //...................................................................................... - double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; - double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; - double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; - double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; - double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; - double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; - // Solid indicator function char *id; - void InitializeRanks(); - void CommInit(MPI_Comm comm); void CommunicateMeshHalo(DoubleArray &Mesh); void AssignComponentLabels(double *phase); - + void CommInit(MPI_Comm comm); void TestCommInit(MPI_Comm comm); //void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np); private: - inline int getRankForBlock( int i, int j, int k ) - { - int i2 = (i+nprocx)%nprocx; - int j2 = (j+nprocy)%nprocy; - int k2 = (k+nprocz)%nprocz; - return i2 + j2*nprocx + k2*nprocx*nprocy; - } + int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; + int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; + int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; + //...................................................................................... + int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; + int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; + int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; + //...................................................................................... + double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; + double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; + double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; + double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; + double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; + double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; }; -double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps); + +// Class to hold data on a patch +template +class PatchData { +public: + + //! Get the raw data pointer + TYPE* data() { return d_data; } + + //! Get the raw data pointer + const TYPE* data() const { return d_data; } + + //! Get the patch + const Patch& getPatch() const { return *d_patch; } + + //! Start communication + void beginCommunication(); + + //! End communication + void endCommunication(); + + //! Access ghost values + TYPE operator()( int, int, int ) const; + + //! Copy data from another PatchData + void copy( const PatchData& rhs ); + +private: + DataLocation d_location; + const Patch *d_patch; + TYPE *d_data; + TYPE *d_gcw; +}; + + + void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad); -void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, - double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, - int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); - void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, - double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, - int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); void WriteLocalSolidID(char *FILENAME, char *ID, int N); @@ -158,4 +250,6 @@ void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np); void ReadBinaryFile(char *FILENAME, double *Data, int N); + + #endif From 4b22f0042f7b3eebc3db8581131acdbb95813f72 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Tue, 15 May 2018 15:32:07 -0400 Subject: [PATCH 023/124] Fixing compile error from merge --- tests/TestInterfaceSpeed.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 92082ab6..d9eb1796 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -30,14 +30,8 @@ int main (int argc, char *argv[]) // Load inputs if (rank==0) printf("Loading input database \n"); auto db = std::make_shared(FILENAME); - auto domain_db= db-> getDatabase("Domain"); - int Nx = domain_db->getVector( "n" )[0]; - int Ny = domain_db->getVector( "n" )[1]; - int Nz = domain_db->getVector( "n" )[2]; - int nprocx = domain_db->getVector( "nproc" )[0]; - int nprocy = domain_db->getVector( "nproc" )[1]; - int nprocz = domain_db->getVector( "nproc" )[2]; - Domain Dm(db); + auto domain_db = db->getDatabase("Domain"); + Domain Dm(domain_db); for (i=0; i Date: Tue, 15 May 2018 15:54:55 -0400 Subject: [PATCH 024/124] refactor --- example/Piston/input.db | 2 +- tests/TestInterfaceSpeed.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/example/Piston/input.db b/example/Piston/input.db index 3e8edefa..78c363e3 100644 --- a/example/Piston/input.db +++ b/example/Piston/input.db @@ -18,7 +18,7 @@ Color { Domain { nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) - n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) + n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz) n_spheres = 1 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) BC = 0 // Boundary condition type diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 92082ab6..5bc91231 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -46,10 +46,6 @@ int main (int argc, char *argv[]) TwoPhase Averages(Dm); int timestep=0; - int Nx = Dm.Nx; - int Ny = Dm.Ny; - int Nz = Dm.Nz; - double Cx,Cy,Cz; double dist1,dist2; From 740ec1e4320c2da19244b7a3ca65a0d395a9f034 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 16:03:29 -0400 Subject: [PATCH 025/124] db refactr --- tests/TestForceMoments.cpp | 3 +-- tests/TestInterfaceSpeed.cpp | 7 +++---- tests/TestPoiseuille.cpp | 1 - tests/TestPressVel.cpp | 4 ++-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index ed917730..8b847a9a 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -88,8 +88,7 @@ int main(int argc, char **argv) string FILENAME = argv[1]; // Load inputs if (rank==0) printf("Loading input database \n"); - auto db = std::make_shared(FILENAME); - auto domain_db= db-> getDatabase("Domain"); + auto domain_db = std::make_shared(FILENAME); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 92082ab6..b301be5d 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -25,19 +25,18 @@ int main (int argc, char *argv[]) MPI_Comm_size(comm,&nprocs); int i,j,k,n; - // Load inputs + + // Load inputs string FILENAME = argv[1]; // Load inputs if (rank==0) printf("Loading input database \n"); - auto db = std::make_shared(FILENAME); - auto domain_db= db-> getDatabase("Domain"); + auto domain_db = std::make_shared(FILENAME); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; int nprocx = domain_db->getVector( "nproc" )[0]; int nprocy = domain_db->getVector( "nproc" )[1]; int nprocz = domain_db->getVector( "nproc" )[2]; - Domain Dm(db); for (i=0; i(FILENAME); - auto domain_db= db-> getDatabase("Domain"); + auto domain_db = std::make_shared(FILENAME); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; From 8ae92c30c23cf01cda10eadfab283b1a68f19060 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 16:11:35 -0400 Subject: [PATCH 026/124] db refactor --- tests/TestForceMoments.cpp | 2 +- tests/TestInterfaceSpeed.cpp | 2 ++ tests/TestPressVel.cpp | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 8b847a9a..050a455c 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -117,7 +117,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(db); + Domain Dm(domain_db); Nx += 2; Ny += 2; diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 0995be90..65ad83f8 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -38,6 +38,8 @@ int main (int argc, char *argv[]) int nprocy = domain_db->getVector( "nproc" )[1]; int nprocz = domain_db->getVector( "nproc" )[2]; + Domain Dm(domain_db); + for (i=0; i Date: Tue, 15 May 2018 16:19:11 -0400 Subject: [PATCH 027/124] db refactor --- tests/TestForceMoments.cpp | 3 ++- tests/TestInterfaceSpeed.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 050a455c..5e2666a2 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -88,7 +88,8 @@ int main(int argc, char **argv) string FILENAME = argv[1]; // Load inputs if (rank==0) printf("Loading input database \n"); - auto domain_db = std::make_shared(FILENAME); + auto db = std::make_shared( FILENAME ); + auto domain_db = db->getDatabase( "Domain" ); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 65ad83f8..3432fc26 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -30,7 +30,8 @@ int main (int argc, char *argv[]) string FILENAME = argv[1]; // Load inputs if (rank==0) printf("Loading input database \n"); - auto domain_db = std::make_shared(FILENAME); + auto db = std::make_shared( FILENAME ); + auto domain_db = db->getDatabase( "Domain" ); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; From b3b7478ec20dacf8e0138287317ce25528e5680f Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 16:21:38 -0400 Subject: [PATCH 028/124] db refactor --- tests/TestPressVel.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index ccafa0cf..11b83f65 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -46,8 +46,9 @@ int main(int argc, char **argv) string FILENAME = argv[1]; // Load inputs if (rank==0) printf("Loading input database \n"); - auto domain_db = std::make_shared(FILENAME); - int Nx = domain_db->getVector( "n" )[0]; + auto db = std::make_shared( FILENAME ); + auto domain_db = db->getDatabase( "Domain" ); + int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; int nprocx = domain_db->getVector( "nproc" )[0]; From 8b00ad62999a69a9337cbf033ad37f132a13bd5c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 16:26:08 -0400 Subject: [PATCH 029/124] db refactor --- tests/TestForceMoments.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 5e2666a2..02bd3f65 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -234,8 +234,8 @@ int main(int argc, char **argv) if (rank==0) printf("No. of timesteps for timing: %i \n", timesteps); while (timestep < 2) { - - ScaLBL_D3Q19_AAeven_MRT(dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); +/* + ScaLBL_D3Q19_AAeven_MRT(dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); MPI_Barrier(comm); @@ -245,6 +245,21 @@ int main(int argc, char **argv) ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + */ + ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + + ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + //************************************************************************/ timestep++; From 9a22d71d0f24edad649fe861632c7d4ebbabdc61 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 15 May 2018 16:29:32 -0400 Subject: [PATCH 030/124] refactr db --- tests/TestForceMoments.cpp | 13 +------------ tests/TestPoiseuille.cpp | 6 ++++-- tests/TestPressVel.cpp | 4 +++- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 02bd3f65..1f9196ae 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -56,7 +56,7 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { // parallel domain size (# of sub-domains) int iproc,jproc,kproc; @@ -234,18 +234,7 @@ int main(int argc, char **argv) if (rank==0) printf("No. of timesteps for timing: %i \n", timesteps); while (timestep < 2) { -/* - ScaLBL_D3Q19_AAeven_MRT(dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, Np, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - */ ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 3598995f..1abdaf4d 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -25,7 +25,7 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { if (rank == 0){ printf("********************************************************\n"); @@ -48,7 +48,9 @@ int main(int argc, char **argv) Fx = 0; Fy = 0; Fz = 1e-3; //1.f; // 1e-3; - string FILENAME = argv[1]; + auto FILENAME = argv[1]; + auto db = std::make_shared( FILENAME ); + auto domain_db = db->getDatabase( "Domain" ); // Load inputs if (rank==0) printf("Loading input database \n"); auto db = std::make_shared(FILENAME); diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index 11b83f65..d2aebeba 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -22,7 +22,7 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { if (rank == 0){ printf("********************************************************\n"); @@ -207,4 +207,6 @@ int main(int argc, char **argv) MPI_Barrier(comm); MPI_Finalize(); // **************************************************** + return check; + } From 17df5958c1cc00377a38493530a4f85c7e95cf94 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 09:21:06 -0400 Subject: [PATCH 031/124] Removed dead code from Domain.cpp, refactored lbpm_sphere_pp --- common/Domain.cpp | 410 +-------------------------------------- common/Domain.h | 17 -- tests/lbpm_sphere_pp.cpp | 305 ++++++++++++++++++++++++----- 3 files changed, 258 insertions(+), 474 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 6005dd41..000e6a13 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -675,388 +675,6 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh) } - -/******************************************************** - * Misc * - ********************************************************/ -double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){ - /* - * This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - */ - - int Q=26; - int q,i,j,k; - double dt=0.1; - int in,jn,kn; - double Dqx,Dqy,Dqz,Dx,Dy,Dz,W; - double nx,ny,nz,Cqx,Cqy,Cqz,sign,norm; - double TotalVariation=0.0; - - const static int D3Q27[26][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, - {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, - {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1},{1,1,1},{-1,-1,-1},{1,1,-1},{-1,-1,1}, - {-1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1}}; - - double weights[26]; - // Compute the weights from the finite differences - for (q=0; q fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); - - int count = 0; - while (count < timesteps){ - - // Communicate the halo of values - fillData.fill(Distance); - - TotalVariation=0.0; - // Execute the next timestep - for (k=1;k 0.0 ){ - for (q=0; q<26; q++){ - Cqx = 1.0*D3Q27[q][0]; - Cqy = 1.0*D3Q27[q][1]; - Cqz = 1.0*D3Q27[q][2]; - // get the associated neighbor - in = i + D3Q27[q][0]; - jn = j + D3Q27[q][1]; - kn = k + D3Q27[q][2]; - - // make sure the neighbor is in the domain (periodic BC) - /* if (in < 0 ) in +=Nx; - * don't need this in parallel since MPI handles the halos - if (jn < 0 ) jn +=Ny; - if (kn < 0 ) kn +=Nz; - if (!(in < Nx) ) in -=Nx; - if (!(jn < Ny) ) jn -=Ny; - if (!(kn < Nz) ) kn -=Nz; - // symmetric boundary - if (in < 0 ) in = i; - if (jn < 0 ) jn = j; - if (kn < 0 ) kn = k; - if (!(in < Nx) ) in = i; - if (!(jn < Ny) ) jn = k; - if (!(kn < Nz) ) kn = k; - */ - - // Compute the gradient using upwind finite differences - Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx; - Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy; - Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz; - - // Only include upwind derivatives - if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){ - - Dx += Dqx; - Dy += Dqy; - Dz += Dqz; - W += weights[q]; - } - } - // Normalize by the weight to get the approximation to the gradient - if (fabs(W) > 0.0){ - Dx /= W; - Dy /= W; - Dz /= W; - } - norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz); - } - else{ - norm = 0.0; - } - Distance(i,j,k) += dt*sign*(1.0 - norm); - TotalVariation += dt*sign*(1.0 - norm); - // Disallow any change in phase - // if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k); - } - } - } - TotalVariation /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2); - count++; - } - - return TotalVariation; -} - - -void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad) -{ - // Read in the full sphere pack - //...... READ IN THE SPHERES................................... - cout << "Reading the packing file..." << endl; - FILE *fid = fopen("pack.out","rb"); - INSIST(fid!=NULL,"Error opening pack.out"); - //.........Trash the header lines.......... - char line[100]; - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - fgetl(line, 100, fid); - //........read the spheres.................. - // We will read until a blank like or end-of-file is reached - int count = 0; - while ( !feof(fid) && fgets(line,100,fid)!=NULL ) { - char* line2 = line; - List_cx[count] = strtod(line2,&line2); - List_cy[count] = strtod(line2,&line2); - List_cz[count] = strtod(line2,&line2); - List_rad[count] = strtod(line2,&line2); - count++; - } - cout << "Number of spheres extracted is: " << count << endl; - INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" ); - // ............................................................. -} - -void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, - double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, - int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) -{ - // Use sphere lists to determine which nodes are in porespace - // Write out binary file for nodes - char value; - int N = Nx*Ny*Nz; // Domain size, including the halo - double hx,hy,hz; - double x,y,z; - double cx,cy,cz,r; - int imin,imax,jmin,jmax,kmin,kmax; - int p,i,j,k,n; - //............................................ - double min_x,min_y,min_z; -// double max_x,max_y,max_z; - //............................................ - // Lattice spacing for the entire domain - // It should generally be true that hx=hy=hz - // Otherwise, you will end up with ellipsoids - hx = Lx/(Nx*nprocx-1); - hy = Ly/(Ny*nprocy-1); - hz = Lz/(Nz*nprocz-1); - //............................................ - // Get maximum and minimum for this domain - // Halo is included ! - min_x = double(iproc*Nx-1)*hx; - min_y = double(jproc*Ny-1)*hy; - min_z = double(kproc*Nz-1)*hz; -// max_x = ((iproc+1)*Nx+1)*hx; -// max_y = ((jproc+1)*Ny+1)*hy; -// max_z = ((kproc+1)*Nz+1)*hz; - //............................................ - - //............................................ - // Pre-initialize local ID - for (n=0;nNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;iNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;iNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;iNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;i( FILENAME ); + auto domain_db = db->getDatabase( "Domain" ); + int Nx = domain_db->getVector( "n" )[0]; + int Ny = domain_db->getVector( "n" )[1]; + int Nz = domain_db->getVector( "n" )[2]; + int nprocx = domain_db->getVector( "nproc" )[0]; + int nprocy = domain_db->getVector( "nproc" )[1]; + int nprocz = domain_db->getVector( "nproc" )[2]; + int nspheres = domain_db->getScalar( "nsphere" ); + int Lx = domain_db->getVector( "L" )[0]; + int Ly = domain_db->getVector( "L" )[1]; + int Lz = domain_db->getVector( "L" )[2]; + int i,j,k,n; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - - // ************************************************************** - if (nprocs != nprocx*nprocy*nprocz){ printf("nprocx = %i \n",nprocx); printf("nprocy = %i \n",nprocy); @@ -106,13 +318,6 @@ int main(int argc, char **argv) INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); } - InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, - rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, - rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, - rank_yz, rank_YZ, rank_yZ, rank_Yz ); - - MPI_Barrier(comm); - Nz += 2; Nx = Ny = Nz; // Cubic domain From 30ca7b8b7c991a6fbfba773eb24d8bb2c299c9c6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 09:24:26 -0400 Subject: [PATCH 032/124] Fixed broken ColorToBinary --- tests/ColorToBinary.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/ColorToBinary.cpp b/tests/ColorToBinary.cpp index 6c3ac5e8..fbb991aa 100644 --- a/tests/ColorToBinary.cpp +++ b/tests/ColorToBinary.cpp @@ -10,6 +10,27 @@ using namespace std; +inline void ReadBinaryFile(char *FILENAME, double *Data, int N) +{ + int n; + double value; + ifstream File(FILENAME,ios::binary); + if (File.good()){ + for (n=0; n Date: Wed, 16 May 2018 09:27:04 -0400 Subject: [PATCH 033/124] refactor Domain --- common/Domain.cpp | 21 ++++++++++++++++++++- common/Domain.h | 1 + tests/ColorToBinary.cpp | 21 --------------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 000e6a13..13487c51 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -674,7 +674,7 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh) UnpackMeshData(recvList_YZ, recvCount_YZ ,recvData_YZ, MeshData); } - +// Ideally stuff below here should be moved somewhere else -- doesn't really belong here void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np) { int q,n; @@ -714,4 +714,23 @@ void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np) } +inline void ReadBinaryFile(char *FILENAME, double *Data, int N) +{ + int n; + double value; + ifstream File(FILENAME,ios::binary); + if (File.good()){ + for (n=0; n Date: Wed, 16 May 2018 09:33:56 -0400 Subject: [PATCH 034/124] kill dead code --- common/ScaLBL.cpp | 1504 --------------------------------------------- common/ScaLBL.h | 148 +---- 2 files changed, 12 insertions(+), 1640 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index d7db9dc4..a551ab93 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1334,1043 +1334,6 @@ void ScaLBL_Communicator::MemoryDenseLayoutFull(IntArray &Map, int *neighborList delete [] TempBuffer; } - -void ScaLBL_Communicator::MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np){ - /* - * Generate a memory optimized layout - * id[n] == 0 implies that site n should be ignored (treat as a mask) - * Map(i,j,k) = idx <- this is the index for the memory optimized layout - * neighborList(idx) <-stores the neighbors for the D3Q19 model - * note that the number of communications remains the same - * the index in the Send and Recv lists is also updated - * this means that the commuincations are no longer valid for regular data structures - */ - int idx,i,j,k,n; - - // Check that Map has size matching sub-domain - if (Map.size(0) != Nx) - ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Map array dimensions do not match! \n"); - - // Initialize Map - for (k=0;k Np ){ - ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Failed to create memory efficient layout!\n"); - } - - // for (k=1;k Np) printf("ScaLBL_Communicator::MemoryOptimizedLayout: Map(%i,%i,%i) = %i > %i \n",i,j,k,Map(i,j,k),Np); - else if (!(idx<0)){ - // store the idx associated with each neighbor - // store idx for self if neighbor is in solid or out of domain - //D3Q19 = {{1,0,0},{-1,0,0} - // {0,1,0},{0,-1,0} - // {0,0,1},{0,0,-1}, - // {1,1,0},{-1,-1,0}, - // {1,-1,0},{-1,1,0}, - // {1,0,1},{-1,0,-1}, - // {1,0,-1},{-1,0,1}, - // {0,1,1},{0,-1,-1}, - // {0,1,-1},{0,-1,1}}; - // note that only odd distributions need to be stored to execute the swap algorithm - int neighbor; // cycle through the neighbors of lattice site idx - neighbor=Map(i+1,j,k); - if (neighbor==-2) neighborList[idx]=-1; - else if (neighbor<0) neighborList[idx]=idx; - else neighborList[idx]=neighbor; - - neighbor=Map(i,j+1,k); - if (neighbor==-2) neighborList[Np+idx]=-1; - else if (neighbor<0) neighborList[Np+idx]=idx; - else neighborList[Np+idx]=neighbor; - - neighbor=Map(i,j,k+1); - if (neighbor==-2) neighborList[2*Np+idx]=-1; - else if (neighbor<0) neighborList[2*Np+idx]=idx; - else neighborList[2*Np+idx]=neighbor; - - neighbor=Map(i+1,j+1,k); - if (neighbor==-2) neighborList[3*Np+idx]=-1; - else if (neighbor<0) neighborList[3*Np+idx]=idx; - else neighborList[3*Np+idx]=neighbor; - - neighbor=Map(i+1,j-1,k); - if (neighbor==-2) neighborList[4*Np+idx]=-1; - else if (neighbor<0) neighborList[4*Np+idx]=idx; - else neighborList[4*Np+idx]=neighbor; - - neighbor=Map(i+1,j,k+1); - if (neighbor==-2) neighborList[5*Np+idx]=-1; - else if (neighbor<0) neighborList[5*Np+idx]=idx; - else neighborList[5*Np+idx]=neighbor; - - neighbor=Map(i+1,j,k-1); - if (neighbor==-2) neighborList[6*Np+idx]=-1; - else if (neighbor<0) neighborList[6*Np+idx]=idx; - else neighborList[6*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k+1); - if (neighbor==-2) neighborList[7*Np+idx]=-1; - else if (neighbor<0) neighborList[7*Np+idx]=idx; - else neighborList[7*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k-1); - if (neighbor==-2) neighborList[8*Np+idx]=-1; - else if (neighbor<0) neighborList[8*Np+idx]=idx; - else neighborList[8*Np+idx]=neighbor; - } - } - } - } - - //for (idx=0; idx Np ){ - ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Failed to create memory efficient layout!\n"); - } - - // for (k=1;k Np) printf("ScaLBL_Communicator::MemoryOptimizedLayout: Map(%i,%i,%i) = %i > %i \n",i,j,k,Map(i,j,k),Np); - else if (!(idx<0)){ - // store the idx associated with each neighbor - // store idx for self if neighbor is in solid or out of domain - //D3Q19 = {{1,0,0},{-1,0,0} - // {0,1,0},{0,-1,0} - // {0,0,1},{0,0,-1}, - // {1,1,0},{-1,-1,0}, - // {1,-1,0},{-1,1,0}, - // {1,0,1},{-1,0,-1}, - // {1,0,-1},{-1,0,1}, - // {0,1,1},{0,-1,-1}, - // {0,1,-1},{0,-1,1}}; - // note that only odd distributions need to be stored to execute the swap algorithm - int neighbor; // cycle through the neighbors of lattice site idx - neighbor=Map(i+1,j,k); - if (neighbor==-2) neighborList[idx]=-1; - else if (neighbor<0) neighborList[idx]=idx; - else neighborList[idx]=neighbor; - - neighbor=Map(i-1,j,k); - if (neighbor==-2) neighborList[Np+idx]=-1; - else if (neighbor<0) neighborList[Np+idx]=idx; - else neighborList[Np+idx]=neighbor; - - neighbor=Map(i,j+1,k); - if (neighbor==-2) neighborList[2*Np+idx]=-1; - else if (neighbor<0) neighborList[2*Np+idx]=idx; - else neighborList[2*Np+idx]=neighbor; - - neighbor=Map(i,j-1,k); - if (neighbor==-2) neighborList[3*Np+idx]=-1; - else if (neighbor<0) neighborList[3*Np+idx]=idx; - else neighborList[3*Np+idx]=neighbor; - - - neighbor=Map(i,j,k+1); - if (neighbor==-2) neighborList[4*Np+idx]=-1; - else if (neighbor<0) neighborList[4*Np+idx]=idx; - else neighborList[4*Np+idx]=neighbor; - - neighbor=Map(i,j,k-1); - if (neighbor==-2) neighborList[5*Np+idx]=-1; - else if (neighbor<0) neighborList[5*Np+idx]=idx; - else neighborList[5*Np+idx]=neighbor; - - neighbor=Map(i+1,j+1,k); - if (neighbor==-2) neighborList[6*Np+idx]=-1; - else if (neighbor<0) neighborList[6*Np+idx]=idx; - else neighborList[6*Np+idx]=neighbor; - - neighbor=Map(i-1,j-1,k); - if (neighbor==-2) neighborList[7*Np+idx]=-1; - else if (neighbor<0) neighborList[7*Np+idx]=idx; - else neighborList[7*Np+idx]=neighbor; - - - neighbor=Map(i+1,j-1,k); - if (neighbor==-2) neighborList[8*Np+idx]=-1; - else if (neighbor<0) neighborList[8*Np+idx]=idx; - else neighborList[8*Np+idx]=neighbor; - - neighbor=Map(i-1,j+1,k); - if (neighbor==-2) neighborList[9*Np+idx]=-1; - else if (neighbor<0) neighborList[9*Np+idx]=idx; - else neighborList[9*Np+idx]=neighbor; - - - neighbor=Map(i+1,j,k+1); - if (neighbor==-2) neighborList[10*Np+idx]=-1; - else if (neighbor<0) neighborList[10*Np+idx]=idx; - else neighborList[10*Np+idx]=neighbor; - - neighbor=Map(i-1,j,k-1); - if (neighbor==-2) neighborList[11*Np+idx]=-1; - else if (neighbor<0) neighborList[11*Np+idx]=idx; - else neighborList[11*Np+idx]=neighbor; - - - neighbor=Map(i+1,j,k-1); - if (neighbor==-2) neighborList[12*Np+idx]=-1; - else if (neighbor<0) neighborList[12*Np+idx]=idx; - else neighborList[12*Np+idx]=neighbor; - - neighbor=Map(i-1,j,k+1); - if (neighbor==-2) neighborList[13*Np+idx]=-1; - else if (neighbor<0) neighborList[13*Np+idx]=idx; - else neighborList[13*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k+1); - if (neighbor==-2) neighborList[14*Np+idx]=-1; - else if (neighbor<0) neighborList[14*Np+idx]=idx; - else neighborList[14*Np+idx]=neighbor; - - neighbor=Map(i,j-1,k-1); - if (neighbor==-2) neighborList[15*Np+idx]=-1; - else if (neighbor<0) neighborList[15*Np+idx]=idx; - else neighborList[15*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k-1); - if (neighbor==-2) neighborList[16*Np+idx]=-1; - else if (neighbor<0) neighborList[16*Np+idx]=idx; - else neighborList[16*Np+idx]=neighbor; - - neighbor=Map(i,j-1,k+1); - if (neighbor==-2) neighborList[17*Np+idx]=-1; - else if (neighbor<0) neighborList[17*Np+idx]=idx; - else neighborList[17*Np+idx]=neighbor; - } - } - } - } - - //for (idx=0; idx Date: Wed, 16 May 2018 09:36:43 -0400 Subject: [PATCH 035/124] kill dead code --- common/ScaLBL.cpp | 991 ---------------------------------------------- 1 file changed, 991 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index a551ab93..8c6b80c1 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -343,997 +343,6 @@ void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, i delete [] ReturnDist; } - -void ScaLBL_Communicator::MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np){ - /* - * Generate a memory optimized layout - * id[n] == 0 implies that site n should be ignored (treat as a mask) - * Map(i,j,k) = idx <- this is the index for the memory optimized layout - * neighborList(idx) <-stores the neighbors for the D3Q19 model - * note that the number of communications remains the same - * the index in the Send and Recv lists is also updated - * this means that the commuincations are no longer valid for regular data structures - */ - int idx,i,j,k,n; - - // Check that Map has size matching sub-domain - if (Map.size(0) != Nx) - ERROR("ScaLBL_Communicator::MemoryDenseLayout: Map array dimensions do not match! \n"); - - // Initialize Map - for (k=0;k Np ){ - ERROR("ScaLBL_Communicator::MemoryDenseLayout: Failed to create memory efficient layout!\n"); - } - - // for (k=1;k Np) printf("ScaLBL_Communicator::MemoryDenseLayout: Map(%i,%i,%i) = %i > %i \n",i,j,k,Map(i,j,k),Np); - else if (!(idx<0)){ - // store the idx associated with each neighbor - // store idx for self if neighbor is in solid or out of domain - //D3Q19 = {{1,0,0},{-1,0,0} - // {0,1,0},{0,-1,0} - // {0,0,1},{0,0,-1}, - // {1,1,0},{-1,-1,0}, - // {1,-1,0},{-1,1,0}, - // {1,0,1},{-1,0,-1}, - // {1,0,-1},{-1,0,1}, - // {0,1,1},{0,-1,-1}, - // {0,1,-1},{0,-1,1}}; - // note that only odd distributions need to be stored to execute the swap algorithm - int neighbor; // cycle through the neighbors of lattice site idx - neighbor=Map(i+1,j,k); - if (neighbor==-2) neighborList[idx]=-1; - else if (neighbor<0) neighborList[idx]=idx; - else neighborList[idx]=neighbor; - - neighbor=Map(i,j+1,k); - if (neighbor==-2) neighborList[Np+idx]=-1; - else if (neighbor<0) neighborList[Np+idx]=idx; - else neighborList[Np+idx]=neighbor; - - neighbor=Map(i,j,k+1); - if (neighbor==-2) neighborList[2*Np+idx]=-1; - else if (neighbor<0) neighborList[2*Np+idx]=idx; - else neighborList[2*Np+idx]=neighbor; - - neighbor=Map(i+1,j+1,k); - if (neighbor==-2) neighborList[3*Np+idx]=-1; - else if (neighbor<0) neighborList[3*Np+idx]=idx; - else neighborList[3*Np+idx]=neighbor; - - neighbor=Map(i+1,j-1,k); - if (neighbor==-2) neighborList[4*Np+idx]=-1; - else if (neighbor<0) neighborList[4*Np+idx]=idx; - else neighborList[4*Np+idx]=neighbor; - - neighbor=Map(i+1,j,k+1); - if (neighbor==-2) neighborList[5*Np+idx]=-1; - else if (neighbor<0) neighborList[5*Np+idx]=idx; - else neighborList[5*Np+idx]=neighbor; - - neighbor=Map(i+1,j,k-1); - if (neighbor==-2) neighborList[6*Np+idx]=-1; - else if (neighbor<0) neighborList[6*Np+idx]=idx; - else neighborList[6*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k+1); - if (neighbor==-2) neighborList[7*Np+idx]=-1; - else if (neighbor<0) neighborList[7*Np+idx]=idx; - else neighborList[7*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k-1); - if (neighbor==-2) neighborList[8*Np+idx]=-1; - else if (neighbor<0) neighborList[8*Np+idx]=idx; - else neighborList[8*Np+idx]=neighbor; - } - } - } - } - - //for (idx=0; idx Np ){ - ERROR("ScaLBL_Communicator::MemoryDenseLayoutFull: Failed to create memory efficient layout!\n"); - } - - // if (rank == 0) { - // printf("* Displaying the final map from rank %d\n",rank); - // - // for (k=1;k Np) printf("ScaLBL_Communicator::MemoryDenseLayoutFull: Map(%i,%i,%i) = %i > %i \n",i,j,k,Map(i,j,k),Np); - else if (!(idx<0)){ - // store the idx associated with each neighbor - // store idx for self if neighbor is in solid or out of domain - //D3Q19 = {{1,0,0},{-1,0,0} - // {0,1,0},{0,-1,0} - // {0,0,1},{0,0,-1}, - // {1,1,0},{-1,-1,0}, - // {1,-1,0},{-1,1,0}, - // {1,0,1},{-1,0,-1}, - // {1,0,-1},{-1,0,1}, - // {0,1,1},{0,-1,-1}, - // {0,1,-1},{0,-1,1}}; - - - /* - * Storing the full neighbor list. The AA algorithm may require fewer neighbors but I'm saving everything for now... - * - */ - - - int neighbor; // cycle through the neighbors of lattice site idx - neighbor=Map(i+1,j,k); - if (neighbor==-2) neighborList[idx]=-1; - else if (neighbor<0) neighborList[idx]=idx; - else neighborList[idx]=neighbor; - - // 2 - neighbor=Map(i-1,j,k); - if (neighbor==-2) neighborList[Np+idx]=-1; - else if (neighbor<0) neighborList[Np+idx]=idx; - else neighborList[Np+idx]=neighbor; - - neighbor=Map(i,j+1,k); - if (neighbor==-2) neighborList[2*Np+idx]=-1; - else if (neighbor<0) neighborList[2*Np+idx]=idx; - else neighborList[2*Np+idx]=neighbor; - - // 4 - neighbor=Map(i,j-1,k); - if (neighbor==-2) neighborList[3*Np+idx]=-1; - else if (neighbor<0) neighborList[3*Np+idx]=idx; - else neighborList[3*Np+idx]=neighbor; - - - neighbor=Map(i,j,k+1); - if (neighbor==-2) neighborList[4*Np+idx]=-1; - else if (neighbor<0) neighborList[4*Np+idx]=idx; - else neighborList[4*Np+idx]=neighbor; - - // 6 - neighbor=Map(i,j,k-1); - if (neighbor==-2) neighborList[5*Np+idx]=-1; - else if (neighbor<0) neighborList[5*Np+idx]=idx; - else neighborList[5*Np+idx]=neighbor; - - neighbor=Map(i+1,j+1,k); - if (neighbor==-2) neighborList[6*Np+idx]=-1; - else if (neighbor<0) neighborList[6*Np+idx]=idx; - else neighborList[6*Np+idx]=neighbor; - - // 8 - neighbor=Map(i-1,j-1,k); - if (neighbor==-2) neighborList[7*Np+idx]=-1; - else if (neighbor<0) neighborList[7*Np+idx]=idx; - else neighborList[7*Np+idx]=neighbor; - - - neighbor=Map(i+1,j-1,k); - if (neighbor==-2) neighborList[8*Np+idx]=-1; - else if (neighbor<0) neighborList[8*Np+idx]=idx; - else neighborList[8*Np+idx]=neighbor; - - // 10 - neighbor=Map(i-1,j+1,k); - if (neighbor==-2) neighborList[9*Np+idx]=-1; - else if (neighbor<0) neighborList[9*Np+idx]=idx; - else neighborList[9*Np+idx]=neighbor; - - - neighbor=Map(i+1,j,k+1); - if (neighbor==-2) neighborList[10*Np+idx]=-1; - else if (neighbor<0) neighborList[10*Np+idx]=idx; - else neighborList[10*Np+idx]=neighbor; - - // 12 - neighbor=Map(i-1,j,k-1); - if (neighbor==-2) neighborList[11*Np+idx]=-1; - else if (neighbor<0) neighborList[11*Np+idx]=idx; - else neighborList[11*Np+idx]=neighbor; - - - neighbor=Map(i+1,j,k-1); - if (neighbor==-2) neighborList[12*Np+idx]=-1; - else if (neighbor<0) neighborList[12*Np+idx]=idx; - else neighborList[12*Np+idx]=neighbor; - - // 14 - neighbor=Map(i-1,j,k+1); - if (neighbor==-2) neighborList[13*Np+idx]=-1; - else if (neighbor<0) neighborList[13*Np+idx]=idx; - else neighborList[13*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k+1); - if (neighbor==-2) neighborList[14*Np+idx]=-1; - else if (neighbor<0) neighborList[14*Np+idx]=idx; - else neighborList[14*Np+idx]=neighbor; - - // 16 - neighbor=Map(i,j-1,k-1); - if (neighbor==-2) neighborList[15*Np+idx]=-1; - else if (neighbor<0) neighborList[15*Np+idx]=idx; - else neighborList[15*Np+idx]=neighbor; - - neighbor=Map(i,j+1,k-1); - if (neighbor==-2) neighborList[16*Np+idx]=-1; - else if (neighbor<0) neighborList[16*Np+idx]=idx; - else neighborList[16*Np+idx]=neighbor; - - // 18 - neighbor=Map(i,j-1,k+1); - if (neighbor==-2) neighborList[17*Np+idx]=-1; - else if (neighbor<0) neighborList[17*Np+idx]=idx; - else neighborList[17*Np+idx]=neighbor; - - - } - } - } - } - - //....................................................................... - // Now map through SendList and RecvList to update indices - // First loop over the send lists - - int *TempBuffer; - TempBuffer = new int [5*RecvCount]; - - //....................................................................... - // Re-index the send lists - ScaLBL_CopyToHost(TempBuffer,dvcSendList_x,sendCount_x*sizeof(int)); - - for (i=0; i Date: Wed, 16 May 2018 09:43:06 -0400 Subject: [PATCH 036/124] kill dead code --- common/ScaLBL.cpp | 117 ---------------------------------------------- 1 file changed, 117 deletions(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 8c6b80c1..6390ad2d 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -896,123 +896,6 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis return(Np); } - - - -void ScaLBL_Communicator::SendD3Q19(double *f_even, double *f_odd){ - - if (Lock==true){ - ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?"); - } - else{ - Lock=true; - } - // assign tag of 19 to D3Q19 communication - sendtag = recvtag = 19; - ScaLBL_DeviceBarrier(); - // Pack the distributions - ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,f_odd,N); - ScaLBL_D3Q19_Pack(4,dvcSendList_X,sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - ScaLBL_D3Q19_Pack(5,dvcSendList_X,2*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - ScaLBL_D3Q19_Pack(6,dvcSendList_X,3*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - ScaLBL_D3Q19_Pack(7,dvcSendList_X,4*sendCount_X,sendCount_X,sendbuf_X,f_odd,N); - //...Packing for X face(1,7,9,11,13)................................ - ScaLBL_D3Q19_Pack(0,dvcSendList_x,0,sendCount_x,sendbuf_x,f_even,N); - ScaLBL_D3Q19_Pack(3,dvcSendList_x,sendCount_x,sendCount_x,sendbuf_x,f_even,N); - ScaLBL_D3Q19_Pack(4,dvcSendList_x,2*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - ScaLBL_D3Q19_Pack(5,dvcSendList_x,3*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - ScaLBL_D3Q19_Pack(6,dvcSendList_x,4*sendCount_x,sendCount_x,sendbuf_x,f_even,N); - //...Packing for y face(4,8,9,16,18)................................. - ScaLBL_D3Q19_Pack(2,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,f_even,N); - ScaLBL_D3Q19_Pack(4,dvcSendList_Y,sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - ScaLBL_D3Q19_Pack(4,dvcSendList_Y,2*sendCount_Y,sendCount_Y,sendbuf_Y,f_odd,N); - ScaLBL_D3Q19_Pack(8,dvcSendList_Y,3*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - ScaLBL_D3Q19_Pack(9,dvcSendList_Y,4*sendCount_Y,sendCount_Y,sendbuf_Y,f_even,N); - //...Packing for Y face(3,7,10,15,17)................................. - ScaLBL_D3Q19_Pack(1,dvcSendList_y,0,sendCount_y,sendbuf_y,f_odd,N); - ScaLBL_D3Q19_Pack(3,dvcSendList_y,sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - ScaLBL_D3Q19_Pack(5,dvcSendList_y,2*sendCount_y,sendCount_y,sendbuf_y,f_even,N); - ScaLBL_D3Q19_Pack(7,dvcSendList_y,3*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - ScaLBL_D3Q19_Pack(8,dvcSendList_y,4*sendCount_y,sendCount_y,sendbuf_y,f_odd,N); - //...Packing for z face(6,12,13,16,17)................................ - ScaLBL_D3Q19_Pack(3,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,f_even,N); - ScaLBL_D3Q19_Pack(6,dvcSendList_Z,sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); - ScaLBL_D3Q19_Pack(6,dvcSendList_Z,2*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - ScaLBL_D3Q19_Pack(8,dvcSendList_Z,3*sendCount_Z,sendCount_Z,sendbuf_Z,f_even,N); - ScaLBL_D3Q19_Pack(8,dvcSendList_Z,4*sendCount_Z,sendCount_Z,sendbuf_Z,f_odd,N); - //...Packing for Z face(5,11,14,15,18)................................ - ScaLBL_D3Q19_Pack(2,dvcSendList_z,0,sendCount_z,sendbuf_z,f_odd,N); - ScaLBL_D3Q19_Pack(5,dvcSendList_z,sendCount_z,sendCount_z,sendbuf_z,f_odd,N); - ScaLBL_D3Q19_Pack(7,dvcSendList_z,2*sendCount_z,sendCount_z,sendbuf_z,f_even,N); - ScaLBL_D3Q19_Pack(7,dvcSendList_z,3*sendCount_z,sendCount_z,sendbuf_z,f_odd,N); - ScaLBL_D3Q19_Pack(9,dvcSendList_z,4*sendCount_z,sendCount_z,sendbuf_z,f_even,N); - //...Pack the xy edge (8)................................ - ScaLBL_D3Q19_Pack(4,dvcSendList_XY,0,sendCount_XY,sendbuf_XY,f_even,N); - //...Pack the Xy edge (9)................................ - ScaLBL_D3Q19_Pack(4,dvcSendList_xY,0,sendCount_xY,sendbuf_xY,f_odd,N); - //...Pack the xY edge (10)................................ - ScaLBL_D3Q19_Pack(5,dvcSendList_Xy,0,sendCount_Xy,sendbuf_Xy,f_even,N); - //...Pack the XY edge (7)................................ - ScaLBL_D3Q19_Pack(3,dvcSendList_xy,0,sendCount_xy,sendbuf_xy,f_odd,N); - //...Pack the xz edge (12)................................ - ScaLBL_D3Q19_Pack(6,dvcSendList_XZ,0,sendCount_XZ,sendbuf_XZ,f_even,N); - //...Pack the xZ edge (14)................................ - ScaLBL_D3Q19_Pack(7,dvcSendList_Xz,0,sendCount_Xz,sendbuf_Xz,f_even,N); - //...Pack the Xz edge (13)................................ - ScaLBL_D3Q19_Pack(6,dvcSendList_xZ,0,sendCount_xZ,sendbuf_xZ,f_odd,N); - //...Pack the XZ edge (11)................................ - ScaLBL_D3Q19_Pack(5,dvcSendList_xz,0,sendCount_xz,sendbuf_xz,f_odd,N); - //...Pack the xz edge (12)................................ - //...Pack the yz edge (16)................................ - ScaLBL_D3Q19_Pack(8,dvcSendList_YZ,0,sendCount_YZ,sendbuf_YZ,f_even,N); - //...Pack the yZ edge (18)................................ - ScaLBL_D3Q19_Pack(9,dvcSendList_Yz,0,sendCount_Yz,sendbuf_Yz,f_even,N); - //...Pack the Yz edge (17)................................ - ScaLBL_D3Q19_Pack(8,dvcSendList_yZ,0,sendCount_yZ,sendbuf_yZ,f_odd,N); - //...Pack the YZ edge (15)................................ - ScaLBL_D3Q19_Pack(7,dvcSendList_yz,0,sendCount_yz,sendbuf_yz,f_odd,N); - //................................................................................... - - //................................................................................... - // Send all the distributions - MPI_Isend(sendbuf_x, 5*sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]); - MPI_Irecv(recvbuf_X, 5*recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]); - MPI_Isend(sendbuf_X, 5*sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]); - MPI_Irecv(recvbuf_x, 5*recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]); - MPI_Isend(sendbuf_y, 5*sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]); - MPI_Irecv(recvbuf_Y, 5*recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]); - MPI_Isend(sendbuf_Y, 5*sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]); - MPI_Irecv(recvbuf_y, 5*recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]); - MPI_Isend(sendbuf_z, 5*sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]); - MPI_Irecv(recvbuf_Z, 5*recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]); - MPI_Isend(sendbuf_Z, 5*sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]); - MPI_Irecv(recvbuf_z, 5*recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]); - MPI_Isend(sendbuf_xy, sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,MPI_COMM_SCALBL,&req1[6]); - MPI_Irecv(recvbuf_XY, recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,MPI_COMM_SCALBL,&req2[6]); - MPI_Isend(sendbuf_XY, sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,MPI_COMM_SCALBL,&req1[7]); - MPI_Irecv(recvbuf_xy, recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,MPI_COMM_SCALBL,&req2[7]); - MPI_Isend(sendbuf_Xy, sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,MPI_COMM_SCALBL,&req1[8]); - MPI_Irecv(recvbuf_xY, recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,MPI_COMM_SCALBL,&req2[8]); - MPI_Isend(sendbuf_xY, sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,MPI_COMM_SCALBL,&req1[9]); - MPI_Irecv(recvbuf_Xy, recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,MPI_COMM_SCALBL,&req2[9]); - MPI_Isend(sendbuf_xz, sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,MPI_COMM_SCALBL,&req1[10]); - MPI_Irecv(recvbuf_XZ, recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,MPI_COMM_SCALBL,&req2[10]); - MPI_Isend(sendbuf_XZ, sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,MPI_COMM_SCALBL,&req1[11]); - MPI_Irecv(recvbuf_xz, recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,MPI_COMM_SCALBL,&req2[11]); - MPI_Isend(sendbuf_Xz, sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,MPI_COMM_SCALBL,&req1[12]); - MPI_Irecv(recvbuf_xZ, recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,MPI_COMM_SCALBL,&req2[12]); - MPI_Isend(sendbuf_xZ, sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,MPI_COMM_SCALBL,&req1[13]); - MPI_Irecv(recvbuf_Xz, recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,MPI_COMM_SCALBL,&req2[13]); - MPI_Isend(sendbuf_yz, sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,MPI_COMM_SCALBL,&req1[14]); - MPI_Irecv(recvbuf_YZ, recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,MPI_COMM_SCALBL,&req2[14]); - MPI_Isend(sendbuf_YZ, sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,MPI_COMM_SCALBL,&req1[15]); - MPI_Irecv(recvbuf_yz, recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,MPI_COMM_SCALBL,&req2[15]); - MPI_Isend(sendbuf_Yz, sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,MPI_COMM_SCALBL,&req1[16]); - MPI_Irecv(recvbuf_yZ, recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,MPI_COMM_SCALBL,&req2[16]); - MPI_Isend(sendbuf_yZ, sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,MPI_COMM_SCALBL,&req1[17]); - MPI_Irecv(recvbuf_Yz, recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,MPI_COMM_SCALBL,&req2[17]); -} - void ScaLBL_Communicator::SendD3Q19AA(double *dist){ // NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2 From 83c15773b4a862ebbadc09f7774b8d48085d4686 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 09:54:48 -0400 Subject: [PATCH 037/124] refactor --- common/Domain.cpp | 4 +--- common/Domain.h | 2 +- tests/ColorToBinary.cpp | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 13487c51..57741f24 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -713,8 +713,7 @@ void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np) File.close(); } - -inline void ReadBinaryFile(char *FILENAME, double *Data, int N) +void ReadBinaryFile(char *FILENAME, double *Data, int N) { int n; double value; @@ -731,6 +730,5 @@ inline void ReadBinaryFile(char *FILENAME, double *Data, int N) for (n=0; n Date: Wed, 16 May 2018 09:57:44 -0400 Subject: [PATCH 038/124] refactor sphere --- tests/GenerateSphereTest.cpp | 238 +++++++++++++++++++++++++++++++++++ tests/lbpm_sphere_pp.cpp | 2 - 2 files changed, 238 insertions(+), 2 deletions(-) diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index add99341..8047e96b 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -19,6 +19,244 @@ using namespace std; + +void WriteLocalSolidID(char *FILENAME, char *ID, int N) +{ + char value; + ofstream File(FILENAME,ios::binary); + for (int n=0; nNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;iNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;i Date: Wed, 16 May 2018 10:16:11 -0400 Subject: [PATCH 039/124] refactor sphere stuff --- common/SpherePack.cpp | 252 ++++++++++++++++++++++++++++++++ common/SpherePack.h | 35 +++++ tests/GenerateSphereTest.cpp | 267 ++-------------------------------- tests/TestSphereCurvature.cpp | 1 + tests/lbpm_sphere_pp.cpp | 238 +----------------------------- 5 files changed, 304 insertions(+), 489 deletions(-) create mode 100644 common/SpherePack.cpp create mode 100644 common/SpherePack.h diff --git a/common/SpherePack.cpp b/common/SpherePack.cpp new file mode 100644 index 00000000..16da5209 --- /dev/null +++ b/common/SpherePack.cpp @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/Array.h" +#include "common/Utilities.h" +#include "common/MPI_Helpers.h" +#include "common/Communication.h" +#include "common/Database.h" +#include "common/SpherePack.h" + +void WriteLocalSolidID(char *FILENAME, char *ID, int N) +{ + char value; + ofstream File(FILENAME,ios::binary); + for (int n=0; nNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;iNx) imin = Nx; + if (imax<0) imax = 0; + if (imax>Nx) imax = Nx; + if (jmin<0) jmin = 0; + if (jmin>Ny) jmin = Ny; + if (jmax<0) jmax = 0; + if (jmax>Ny) jmax = Ny; + if (kmin<0) kmin = 0; + if (kmin>Nz) kmin = Nz; + if (kmax<0) kmax = 0; + if (kmax>Nz) kmax = Nz; + // Loop over the domain for this sphere (may be null) + for (i=imin;i +#include +#include +#include +#include +#include +#include +#include + +#include "common/Array.h" +#include "common/Utilities.h" +#include "common/MPI_Helpers.h" +#include "common/Communication.h" +#include "common/Database.h" + +/* +Simple tools to work with sphere packs + */ + +void WriteLocalSolidID(char *FILENAME, char *ID, int N); + +void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N); + +void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad); + +void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); + +void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, + double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 8047e96b..6c9edba8 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -8,6 +8,7 @@ //#include "common/pmmc.h" #include "common/Domain.h" +#include "common/SpherePack.h" #include "common/MPI_Helpers.h" #include "common/Communication.h" @@ -19,265 +20,27 @@ using namespace std; - -void WriteLocalSolidID(char *FILENAME, char *ID, int N) -{ - char value; - ofstream File(FILENAME,ios::binary); - for (int n=0; nNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;iNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;i #include "analysis/pmmc.h" #include "common/Domain.h" +#include "common/SpherePack.h" using namespace std; diff --git a/tests/lbpm_sphere_pp.cpp b/tests/lbpm_sphere_pp.cpp index a081f338..05bdd0e3 100644 --- a/tests/lbpm_sphere_pp.cpp +++ b/tests/lbpm_sphere_pp.cpp @@ -8,6 +8,7 @@ #include "analysis/pmmc.h" #include "common/Domain.h" +#include "common/SpherePack.h" #include "common/MPI_Helpers.h" #include "common/Communication.h" @@ -19,243 +20,6 @@ using namespace std; -void WriteLocalSolidID(char *FILENAME, char *ID, int N) -{ - char value; - ofstream File(FILENAME,ios::binary); - for (int n=0; nNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;iNx) imin = Nx; - if (imax<0) imax = 0; - if (imax>Nx) imax = Nx; - if (jmin<0) jmin = 0; - if (jmin>Ny) jmin = Ny; - if (jmax<0) jmax = 0; - if (jmax>Ny) jmax = Ny; - if (kmin<0) kmin = 0; - if (kmin>Nz) kmin = Nz; - if (kmax<0) kmax = 0; - if (kmax>Nz) kmax = Nz; - // Loop over the domain for this sphere (may be null) - for (i=imin;i Date: Wed, 16 May 2018 10:19:31 -0400 Subject: [PATCH 040/124] refactor spheres --- common/Domain.cpp | 5 ----- common/SpherePack.cpp | 8 ++++++++ common/SpherePack.h | 2 ++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 57741f24..81a5f274 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -15,9 +15,6 @@ #include "common/MPI_Helpers.h" #include "common/Communication.h" - - - // Inline function to read line without a return argument static inline void fgetl( char * str, int num, FILE * stream ) { @@ -25,8 +22,6 @@ static inline void fgetl( char * str, int num, FILE * stream ) if ( 0 ) {char *temp = (char *)&ptr; temp++;} } - - /******************************************************** * Constructors/Destructor * ********************************************************/ diff --git a/common/SpherePack.cpp b/common/SpherePack.cpp index 16da5209..5fd19fa6 100644 --- a/common/SpherePack.cpp +++ b/common/SpherePack.cpp @@ -14,6 +14,14 @@ #include "common/Database.h" #include "common/SpherePack.h" +// Inline function to read line without a return argument +static inline void fgetl( char * str, int num, FILE * stream ) +{ + char* ptr = fgets( str, num, stream ); + if ( 0 ) {char *temp = (char *)&ptr; temp++;} +} + + void WriteLocalSolidID(char *FILENAME, char *ID, int N) { char value; diff --git a/common/SpherePack.h b/common/SpherePack.h index ddacc51e..5075b289 100644 --- a/common/SpherePack.h +++ b/common/SpherePack.h @@ -33,3 +33,5 @@ void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad, double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz); + +#endif From 39c13fd287df3a5bbc3e8c4a3f9d255e57314d08 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 10:22:55 -0400 Subject: [PATCH 041/124] refactor spheres --- tests/lbpm_disc_pp.cpp | 2 +- tests/lbpm_juanes_bench_disc_pp.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lbpm_disc_pp.cpp b/tests/lbpm_disc_pp.cpp index 6c427484..92036000 100644 --- a/tests/lbpm_disc_pp.cpp +++ b/tests/lbpm_disc_pp.cpp @@ -10,7 +10,7 @@ #include "common/Domain.h" #include "common/Communication.h" #include "common/MPI_Helpers.h" // This includes mpi.h - +#include "common/SpherePack.h" /* * Pre-Processor to generate signed distance function from disc packing diff --git a/tests/lbpm_juanes_bench_disc_pp.cpp b/tests/lbpm_juanes_bench_disc_pp.cpp index 843a1328..6f04cffa 100644 --- a/tests/lbpm_juanes_bench_disc_pp.cpp +++ b/tests/lbpm_juanes_bench_disc_pp.cpp @@ -10,7 +10,7 @@ #include "common/Domain.h" #include "common/Communication.h" #include "common/MPI_Helpers.h" // This includes mpi.h - +#include "common/SpherePack.h" /* * Pre-Processor to generate signed distance function from disc packing From f747641d61b46f219d5eba967a7db34b68eaecd8 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 10:25:34 -0400 Subject: [PATCH 042/124] refactor spheres --- tests/lbpm_sphere_pp.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/lbpm_sphere_pp.cpp b/tests/lbpm_sphere_pp.cpp index 05bdd0e3..98778b8d 100644 --- a/tests/lbpm_sphere_pp.cpp +++ b/tests/lbpm_sphere_pp.cpp @@ -32,7 +32,6 @@ int main(int argc, char **argv) MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; int iproc,jproc,kproc; int sendtag,recvtag; //***************************************** From 66a33c543bcf74eecdd3e6e1fcaf0fadd702a6a8 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 10:31:02 -0400 Subject: [PATCH 043/124] refactor spheres --- tests/TestBlobAnalyze.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/TestBlobAnalyze.cpp b/tests/TestBlobAnalyze.cpp index c5db51cf..d6733514 100644 --- a/tests/TestBlobAnalyze.cpp +++ b/tests/TestBlobAnalyze.cpp @@ -8,6 +8,7 @@ #include "common/Communication.h" #include "analysis/analysis.h" #include "analysis/TwoPhase.h" +#include "common/SpherePack.h" //#include "Domain.h" From 5148842447bc2f591f6319f7c19c0a897438c175 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 10:35:17 -0400 Subject: [PATCH 044/124] sphere refactor --- common/SpherePack.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/common/SpherePack.cpp b/common/SpherePack.cpp index 5fd19fa6..a7246b72 100644 --- a/common/SpherePack.cpp +++ b/common/SpherePack.cpp @@ -21,7 +21,6 @@ static inline void fgetl( char * str, int num, FILE * stream ) if ( 0 ) {char *temp = (char *)&ptr; temp++;} } - void WriteLocalSolidID(char *FILENAME, char *ID, int N) { char value; From 27898d890ff83b1341c3d1cc4654a89d96720523 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Wed, 16 May 2018 14:00:35 -0400 Subject: [PATCH 045/124] Fixing bug in Domain --- analysis/TwoPhase.cpp | 92 ++++---- analysis/eikonal.cpp | 336 ++++++++------------------- analysis/eikonal.h | 7 +- analysis/runAnalysis.cpp | 2 +- analysis/uCT.cpp | 2 +- common/Domain.cpp | 240 +++++++++---------- common/Domain.h | 152 ++++++------ common/ScaLBL.cpp | 50 ++-- tests/TestColorSquareTube.cpp | 4 +- tests/TestPoiseuille.cpp | 7 +- tests/TestSegDist.cpp | 106 +++++---- tests/TestTorus.cpp | 14 +- tests/lbpm_captube_pp.cpp | 4 +- tests/lbpm_color_macro_simulator.cpp | 8 +- tests/lbpm_dfh_simulator.cpp | 4 +- tests/lbpm_inkbottle_pp.cpp | 4 +- tests/lbpm_morph_pp.cpp | 6 +- tests/lbpm_morphdrain_pp.cpp | 78 +++---- tests/lbpm_morphopen_pp.cpp | 78 +++---- tests/lbpm_random_pp.cpp | 76 +++--- tests/lbpm_segmented_decomp.cpp | 8 +- tests/lbpm_segmented_pp.cpp | 13 +- tests/lbpm_sphere_pp.cpp | 2 - tests/lbpm_squaretube_pp.cpp | 12 +- 24 files changed, 589 insertions(+), 716 deletions(-) diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index 72acce40..492c295e 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -70,7 +70,7 @@ TwoPhase::TwoPhase(Domain &dm): As_global(0), wwndnw_global(0), wwnsdnwn_global(0), Jwnwwndnw_global(0), dEs(0), dAwn(0), dAns(0) { Nx=dm.Nx; Ny=dm.Ny; Nz=dm.Nz; - Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz*1.0; + Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm.nprocx()*Dm.nprocy()*Dm.nprocz()*1.0; TempID = new char[Nx*Ny*Nz]; @@ -135,7 +135,7 @@ TwoPhase::TwoPhase(Domain &dm): Gns_global.resize(6); Gws_global.resize(6); //......................................... - if (Dm.rank==0){ + if (Dm.rank()==0){ TIMELOG = fopen("timelog.tcat","a+"); if (fseek(TIMELOG,0,SEEK_SET) == fseek(TIMELOG,0,SEEK_CUR)) { @@ -165,7 +165,7 @@ TwoPhase::TwoPhase(Domain &dm): } else{ char LocalRankString[8]; - sprintf(LocalRankString,"%05d",Dm.rank); + sprintf(LocalRankString,"%05d",Dm.rank()); char LocalRankFilename[40]; sprintf(LocalRankFilename,"%s%s","timelog.tcat.",LocalRankString); TIMELOG = fopen(LocalRankFilename,"a+"); @@ -418,8 +418,8 @@ void TwoPhase::ComputeLocal() // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; - if (Dm.BoundaryCondition > 0 && Dm.kproc == 0) kmin=4; - if (Dm.BoundaryCondition > 0 && Dm.kproc == Dm.nprocz-1) kmax=Nz-4; + if (Dm.BoundaryCondition > 0 && Dm.kproc() == 0) kmin=4; + if (Dm.BoundaryCondition > 0 && Dm.kproc() == Dm.nprocz()-1) kmax=Nz-4; for (k=kmin; k 0 && Dm.kproc == 0) kmin=4; - if (Dm.BoundaryCondition > 0 && Dm.kproc == Dm.nprocz-1) kmax=Nz-4; + if (Dm.BoundaryCondition > 0 && Dm.kproc() == 0) kmin=4; + if (Dm.BoundaryCondition > 0 && Dm.kproc() == Dm.nprocz()-1) kmax=Nz-4; for (k=kmin; kA.push_back(A); wn_mesh->B.push_back(B); wn_mesh->C.push_back(C); @@ -1070,15 +1070,15 @@ void TwoPhase::WriteSurfaces(int logcount) B = ws_pts(ws_tris(1,r)); C = ws_pts(ws_tris(2,r)); // Remap the points - A.x += 1.0*Dm.iproc*(Nx-2); - A.y += 1.0*Dm.jproc*(Nx-2); - A.z += 1.0*Dm.kproc*(Nx-2); - B.x += 1.0*Dm.iproc*(Nx-2); - B.y += 1.0*Dm.jproc*(Nx-2); - B.z += 1.0*Dm.kproc*(Nx-2); - C.x += 1.0*Dm.iproc*(Nx-2); - C.y += 1.0*Dm.jproc*(Nx-2); - C.z += 1.0*Dm.kproc*(Nx-2); + A.x += 1.0*Dm.iproc()*(Nx-2); + A.y += 1.0*Dm.jproc()*(Nx-2); + A.z += 1.0*Dm.kproc()*(Nx-2); + B.x += 1.0*Dm.iproc()*(Nx-2); + B.y += 1.0*Dm.jproc()*(Nx-2); + B.z += 1.0*Dm.kproc()*(Nx-2); + C.x += 1.0*Dm.iproc()*(Nx-2); + C.y += 1.0*Dm.jproc()*(Nx-2); + C.z += 1.0*Dm.kproc()*(Nx-2); ws_mesh->A.push_back(A); ws_mesh->B.push_back(B); ws_mesh->C.push_back(C); @@ -1088,15 +1088,15 @@ void TwoPhase::WriteSurfaces(int logcount) B = ns_pts(ns_tris(1,r)); C = ns_pts(ns_tris(2,r)); // Remap the points - A.x += 1.0*Dm.iproc*(Nx-2); - A.y += 1.0*Dm.jproc*(Nx-2); - A.z += 1.0*Dm.kproc*(Nx-2); - B.x += 1.0*Dm.iproc*(Nx-2); - B.y += 1.0*Dm.jproc*(Nx-2); - B.z += 1.0*Dm.kproc*(Nx-2); - C.x += 1.0*Dm.iproc*(Nx-2); - C.y += 1.0*Dm.jproc*(Nx-2); - C.z += 1.0*Dm.kproc*(Nx-2); + A.x += 1.0*Dm.iproc()*(Nx-2); + A.y += 1.0*Dm.jproc()*(Nx-2); + A.z += 1.0*Dm.kproc()*(Nx-2); + B.x += 1.0*Dm.iproc()*(Nx-2); + B.y += 1.0*Dm.jproc()*(Nx-2); + B.z += 1.0*Dm.kproc()*(Nx-2); + C.x += 1.0*Dm.iproc()*(Nx-2); + C.y += 1.0*Dm.jproc()*(Nx-2); + C.z += 1.0*Dm.kproc()*(Nx-2); ns_mesh->A.push_back(A); ns_mesh->B.push_back(B); ns_mesh->C.push_back(C); @@ -1224,7 +1224,7 @@ void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT) void TwoPhase::PrintAll(int timestep) { - if (Dm.rank==0){ + if (Dm.rank()==0){ fprintf(TIMELOG,"%i %.5g ",timestep,dEs); // change in surface energy fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw_global,pan_global); // saturation and pressure fprintf(TIMELOG,"%.5g %.5g %.5g ",awn_global,ans_global,aws_global); // interfacial areas @@ -1277,7 +1277,7 @@ void TwoPhase::PrintAll(int timestep) void TwoPhase::PrintComponents(int timestep) { - if (Dm.rank==0){ + if (Dm.rank()==0){ printf("PRINT %i COMPONENT AVEREAGES: time = %i \n",(int)ComponentAverages_NWP.size(1),timestep); for (int b=0; b 0.0){ diff --git a/analysis/eikonal.cpp b/analysis/eikonal.cpp index 82d46246..01e00405 100644 --- a/analysis/eikonal.cpp +++ b/analysis/eikonal.cpp @@ -27,199 +27,55 @@ static inline double minmod(double &a, double &b){ } +template +static inline MPI_Datatype getType( ); +template<> inline MPI_Datatype getType() { return MPI_FLOAT; } +template<> inline MPI_Datatype getType() { return MPI_DOUBLE; } + + /****************************************************************** * Solve the eikonal equation * ******************************************************************/ -double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps) +template +TYPE Eikonal( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps) { - /* - * This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - * - * Reference: - * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 - */ - - int i,j,k; - double dt=0.1; - double Dx,Dy,Dz; - double Dxp,Dxm,Dyp,Dym,Dzp,Dzm; - double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm; - double sign,norm; - double LocalVar,GlobalVar,LocalMax,GlobalMax; - - int xdim,ydim,zdim; - xdim=Dm.Nx-2; - ydim=Dm.Ny-2; - zdim=Dm.Nz-2; - fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); - - // Arrays to store the second derivatives - DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz); - DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz); - DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz); - - int count = 0; - while (count < timesteps){ - - // Communicate the halo of values - fillData.fill(Distance); - - // Compute second order derivatives - for (k=1;k 0.f) Dx = Dxp*Dxp; - else Dx = Dxm*Dxm; - - if (Dyp + Dym > 0.f) Dy = Dyp*Dyp; - else Dy = Dym*Dym; - - if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp; - else Dz = Dzm*Dzm; - } - else{ - - if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp; - else Dx = Dxm*Dxm; - - if (Dyp + Dym < 0.f) Dy = Dyp*Dyp; - else Dy = Dym*Dym; - - if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp; - else Dz = Dzm*Dzm; - } - - //Dx = max(Dxp*Dxp,Dxm*Dxm); - //Dy = max(Dyp*Dyp,Dym*Dym); - //Dz = max(Dzp*Dzp,Dzm*Dzm); - - norm=sqrt(Dx + Dy + Dz); - if (norm > 1.0) norm=1.0; - - Distance(i,j,k) += dt*sign*(1.0 - norm); - LocalVar += dt*sign*(1.0 - norm); - - if (fabs(dt*sign*(1.0 - norm)) > LocalMax) - LocalMax = fabs(dt*sign*(1.0 - norm)); - } - } - } - - MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); - GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz; - count++; - - - if (count%50 == 0 && Dm.rank==0 ){ - printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); - fflush(stdout); - } - - if (fabs(GlobalMax) < 1e-5){ - if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n"); - count=timesteps; - } - } - return GlobalVar; -} - -float Eikonal3D( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps) -{ - PROFILE_START("Eikonal3D"); - /* - * This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign*(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - * - * Reference: - * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 - */ + * This routine converts the data in the Distance array to a signed distance + * by solving the equation df/dt = sign(1-|grad f|), where Distance provides + * the values of f on the mesh associated with domain Dm + * It has been tested with segmented data initialized to values [-1,1] + * and will converge toward the signed distance to the surface bounding the associated phases + * + * Reference: + * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 + */ - int i,j,k; - float dt=0.1; - float Dx,Dy,Dz; - float Dxp,Dxm,Dyp,Dym,Dzp,Dzm; - float Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm; - float sign,norm; - float LocalVar,GlobalVar,LocalMax,GlobalMax; - - int xdim,ydim,zdim; - xdim=Dm.Nx-2; - ydim=Dm.Ny-2; - zdim=Dm.Nz-2; - fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + int xdim = Dm.Nx-2; + int ydim = Dm.Ny-2; + int zdim = Dm.Nz-2; + fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); // Arrays to store the second derivatives - Array Dxx(Dm.Nx,Dm.Ny,Dm.Nz); - Array Dyy(Dm.Nx,Dm.Ny,Dm.Nz); - Array Dzz(Dm.Nx,Dm.Ny,Dm.Nz); + Array Dxx(Dm.Nx,Dm.Ny,Dm.Nz); + Array Dyy(Dm.Nx,Dm.Ny,Dm.Nz); + Array Dzz(Dm.Nx,Dm.Ny,Dm.Nz); + Array sign(ID.size()); + for (size_t i=0; i &Distance, const Array &ID, const Domain &Dm fillData.fill(Dyy); fillData.fill(Dzz); - LocalMax=LocalVar=0.0; + double LocalMax = 0.0; + double LocalVar = 0.0; // Execute the next timestep - // f(n+1) = f(n) + dt*sign(1-|grad f|) - for (k=1;k Dxm) Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp; - else Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm; - - if (Dyp > Dym) Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp; - else Dy = Distance(i,j,k) - Dym + 0.5*Dyym; - - if (Dzp > Dzm) Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp; - else Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm; + double Dx, Dy, Dz; + if ( s < 0.0){ + if (Dxp + Dxm > 0.f) + Dx = Dxp*Dxp; + else + Dx = Dxm*Dxm; + if (Dyp + Dym > 0.f) + Dy = Dyp*Dyp; + else + Dy = Dym*Dym; + if (Dzp + Dzm > 0.f) + Dz = Dzp*Dzp; + else + Dz = Dzm*Dzm; } else{ - if (Dxp < Dxm) Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp; - else Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm; - - if (Dyp < Dym) Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp; - else Dy = Distance(i,j,k) - Dym + 0.5*Dyym; - - if (Dzp < Dzm) Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp; - else Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm; + if (Dxp + Dxm < 0.f) + Dx = Dxp*Dxp; + else + Dx = Dxm*Dxm; + if (Dyp + Dym < 0.f) + Dy = Dyp*Dyp; + else + Dy = Dym*Dym; + if (Dzp + Dzm < 0.f) + Dz = Dzp*Dzp; + else + Dz = Dzm*Dzm; } - norm=sqrt(Dx*Dx+Dy*Dy+Dz*Dz); - if (norm > 1.0) norm=1.0; + //Dx = max(Dxp*Dxp,Dxm*Dxm); + //Dy = max(Dyp*Dyp,Dym*Dym); + //Dz = max(Dzp*Dzp,Dzm*Dzm); - Distance(i,j,k) += dt*sign*(1.0 - norm); - LocalVar += dt*sign*(1.0 - norm); + double norm = sqrt(Dx + Dy + Dz); + if (norm > 1.0) + norm = 1.0; - if (fabs(dt*sign*(1.0 - norm)) > LocalMax) - LocalMax = fabs(dt*sign*(1.0 - norm)); + Distance(i,j,k) += dt*s*(1.0 - norm); + LocalVar += dt*s*(1.0 - norm); + + if (fabs(dt*s*(1.0 - norm)) > LocalMax) + LocalMax = fabs(dt*s*(1.0 - norm)); } } } - MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_FLOAT,MPI_SUM,Dm.Comm); - MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_FLOAT,MPI_MAX,Dm.Comm); - GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz; + double GlobalMax; + MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); + GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx()*Dm.nprocy()*Dm.nprocz(); count++; - if (count%50 == 0 && Dm.rank==0 ) - printf(" Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); + + if (count%50 == 0 && Dm.rank()==0 ){ + printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); + fflush(stdout); + } if (fabs(GlobalMax) < 1e-5){ - if (Dm.rank==0) printf(" Exiting with max tolerance of 1e-5 \n"); + if (Dm.rank()==0) + printf("Exiting with max tolerance of 1e-5 \n"); count=timesteps; } } - PROFILE_STOP("Eikonal3D"); return GlobalVar; - } +template float Eikonal( Array&, const Array&, const Domain&, int ); +template double Eikonal( Array&, const Array&, const Domain&, int ); /****************************************************************** * A fast distance calculation * ******************************************************************/ -bool CalcDist3DIteration( Array &Distance, const Domain &Dm ) +bool CalcDist3DIteration( Array &Distance, const Domain & ) { const float sq2 = sqrt(2.0f); const float sq3 = sqrt(3.0f); diff --git a/analysis/eikonal.h b/analysis/eikonal.h index 5157e1c5..d89a7c0e 100644 --- a/analysis/eikonal.h +++ b/analysis/eikonal.h @@ -15,23 +15,20 @@ * Reference: * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 * - * @return Returns the number of cubes in the blob * @param[in/out] Distance Distance function * @param[in] ID Segmentation id * @param[in] DM Domain information * @param[in] timesteps Maximum number of timesteps to process * @return Returns the global variation */ -double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps); - -float Eikonal3D( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps); +template +TYPE Eikonal( Array &Distance, const Array &ID, const Domain &Dm, int timesteps); /*! * @brief Calculate the distance using a simple method * @details This routine calculates the distance using a very simple method working off the segmentation id. * - * @return Returns the number of cubes in the blob * @param[in/out] Distance Distance function * @param[in] ID Segmentation id * @param[in] DM Domain information diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 8f9c05e8..2f76b38c 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -294,7 +294,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, NULL_USE( pBC ); INSIST( db, "Input database is empty" ); char rankString[20]; - sprintf(rankString,"%05d",Dm.rank); + sprintf(rankString,"%05d",Dm.rank()); d_N[0] = Dm.Nx; d_N[1] = Dm.Ny; d_N[2] = Dm.Nz; diff --git a/analysis/uCT.cpp b/analysis/uCT.cpp index efa43d47..4ee879af 100644 --- a/analysis/uCT.cpp +++ b/analysis/uCT.cpp @@ -149,7 +149,7 @@ void solve( const Array& VOL, Array& Mean, Array& ID, fillFloat.fill( Mean ); segment( Mean, ID, 0.01 ); // Compute the distance using the segmented volume - Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx ); + Eikonal( Dist, ID, Dm, ID.size(0)*nprocx ); fillFloat.fill(Dist); smooth( VOL, Dist, 2.0, MultiScaleSmooth, fillFloat ); // Compute non-local mean diff --git a/common/Domain.cpp b/common/Domain.cpp index 13487c51..66b958ac 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -33,7 +33,7 @@ static inline void fgetl( char * str, int num, FILE * stream ) Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC): Nx(0), Ny(0), Nz(0), - Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), + Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), Comm(MPI_COMM_NULL), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), @@ -70,8 +70,8 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, initialize( db ); } Domain::Domain( std::shared_ptr db ): - Nx(0), Ny(0), Nz(0), iproc(0), jproc(0), - Lx(0), Ly(0), Lz(0), Volume(0), rank(0), BoundaryCondition(0), + Nx(0), Ny(0), Nz(0), + Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), Comm(MPI_COMM_NULL), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), @@ -126,13 +126,13 @@ void Domain::initialize( std::shared_ptr db ) rank_info = RankInfoStruct(myrank,nproc[0],nproc[1],nproc[2]); // Fill remaining variables N = Nx*Ny*Nz; - Volume = nx*ny*nx*nprocx*nprocy*nprocz*1.0; + Volume = nx*ny*nx*nproc[0]*nproc[1]*nproc[2]*1.0; id = new char[N]; memset(id,0,N); BoundaryCondition = d_db->getScalar("BC"); int nprocs; MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); + INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!"); } Domain::~Domain() { @@ -199,7 +199,9 @@ void Domain::CommInit(MPI_Comm Communicator) MPI_Comm_dup(Communicator,&Comm); // set up the neighbor ranks - rank_info = RankInfoStruct( rank, nprocx, nprocy, nprocz ); + int myrank; + MPI_Comm_rank( Comm, &myrank ); + rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); MPI_Barrier(Communicator); @@ -319,42 +321,42 @@ void Domain::CommInit(MPI_Comm Communicator) sendBuf_YZ = new int [sendCount_YZ]; sendBuf_XZ = new int [sendCount_XZ]; //...................................................................................... - MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]); - MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]); - MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]); - MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]); - MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]); - MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]); - MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]); - MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]); - MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]); - MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]); - MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]); - MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]); - MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]); - MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]); - MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]); - MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]); - MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]); - MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]); - MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]); - MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]); - MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]); - MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]); - MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]); - MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]); - MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]); - MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]); - MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]); - MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]); - MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]); - MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]); - MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]); - MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]); - MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]); - MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]); - MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]); - MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]); + MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x(),sendtag+0,Communicator,&req1[0]); + MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X(),recvtag+0,Communicator,&req2[0]); + MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X(),sendtag+1,Communicator,&req1[1]); + MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x(),recvtag+1,Communicator,&req2[1]); + MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y(),sendtag+2,Communicator,&req1[2]); + MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y(),recvtag+2,Communicator,&req2[2]); + MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y(),sendtag+3,Communicator,&req1[3]); + MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y(),recvtag+3,Communicator,&req2[3]); + MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z(),sendtag+4,Communicator,&req1[4]); + MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z(),recvtag+4,Communicator,&req2[4]); + MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z(),sendtag+5,Communicator,&req1[5]); + MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z(),recvtag+5,Communicator,&req2[5]); + MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy(),sendtag+6,Communicator,&req1[6]); + MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY(),recvtag+6,Communicator,&req2[6]); + MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY(),sendtag+7,Communicator,&req1[7]); + MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy(),recvtag+7,Communicator,&req2[7]); + MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy(),sendtag+8,Communicator,&req1[8]); + MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY(),recvtag+8,Communicator,&req2[8]); + MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY(),sendtag+9,Communicator,&req1[9]); + MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy(),recvtag+9,Communicator,&req2[9]); + MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz(),sendtag+10,Communicator,&req1[10]); + MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ(),recvtag+10,Communicator,&req2[10]); + MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ(),sendtag+11,Communicator,&req1[11]); + MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz(),recvtag+11,Communicator,&req2[11]); + MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz(),sendtag+12,Communicator,&req1[12]); + MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ(),recvtag+12,Communicator,&req2[12]); + MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ(),sendtag+13,Communicator,&req1[13]); + MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz(),recvtag+13,Communicator,&req2[13]); + MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz(),sendtag+14,Communicator,&req1[14]); + MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ(),recvtag+14,Communicator,&req2[14]); + MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ(),sendtag+15,Communicator,&req1[15]); + MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz(),recvtag+15,Communicator,&req2[15]); + MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz(),sendtag+16,Communicator,&req1[16]); + MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ(),recvtag+16,Communicator,&req2[16]); + MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ(),sendtag+17,Communicator,&req1[17]); + MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz(),recvtag+17,Communicator,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); MPI_Barrier(Communicator); @@ -379,42 +381,42 @@ void Domain::CommInit(MPI_Comm Communicator) recvList_YZ = new int [recvCount_YZ]; recvList_XZ = new int [recvCount_XZ]; //...................................................................................... - MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]); - MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]); - MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]); - MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]); - MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]); - MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]); - MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]); - MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]); - MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]); - MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]); - MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]); - MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]); - MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]); - MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]); - MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]); - MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]); - MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]); - MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]); - MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]); - MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]); - MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]); - MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]); - MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]); - MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]); - MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]); - MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]); - MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]); - MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]); - MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]); - MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]); - MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]); - MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]); - MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]); - MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]); - MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]); - MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x(),sendtag,Communicator,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X(),recvtag,Communicator,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X(),sendtag,Communicator,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x(),recvtag,Communicator,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y(),sendtag,Communicator,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y(),recvtag,Communicator,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y(),sendtag,Communicator,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y(),recvtag,Communicator,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z(),sendtag,Communicator,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z(),recvtag,Communicator,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z(),sendtag,Communicator,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z(),recvtag,Communicator,&req2[5]); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy(),sendtag,Communicator,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY(),recvtag,Communicator,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY(),sendtag,Communicator,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy(),recvtag,Communicator,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy(),sendtag,Communicator,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY(),recvtag,Communicator,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY(),sendtag,Communicator,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy(),recvtag,Communicator,&req2[9]); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz(),sendtag,Communicator,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ(),recvtag,Communicator,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ(),sendtag,Communicator,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz(),recvtag,Communicator,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz(),sendtag,Communicator,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ(),recvtag,Communicator,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ(),sendtag,Communicator,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz(),recvtag,Communicator,&req2[13]); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz(),sendtag,Communicator,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ(),recvtag,Communicator,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ(),sendtag,Communicator,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz(),recvtag,Communicator,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz(),sendtag,Communicator,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ(),recvtag,Communicator,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ(),sendtag,Communicator,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz(),recvtag,Communicator,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); //...................................................................................... @@ -513,7 +515,7 @@ void Domain::AssignComponentLabels(double *phase) vector Label; vector Affinity; // Read the labels - if (rank==0){ + if (rank()==0){ printf("Component labels:\n"); ifstream iFILE("ComponentLabels.csv"); if (iFILE.good()){ @@ -549,7 +551,7 @@ void Domain::AssignComponentLabels(double *phase) // Broadcast the list MPI_Bcast(&NLABELS,1,MPI_INT,0,Comm); - //printf("rank=%i, NLABELS=%i \n ",rank,NLABELS); + //printf("rank=%i, NLABELS=%i \n ",rank(),NLABELS); // Copy into contiguous buffers char *LabelList; @@ -557,11 +559,11 @@ void Domain::AssignComponentLabels(double *phase) LabelList=new char[NLABELS]; AffinityList=new double[NLABELS]; - if (rank==0){ + if (rank()==0){ for (int idx=0; idx < NLABELS; idx++){ VALUE=Label[idx]; AFFINITY=Affinity[idx]; - printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank,idx,VALUE,AFFINITY); + printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank(),idx,VALUE,AFFINITY); LabelList[idx]=VALUE; AffinityList[idx]=AFFINITY; } @@ -579,7 +581,7 @@ void Domain::AssignComponentLabels(double *phase) VALUE=id[n]; // Assign the affinity from the paired list for (int idx=0; idx < NLABELS; idx++){ - //printf("rank=%i, idx=%i, value=%i, %i, \n",rank,idx, VALUE,LabelList[idx]); + //printf("rank=%i, idx=%i, value=%i, %i, \n",rank(),idx, VALUE,LabelList[idx]); if (VALUE == LabelList[idx]){ AFFINITY=AffinityList[idx]; idx = NLABELS; @@ -617,42 +619,42 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh) PackMeshData(sendList_yZ, sendCount_yZ ,sendData_yZ, MeshData); PackMeshData(sendList_YZ, sendCount_YZ ,sendData_YZ, MeshData); //...................................................................................... - MPI_Sendrecv(sendData_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag, - recvData_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag, - recvData_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag, - recvData_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag, - recvData_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag, - recvData_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag, - recvData_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag, - recvData_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag, - recvData_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag, - recvData_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag, - recvData_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag, - recvData_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag, - recvData_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag, - recvData_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag, - recvData_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag, - recvData_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag, - recvData_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag, - recvData_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendData_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag, - recvData_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_x,sendCount_x,MPI_DOUBLE,rank_x(),sendtag, + recvData_X,recvCount_X,MPI_DOUBLE,rank_X(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_X,sendCount_X,MPI_DOUBLE,rank_X(),sendtag, + recvData_x,recvCount_x,MPI_DOUBLE,rank_x(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_y,sendCount_y,MPI_DOUBLE,rank_y(),sendtag, + recvData_Y,recvCount_Y,MPI_DOUBLE,rank_Y(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_Y,sendCount_Y,MPI_DOUBLE,rank_Y(),sendtag, + recvData_y,recvCount_y,MPI_DOUBLE,rank_y(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_z,sendCount_z,MPI_DOUBLE,rank_z(),sendtag, + recvData_Z,recvCount_Z,MPI_DOUBLE,rank_Z(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_Z,sendCount_Z,MPI_DOUBLE,rank_Z(),sendtag, + recvData_z,recvCount_z,MPI_DOUBLE,rank_z(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_xy,sendCount_xy,MPI_DOUBLE,rank_xy(),sendtag, + recvData_XY,recvCount_XY,MPI_DOUBLE,rank_XY(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_XY,sendCount_XY,MPI_DOUBLE,rank_XY(),sendtag, + recvData_xy,recvCount_xy,MPI_DOUBLE,rank_xy(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy(),sendtag, + recvData_xY,recvCount_xY,MPI_DOUBLE,rank_xY(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_xY,sendCount_xY,MPI_DOUBLE,rank_xY(),sendtag, + recvData_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_xz,sendCount_xz,MPI_DOUBLE,rank_xz(),sendtag, + recvData_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ(),sendtag, + recvData_xz,recvCount_xz,MPI_DOUBLE,rank_xz(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz(),sendtag, + recvData_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ(),sendtag, + recvData_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_yz,sendCount_yz,MPI_DOUBLE,rank_yz(),sendtag, + recvData_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ(),sendtag, + recvData_yz,recvCount_yz,MPI_DOUBLE,rank_yz(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz(),sendtag, + recvData_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ(),recvtag,Comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendData_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ(),sendtag, + recvData_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz(),recvtag,Comm,MPI_STATUS_IGNORE); //........................................................................................ UnpackMeshData(recvList_x, recvCount_x ,recvData_x, MeshData); UnpackMeshData(recvList_X, recvCount_X ,recvData_X, MeshData); diff --git a/common/Domain.h b/common/Domain.h index 7563b013..c2578855 100755 --- a/common/Domain.h +++ b/common/Domain.h @@ -64,7 +64,7 @@ private: class Domain{ public: //! Default constructor - Domain( std::shared_ptr db ); + Domain( std::shared_ptr db ); //! Obsolete constructor Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, @@ -80,7 +80,7 @@ public: Domain& operator=( const Domain& ) = delete; //! Destructor - ~Domain(); + ~Domain(); //! Get the database inline std::shared_ptr getDatabase() const { return d_db; } @@ -108,91 +108,91 @@ private: public: // Public variables (need to create accessors instead) double Lx,Ly,Lz,Volume; - int Nx,Ny,Nz,N; - RankInfoStruct rank_info; + int Nx,Ny,Nz,N; + RankInfoStruct rank_info; - MPI_Comm Comm; // MPI Communicator for this domain + MPI_Comm Comm; // MPI Communicator for this domain - int BoundaryCondition; + int BoundaryCondition; - MPI_Group Group; // Group of processors associated with this domain + MPI_Group Group; // Group of processors associated with this domain - //********************************** - // MPI ranks for all 18 neighbors - //********************************** - const int& iproc = rank_info.ix; - const int& jproc = rank_info.jy; - const int& kproc = rank_info.kz; - const int& nprocx = rank_info.nx; - const int& nprocy = rank_info.ny; - const int& nprocz = rank_info.nz; - const int& rank = rank_info.rank[1][1][1]; - const int& rank_X = rank_info.rank[2][1][1]; - const int& rank_x = rank_info.rank[0][1][1]; - const int& rank_Y = rank_info.rank[1][2][1]; - const int& rank_y = rank_info.rank[1][0][1]; - const int& rank_Z = rank_info.rank[1][1][2]; - const int& rank_z = rank_info.rank[1][1][0]; - const int& rank_XY = rank_info.rank[2][2][1]; - const int& rank_xy = rank_info.rank[0][0][1]; - const int& rank_Xy = rank_info.rank[2][0][1]; - const int& rank_xY = rank_info.rank[0][2][1]; - const int& rank_XZ = rank_info.rank[2][1][2]; - const int& rank_xz = rank_info.rank[0][1][0]; - const int& rank_Xz = rank_info.rank[2][1][0]; - const int& rank_xZ = rank_info.rank[0][1][2]; - const int& rank_YZ = rank_info.rank[1][2][2]; - const int& rank_yz = rank_info.rank[1][0][0]; - const int& rank_Yz = rank_info.rank[1][2][0]; - const int& rank_yZ = rank_info.rank[1][0][2]; + //********************************** + // MPI ranks for all 18 neighbors + //********************************** + inline int iproc() const { return rank_info.ix; } + inline int jproc() const { return rank_info.jy; } + inline int kproc() const { return rank_info.kz; } + inline int nprocx() const { return rank_info.nx; } + inline int nprocy() const { return rank_info.ny; } + inline int nprocz() const { return rank_info.nz; } + inline int rank() const { return rank_info.rank[1][1][1]; } + inline int rank_X() const { return rank_info.rank[2][1][1]; } + inline int rank_x() const { return rank_info.rank[0][1][1]; } + inline int rank_Y() const { return rank_info.rank[1][2][1]; } + inline int rank_y() const { return rank_info.rank[1][0][1]; } + inline int rank_Z() const { return rank_info.rank[1][1][2]; } + inline int rank_z() const { return rank_info.rank[1][1][0]; } + inline int rank_XY() const { return rank_info.rank[2][2][1]; } + inline int rank_xy() const { return rank_info.rank[0][0][1]; } + inline int rank_Xy() const { return rank_info.rank[2][0][1]; } + inline int rank_xY() const { return rank_info.rank[0][2][1]; } + inline int rank_XZ() const { return rank_info.rank[2][1][2]; } + inline int rank_xz() const { return rank_info.rank[0][1][0]; } + inline int rank_Xz() const { return rank_info.rank[2][1][0]; } + inline int rank_xZ() const { return rank_info.rank[0][1][2]; } + inline int rank_YZ() const { return rank_info.rank[1][2][2]; } + inline int rank_yz() const { return rank_info.rank[1][0][0]; } + inline int rank_Yz() const { return rank_info.rank[1][2][0]; } + inline int rank_yZ() const { return rank_info.rank[1][0][2]; } - //********************************** - //...................................................................................... - // Get the actual D3Q19 communication counts (based on location of solid phase) - // Discrete velocity set symmetry implies the sendcount = recvcount - //...................................................................................... - int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; - int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; - int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; - //...................................................................................... - int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z; - int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ; - int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ; - //...................................................................................... - int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; - int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; - int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; - //...................................................................................... - int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; - int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; - int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ; - //...................................................................................... - // Solid indicator function - char *id; + //********************************** + //...................................................................................... + // Get the actual D3Q19 communication counts (based on location of solid phase) + // Discrete velocity set symmetry implies the sendcount = recvcount + //...................................................................................... + int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; + int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; + int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; + //...................................................................................... + int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z; + int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ; + int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ; + //...................................................................................... + int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; + int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; + int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; + //...................................................................................... + int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z; + int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ; + int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ; + //...................................................................................... + // Solid indicator function + char *id; - void CommunicateMeshHalo(DoubleArray &Mesh); - void AssignComponentLabels(double *phase); - void CommInit(MPI_Comm comm); - void TestCommInit(MPI_Comm comm); + void CommunicateMeshHalo(DoubleArray &Mesh); + void AssignComponentLabels(double *phase); + void CommInit(MPI_Comm comm); + void TestCommInit(MPI_Comm comm); //void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np); private: - int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; - int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; - int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; - //...................................................................................... - int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; - int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; - int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; - //...................................................................................... - double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; - double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; - double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; - double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; - double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; - double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; + int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; + int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; + int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ; + //...................................................................................... + int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z; + int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ; + int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ; + //...................................................................................... + double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z; + double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ; + double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ; + double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z; + double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ; + double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ; }; diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 6390ad2d..59785a79 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -15,25 +15,25 @@ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ Nz = Dm.Nz; N = Nx*Ny*Nz; next=0; - rank=Dm.rank; - rank_x=Dm.rank_x; - rank_y=Dm.rank_y; - rank_z=Dm.rank_z; - rank_X=Dm.rank_X; - rank_Y=Dm.rank_Y; - rank_Z=Dm.rank_Z; - rank_xy=Dm.rank_xy; - rank_XY=Dm.rank_XY; - rank_xY=Dm.rank_xY; - rank_Xy=Dm.rank_Xy; - rank_xz=Dm.rank_xz; - rank_XZ=Dm.rank_XZ; - rank_xZ=Dm.rank_xZ; - rank_Xz=Dm.rank_Xz; - rank_yz=Dm.rank_yz; - rank_YZ=Dm.rank_YZ; - rank_yZ=Dm.rank_yZ; - rank_Yz=Dm.rank_Yz; + rank=Dm.rank(); + rank_x=Dm.rank_x(); + rank_y=Dm.rank_y(); + rank_z=Dm.rank_z(); + rank_X=Dm.rank_X(); + rank_Y=Dm.rank_Y(); + rank_Z=Dm.rank_Z(); + rank_xy=Dm.rank_xy(); + rank_XY=Dm.rank_XY(); + rank_xY=Dm.rank_xY(); + rank_Xy=Dm.rank_Xy(); + rank_xz=Dm.rank_xz(); + rank_XZ=Dm.rank_XZ(); + rank_xZ=Dm.rank_xZ(); + rank_Xz=Dm.rank_Xz(); + rank_yz=Dm.rank_yz(); + rank_YZ=Dm.rank_YZ(); + rank_yZ=Dm.rank_yZ(); + rank_Yz=Dm.rank_Yz(); sendCount_x=Dm.sendCount_x; sendCount_y=Dm.sendCount_y; sendCount_z=Dm.sendCount_z; @@ -71,12 +71,12 @@ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ recvCount_YZ=Dm.recvCount_YZ; recvCount_XZ=Dm.recvCount_XZ; - iproc = Dm.iproc; - jproc = Dm.jproc; - kproc = Dm.kproc; - nprocx = Dm.nprocx; - nprocy = Dm.nprocy; - nprocz = Dm.nprocz; + iproc = Dm.iproc(); + jproc = Dm.jproc(); + kproc = Dm.kproc(); + nprocx = Dm.nprocx(); + nprocy = Dm.nprocy(); + nprocz = Dm.nprocz(); BoundaryCondition = Dm.BoundaryCondition; //...................................................................................... diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index c219e9b4..be72c0e7 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -277,12 +277,12 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); // Initialize the phase field and variables ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); - if (Dm.kproc==0){ + if (Dm.kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); } - if (Dm.kproc == nprocz-1){ + if (Dm.kproc() == nprocz-1){ ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 1abdaf4d..f1f0ad03 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -48,13 +48,10 @@ int main(int argc, char **argv) Fx = 0; Fy = 0; Fz = 1e-3; //1.f; // 1e-3; - auto FILENAME = argv[1]; - auto db = std::make_shared( FILENAME ); - auto domain_db = db->getDatabase( "Domain" ); // Load inputs if (rank==0) printf("Loading input database \n"); - auto db = std::make_shared(FILENAME); - auto domain_db= db-> getDatabase("Domain"); + auto db = std::make_shared(argv[1]); + auto domain_db = db->getDatabase("Domain"); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index be7d00a4..18405907 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -12,6 +12,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "analysis/eikonal.h" +#include "IO/Writer.h" std::shared_ptr loadInputs( int nprocs ) @@ -37,9 +38,6 @@ int main(int argc, char **argv) MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); { - int i,j,k,n,nn; - int iproc,jproc,kproc; - // Load inputs @@ -47,17 +45,14 @@ int main(int argc, char **argv) int Nx = db->getVector( "n" )[0]; int Ny = db->getVector( "n" )[1]; int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; // Get the rank info Domain Dm(db); - for (k=0;k id(nx,ny,nz); + id.fill(0); - for (k=1;k ID0(id.size()); + ID0.copy( id ); + Array ID(Nx,Ny,Nz); + Array dist1(Nx,Ny,Nz); + Array dist2(Nx,Ny,Nz); + fillHalo fillData(Dm.Comm, Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); + fillData.copy( ID0, ID ); + fillData.copy( Distance, dist1 ); + fillData.copy( TrueDist, dist2 ); + std::vector data(1); + data[0].meshName = "mesh"; + data[0].mesh.reset( new IO::DomainMesh( Dm.rank_info, Nx, Ny, Nz, Dm.Lx, Dm.Ly, Dm.Lz ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "ID", ID ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "Distance", dist1 ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "TrueDist", dist2 ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "error", dist1-dist2 ) ); + IO::initialize( "", "silo", false ); + IO::writeData( "testSegDist", data, MPI_COMM_WORLD ); - MPI_Barrier(comm); } + MPI_Barrier(comm); MPI_Finalize(); return 0; diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index 38b6d45c..5c2fa519 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -103,9 +103,9 @@ int main(int argc, char **argv) n = k*Nx*Ny+j*Nx+i; // global position relative to center - x = Dm.iproc*Nx+i - CX; - y = Dm.jproc*Ny+j - CY; - z = Dm.kproc*Nz+k - CZ; + x = Dm.iproc()*Nx+i - CX; + y = Dm.jproc()*Ny+j - CY; + z = Dm.kproc()*Nz+k - CZ; // Shrink the sphere sizes by two voxels to make sure they don't touch Averages.SDs(i,j,k) = 100.0; @@ -113,12 +113,12 @@ int main(int argc, char **argv) // Single torus Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; // Double torus - /* y = Dm.jproc*Ny+j - CY1; - //z = Dm.kproc*Nz+k - CZ +R1; + /* y = Dm.jproc()*Ny+j - CY1; + //z = Dm.kproc()*Nz+k - CZ +R1; Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; - y = Dm.jproc*Ny+j - CY2; - //z = Dm.kproc*Nz+k - CZ-R1; + y = Dm.jproc()*Ny+j - CY2; + //z = Dm.kproc()*Nz+k - CZ-R1; Averages.Phase(i,j,k) = min(Averages.Phase(i,j,k), sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2); *///.............................................................................. diff --git a/tests/lbpm_captube_pp.cpp b/tests/lbpm_captube_pp.cpp index 348fabd5..ebe5480f 100644 --- a/tests/lbpm_captube_pp.cpp +++ b/tests/lbpm_captube_pp.cpp @@ -140,11 +140,11 @@ int main(int argc, char **argv) if (Averages.SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.kproc*Nz+k 0 && Dm.kproc == 0){ + if (BoundaryCondition > 0 && Dm.kproc() == 0){ for (k=0; k<3; k++){ for (j=0;j 0 && Dm.kproc == nprocz-1){ + if (BoundaryCondition > 0 && Dm.kproc() == nprocz-1){ for (k=Nz-3; k0 ){ - if (Dm.kproc==0){ + if (Dm.kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); } - if (Dm.kproc == nprocz-1){ + if (Dm.kproc() == nprocz-1){ ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index fbf5d554..d17cdad8 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -288,7 +288,7 @@ int main(int argc, char **argv) if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm.kproc == 0){ + if (BoundaryCondition > 0 && Dm.kproc() == 0){ for (int k=0; k<3; k++){ for (int j=0;j 0 && Dm.kproc == nprocz-1){ + if (BoundaryCondition > 0 && Dm.kproc() == nprocz-1){ for (int k=Nz-3; k id(nx,ny,nz); TwoPhase Averages(Dm); // DoubleArray Distance(nx,ny,nz); // DoubleArray Phase(nx,ny,nz); @@ -235,8 +234,8 @@ int main(int argc, char **argv) for (i=0;i Date: Wed, 16 May 2018 14:07:55 -0400 Subject: [PATCH 046/124] uncertain changes --- tests/TestPoiseuille.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 1abdaf4d..6c3f7de9 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -47,14 +47,11 @@ int main(int argc, char **argv) double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); Fx = 0; Fy = 0; Fz = 1e-3; //1.f; // 1e-3; - + // Load inputs + if (rank==0) printf("Loading input database \n"); auto FILENAME = argv[1]; auto db = std::make_shared( FILENAME ); auto domain_db = db->getDatabase( "Domain" ); - // Load inputs - if (rank==0) printf("Loading input database \n"); - auto db = std::make_shared(FILENAME); - auto domain_db= db-> getDatabase("Domain"); int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; From 1e3a77f8a939519ea8e49369cf29ce0e33589353 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:37:21 -0400 Subject: [PATCH 047/124] refactor --- tests/GenerateSphereTest.cpp | 86 ++++++++++++++++-------------------- tests/TestBlobAnalyze.cpp | 6 +-- tests/TestBubbleDFH.cpp | 6 +-- tests/TestColorBubble.cpp | 8 ++-- tests/lbpm_random_pp.cpp | 2 +- tests/lbpm_refine_pp.cpp | 17 ++++--- 6 files changed, 57 insertions(+), 68 deletions(-) diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 6c9edba8..3293205c 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -46,17 +46,14 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny, int nz, int rank, double SW){ - int iproc = Dm.iproc; - int jproc = Dm.jproc; - int kproc = Dm.kproc; int i,j,k,n; - int nprocx=Dm.nprocx; - int nprocy=Dm.nprocy; - int nprocz=Dm.nprocz; double count,countGlobal,totalGlobal; count = 0.f; double maxdist=0.f; double maxdistGlobal; + int nprocx=Dm.nprocx(); + int nprocy=Dm.nprocy(); + int nprocz=Dm.nprocz(); for (int k=0; k Date: Wed, 16 May 2018 14:40:38 -0400 Subject: [PATCH 048/124] Color model --- CMakeLists.txt | 1 + models/ColorModel.cpp | 488 ++++++++++++++++++++++++++++++++++++++++++ models/ColorModel.h | 73 +++++++ 3 files changed, 562 insertions(+) create mode 100644 models/ColorModel.cpp create mode 100644 models/ColorModel.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ec21350..de984c16 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,6 +158,7 @@ IF ( NOT ONLY_BUILD_DOCS ) ADD_PACKAGE_SUBDIRECTORY( analysis ) ADD_PACKAGE_SUBDIRECTORY( IO ) ADD_PACKAGE_SUBDIRECTORY( threadpool ) + ADD_PACKAGE_SUBDIRECTORY( models ) IF ( USE_CUDA ) ADD_PACKAGE_SUBDIRECTORY( gpu ) ELSE() diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp new file mode 100644 index 00000000..3f98dec9 --- /dev/null +++ b/models/ColorModel.cpp @@ -0,0 +1,488 @@ +/* +color lattice boltzmann model + */ + +ScaLBL_ColorModel::ScaLBL_ColorModel(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + color_db = db->getDatabase( "Color" ); + analysis_db = db->getDatabase( "Analysis" ); + + // Color Model parameters + timestepMax = domain_db->getScalar( "timestepMax" ); + tauA = domain_db->getScalar( "tauA" ); + tauB = domain_db->getScalar( "tauB" ); + rhoA = domain_db->getScalar( "rhoA" ); + rhoB = domain_db->getScalar( "rhoB" ); + Fx = domain_db->getVector( "F" )[0]; + Fy = domain_db->getVector( "F" )[1]; + Fz = domain_db->getVector( "F" )[2]; + alpha = domain_db->getScalar( "alpha" ); + beta = domain_db->getScalar( "beta" ); + Restart = domain_db->getScalar( "Restart" ); + din = domain_db->getScalar( "din" ); + dout = domain_db->getScalar( "dout" ); + flux = domain_db->getScalar( "flux" );; + inletA=1.f; + inletB=0.f; + outletA=0.f; + outletB=1.f; + + // Read domain parameters + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + BoundaryCondition = domain_db->getScalar( "BC" ); + Nx = size[0]; + Ny = size[1]; + Nz = size[2]; + Lx = L[0]; + Ly = L[1]; + Lz = L[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) + + // Full domain used for analysis + Domain Dm(domain_db); + for (int i=0; i ( new TwoPhase(Dm) ); + // TwoPhase Averages(Dm); + Dm.CommInit(comm); + + // Mask that excludes the immobile phases + Domain Mask(domain_db); + MPI_Barrier(comm); + + Nx+=2; Ny+=2; Nz += 2; + N = Nx*Ny*Nz; + +} + +ScaLBL_ColorModel::~ScaLBL_ColorModel(){ + +} + +void ScaLBL_ColorModel::ReadInput(){ + //....................................................................... + if (rank == 0) printf("Read input media... \n"); + //....................................................................... + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + // .......... READ THE INPUT FILE ....................................... + id = new char[N]; + double sum, sum_local; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + //........................................................................... + if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; + //....................................................................... + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); + ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); + MPI_Barrier(comm); + if (rank == 0) cout << "Domain set." << endl; + + if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); + sprintf(LocalRankFilename,"ID.%05i",rank); + size_t readID; + FILE *IDFILE = fopen(LocalRankFilename,"rb"); + if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: ID.xxxxx"); + readID=fread(id,1,N,IDFILE); + if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading ID (rank=%i) \n",rank); + fclose(IDFILE); + + // Read restart file + if (Restart == true){ + if (rank==0){ + printf("Reading restart file! \n"); + ifstream restart("Restart.txt"); + if (restart.is_open()){ + restart >> timestep; + printf("Restarting from timestep =%i \n",timestep); + } + else{ + printf("WARNING:No Restart.txt file, setting timestep=0 \n"); + timestep=0; + } + } + MPI_Bcast(×tep,1,MPI_INT,0,comm); + FILE *RESTART = fopen(LocalRestartFile,"rb"); + if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); + readID=fread(id,1,N,RESTART); + if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading Restart (rank=%i) \n",rank); + fclose(RESTART); + + MPI_Barrier(comm); + } +} +void ScaLBL_ColorModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + //....................................................................... + // Compute the media porosity, assign phase labels and solid composition + //....................................................................... + double porosity; + double sum_local=0.0; + Np=0; // number of local pore nodes + //....................................................................... + for (int k=1;k 0){ + sum_local+=1.0; + Np++; + } + } + } + } + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); + porosity = sum*iVol_global; + if (rank==0) printf("Media porosity = %f \n",porosity); + //......................................................... + // If external boundary conditions are applied remove solid + if (BoundaryCondition > 0 && Dm.kproc == 0){ + for (int k=0; k<3; k++){ + for (int j=0;jSDs(n) = max(Averages->SDs(n),1.0*(2.5-k)); + } + } + } + } + if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){ + for (int k=Nz-3; kSDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5)); + } + } + } + } + //......................................................... + // don't perform computations at the eight corners + id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + //......................................................... + + // Initialize communication structures in averaging domain + for (int i=0; iUpdateMeshValues(); // this computes the gradient of distance field (among other things) + // Create the distance stencil + // Compute solid forces based on mean field approximation + double *Dst; + Dst = new double [5*5*5]; + for (int kk=0; kk<5; kk++){ + for (int jj=0; jj<5; jj++){ + for (int ii=0; ii<5; ii++){ + int index = kk*25+jj*5+ii; + Dst[index] = sqrt(double(ii-2)*double(ii-2) + double(jj-2)*double(jj-2)+ double(kk-2)*double(kk-2)); + } + } + } + for (int k=1; k 0)){ + double vec_x = double(ii-2); + double vec_y = double(jj-2); + double vec_z = double(kk-2); + + double ALPHA=PhaseLabel[nn]; + double GAMMA=-2.f; + if (distval > 2.f) ALPHA=0.f; // symmetric cutoff distance + phi_x += ALPHA*exp(GAMMA*distval)*vec_x/distval; + phi_y += ALPHA*exp(GAMMA*distval)*vec_y/distval; + phi_z += ALPHA*exp(GAMMA*distval)*vec_z/distval; + } + } + } + } + Tmp[idx] = phi_x; + Tmp[idx+Np] = phi_y; + Tmp[idx+2*Np] = phi_z; + + /* double d = Averages->SDs(n); + double dx = Averages->SDs_x(n); + double dy = Averages->SDs_y(n); + double dz = Averages->SDs_z(n); + double value=cns*exp(-bns*fabs(d))-cws*exp(-bns*fabs(d)); + + Tmp[idx] = value*dx; + Tmp[idx+Np] = value*dy; + Tmp[idx+2*Np] = value*dz; + */ + } + } + } + } + ScaLBL_CopyToDevice(SolidPotential, Tmp, 3*sizeof(double)*Np); + ScaLBL_DeviceBarrier(); + delete [] Tmp; + delete [] Dst; + + DoubleArray Psx(Nx,Ny,Nz); + DoubleArray Psy(Nx,Ny,Nz); + DoubleArray Psz(Nx,Ny,Nz); + DoubleArray Psnorm(Nx,Ny,Nz); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[0],Psx); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[Np],Psy); + ScaLBL_Comm.RegularLayout(Map,&SolidPotential[2*Np],Psz); + for (int n=0; n 0){ + ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + if (BoundaryCondition == 3){ + ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + if (BoundaryCondition == 4){ + din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + + // *************EVEN TIMESTEP************* + timestep++; + // Compute the Phase indicator field + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); + + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + // Set boundary conditions + if (BoundaryCondition > 0){ + ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + } + if (BoundaryCondition == 3){ + ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + else if (BoundaryCondition == 4){ + din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + } + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************ + MPI_Barrier(comm); + PROFILE_STOP("Update"); + + // Run the analysis + analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); + + } + analysis.finish(); + PROFILE_STOP("Loop"); + PROFILE_SAVE("lbpm_color_simulator",1); + //************************************************************************ + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); + stoptime = MPI_Wtime(); + if (rank==0) printf("-------------------------------------------------------------------\n"); + // Compute the walltime per timestep + cputime = (stoptime - starttime)/timestep; + // Performance obtained from each node + double MLUPS = double(Np)/cputime/1000000; + + if (rank==0) printf("********************************************************\n"); + if (rank==0) printf("CPU time = %f \n", cputime); + if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; + if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); + if (rank==0) printf("********************************************************\n"); + + // ************************************************************************ +} diff --git a/models/ColorModel.h b/models/ColorModel.h new file mode 100644 index 00000000..f68a4ef0 --- /dev/null +++ b/models/ColorModel.h @@ -0,0 +1,73 @@ +/* +Implementation of color lattice boltzmann model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "analysis/TwoPhase.h" +#include "analysis/runAnalysis.h" +#include "common/MPI_Helpers.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" + +class ScaLBL_ColorModel{ +public: + ScaLBL_model_color(); + ~ScaLBL_model_color(); + + // functions in they should be run + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + + bool Restart + int timestep,timestepMax; + double tauA,tauB,rhoA,rhoB,alpha,beta; + double Fx,Fy,Fz,flux; + double din,dout,inletA,inletB,outletA,outletB; + + int Nx,Ny,Nz,N,Np; + int nprocx,nprocy,nprocz,BC; + double Lx,Ly,Lz; + +private: + MPI_Comm comm; + Database db; + Database domain_db; + Database color_db; + Database analysis_db; + + Domain Dm; // this domain is for analysis + Domain Mask; // this domain is for lbm + + ScaLBL_Communicator ScaLBL_Comm; + + std::shared_ptr Averages; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + IntArray Map; + + char *id; + int *NeighborList; + int *dvcMap; + double *fq, *Aq, *Bq; + double *Den, *Phi; + double *SolidPotential; + double *Velocity; + double *Gradient; + double *Pressure; + + +}; + From ff8bd74e970f38322740c1d211ae16eeb41353f8 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:41:25 -0400 Subject: [PATCH 049/124] some dfh change probably deleted a line --- tests/lbpm_dfh_simulator.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index fbf5d554..0d12f814 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -40,7 +40,6 @@ int main(int argc, char **argv) return -1; } auto filename = argv[1]; - auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); auto color_db = db->getDatabase( "Color" ); @@ -103,7 +102,6 @@ int main(int argc, char **argv) int timestep = 6; - flux = 0.f; if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details // Get the rank info @@ -543,14 +541,12 @@ int main(int argc, char **argv) if (rank==0) printf("********************************************************\n"); if (rank==0) printf("No. of timesteps: %i \n", timestepMax); - //.......create and start timer............ double starttime,stoptime,cputime; ScaLBL_DeviceBarrier(); MPI_Barrier(comm); starttime = MPI_Wtime(); //......................................... - //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); From 541dce4f63c0e7936c947b28afefc17b9dd57670 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:43:22 -0400 Subject: [PATCH 050/124] dbg --- models/ColorModel.cpp | 14 ++++++++++++++ models/ColorModel.h | 1 + 2 files changed, 15 insertions(+) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 3f98dec9..b6724213 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1,6 +1,20 @@ /* color lattice boltzmann model */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "analysis/TwoPhase.h" +#include "analysis/runAnalysis.h" +#include "common/MPI_Helpers.h" +#include "ProfilerApp.h" +#include "threadpool/thread_pool.h" ScaLBL_ColorModel::ScaLBL_ColorModel(string filename){ // read the input database diff --git a/models/ColorModel.h b/models/ColorModel.h index f68a4ef0..a1640c5d 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -68,6 +68,7 @@ private: double *Gradient; double *Pressure; + int rank; }; From 0663492d0279d96299c3cd89d8e124e7d5286ae6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:46:11 -0400 Subject: [PATCH 051/124] dbg --- models/ColorModel.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index a1640c5d..ec16d49e 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -18,8 +18,8 @@ Implementation of color lattice boltzmann model class ScaLBL_ColorModel{ public: - ScaLBL_model_color(); - ~ScaLBL_model_color(); + ScaLBL_ColorModel(); + ~ScaLBL_ColorModel(); // functions in they should be run void ReadInput(); From a66ad99380a58f1c6d6f3d96fa3cc339127ad46a Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:50:40 -0400 Subject: [PATCH 052/124] dbg --- models/ColorModel.cpp | 15 +-------------- models/ColorModel.h | 4 ++-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index b6724213..7a3e283b 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -1,20 +1,7 @@ /* color lattice boltzmann model */ -#include -#include -#include -#include -#include -#include -#include - -#include "common/Communication.h" -#include "analysis/TwoPhase.h" -#include "analysis/runAnalysis.h" -#include "common/MPI_Helpers.h" -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" +#include "models/ColorModel.h" ScaLBL_ColorModel::ScaLBL_ColorModel(string filename){ // read the input database diff --git a/models/ColorModel.h b/models/ColorModel.h index ec16d49e..9e28f859 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -18,7 +18,7 @@ Implementation of color lattice boltzmann model class ScaLBL_ColorModel{ public: - ScaLBL_ColorModel(); + ScaLBL_ColorModel(string filename); ~ScaLBL_ColorModel(); // functions in they should be run @@ -27,7 +27,7 @@ public: void Initialize(); void Run(); - bool Restart + bool Restart; int timestep,timestepMax; double tauA,tauB,rhoA,rhoB,alpha,beta; double Fx,Fy,Fz,flux; From 6f2d7283da887df75ee72a8a8d4ef5f8313ee0fc Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 14:54:16 -0400 Subject: [PATCH 053/124] dbg --- models/ColorModel.cpp | 16 +++++++++++----- models/ColorModel.h | 7 ------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 7a3e283b..3e91bfb0 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -3,12 +3,18 @@ color lattice boltzmann model */ #include "models/ColorModel.h" -ScaLBL_ColorModel::ScaLBL_ColorModel(string filename){ +ScaLBL_ColorModel::ScaLBL_ColorModel(){ +} +ScaLBL_ColorModel::~ScaLBL_ColorModel(){ + +} + +void ScaLBL_ColorModel::ReadParams(){ // read the input database - db = std::make_shared( filename ); - domain_db = db->getDatabase( "Domain" ); - color_db = db->getDatabase( "Color" ); - analysis_db = db->getDatabase( "Analysis" ); + auto db = std::make_shared( filename ); + auto domain_db = db->getDatabase( "Domain" ); + auto color_db = db->getDatabase( "Color" ); + auto analysis_db = db->getDatabase( "Analysis" ); // Color Model parameters timestepMax = domain_db->getScalar( "timestepMax" ); diff --git a/models/ColorModel.h b/models/ColorModel.h index 9e28f859..6052ffd0 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -39,16 +39,9 @@ public: private: MPI_Comm comm; - Database db; - Database domain_db; - Database color_db; - Database analysis_db; - Domain Dm; // this domain is for analysis Domain Mask; // this domain is for lbm - ScaLBL_Communicator ScaLBL_Comm; - std::shared_ptr Averages; // filenames From 4d43977f98800949df67e3b3f1598dc390df15da Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 15:08:47 -0400 Subject: [PATCH 054/124] dbg --- models/ColorModel.cpp | 9 +++++++-- models/ColorModel.h | 10 ++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 3e91bfb0..8c2fac4e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -3,13 +3,18 @@ color lattice boltzmann model */ #include "models/ColorModel.h" -ScaLBL_ColorModel::ScaLBL_ColorModel(){ +ScaLBL_ColorModel::ScaLBL_ColorModel(): + Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), + Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), + Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0) +{ + } ScaLBL_ColorModel::~ScaLBL_ColorModel(){ } -void ScaLBL_ColorModel::ReadParams(){ +void ScaLBL_ColorModel::ReadParams(string filename){ // read the input database auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); diff --git a/models/ColorModel.h b/models/ColorModel.h index 6052ffd0..88cce645 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -18,23 +18,25 @@ Implementation of color lattice boltzmann model class ScaLBL_ColorModel{ public: - ScaLBL_ColorModel(string filename); + ScaLBL_ColorModel(); ~ScaLBL_ColorModel(); // functions in they should be run + void ReadParams(string filename); void ReadInput(); void Create(); void Initialize(); void Run(); - bool Restart; + bool Restart,pBC; int timestep,timestepMax; + int BoundaryCondition; double tauA,tauB,rhoA,rhoB,alpha,beta; double Fx,Fy,Fz,flux; double din,dout,inletA,inletB,outletA,outletB; int Nx,Ny,Nz,N,Np; - int nprocx,nprocy,nprocz,BC; + int nprocx,nprocy,nprocz; double Lx,Ly,Lz; private: @@ -61,7 +63,7 @@ private: double *Gradient; double *Pressure; - int rank; + //int rank,nprocs; }; From 4a4125795534fd5bcbcdab64bbd99aa6e13a680c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 15:15:01 -0400 Subject: [PATCH 055/124] dbg --- models/ColorModel.cpp | 6 ++++-- models/ColorModel.h | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 8c2fac4e..ab277ab3 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -59,7 +59,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) // Full domain used for analysis - Domain Dm(domain_db); + Dm = std::shared_ptr(new Domain(domain_db)); for (int i=0; i ( new TwoPhase(Dm) ); // TwoPhase Averages(Dm); @@ -197,7 +197,9 @@ void ScaLBL_ColorModel::Create(){ //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Mask); + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + //Create a second communicator based on the regular data layout //ScaLBL_Communicator ScaLBL_Comm_Regular(Mask); diff --git a/models/ColorModel.h b/models/ColorModel.h index 88cce645..8fcfa299 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -41,9 +41,9 @@ public: private: MPI_Comm comm; - Domain Dm; // this domain is for analysis - Domain Mask; // this domain is for lbm - ScaLBL_Communicator ScaLBL_Comm; + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; std::shared_ptr Averages; // filenames From 16d321b4da949d5ceeb25ba3666ea63a5dba7d21 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 15:16:26 -0400 Subject: [PATCH 056/124] dbg --- models/ColorModel.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index ab277ab3..9205b8a0 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -58,16 +58,15 @@ void ScaLBL_ColorModel::ReadParams(string filename){ if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) - // Full domain used for analysis - Dm = std::shared_ptr(new Domain(domain_db)); - for (int i=0; i ( new TwoPhase(Dm) ); - // TwoPhase Averages(Dm); - Dm.CommInit(comm); + Dm = std::shared_ptr(new Domain(domain_db)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db)); // mask domain removes immobile phases + for (int i=0; i ( new TwoPhase(Dm) ); // TwoPhase analysis object - // Mask that excludes the immobile phases - Domain Mask(domain_db); MPI_Barrier(comm); + Dm.CommInit(comm); + MPI_Barrier(comm); + Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; From 0573438872e80b27d9f364c283055726fa3b869a Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 15:37:07 -0400 Subject: [PATCH 057/124] dbg --- models/ColorModel.cpp | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 9205b8a0..8f862f21 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -58,19 +58,17 @@ void ScaLBL_ColorModel::ReadParams(string filename){ if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) - Dm = std::shared_ptr(new Domain(domain_db)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db)); // mask domain removes immobile phases - for (int i=0; i ( new TwoPhase(Dm) ); // TwoPhase analysis object - - MPI_Barrier(comm); - Dm.CommInit(comm); - MPI_Barrier(comm); - + Dm = std::shared_ptr(new Domain(domain_db)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db)); // mask domain removes immobile phases Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; - + for (int i=0; iid[i] = 1; // initialize this way + Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + + MPI_Barrier(comm); + Dm->CommInit(comm); + MPI_Barrier(comm); } ScaLBL_ColorModel::~ScaLBL_ColorModel(){ @@ -158,7 +156,7 @@ void ScaLBL_ColorModel::Create(){ if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm.kproc == 0){ + if (BoundaryCondition > 0 && Dm->kproc() == 0){ for (int k=0; k<3; k++){ for (int j=0;j 0 && Dm.kproc == nprocz-1){ + if (BoundaryCondition > 0 && Dm->kproc() == nprocz-1){ for (int k=Nz-3; kNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; + Mask->CommInit(comm); double *PhaseLabel; PhaseLabel = new double[N]; - Mask.AssignComponentLabels(PhaseLabel); + Mask->AssignComponentLabels(PhaseLabel); //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); @@ -206,7 +204,7 @@ void ScaLBL_ColorModel::Create(){ if (rank==0) printf ("Set up memory efficient layout \n"); Map.resize(Nx,Ny,Nz); Map.fill(0); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask.id,Np); + Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); MPI_Barrier(comm); //........................................................................... @@ -299,7 +297,7 @@ void ScaLBL_ColorModel::Initialize(){ if (!(idk < Nz)) idk=Nz-1; int nn = idk*Nx*Ny + idj*Nx + idi; - if (!(Mask.id[nn] > 0)){ + if (!(Mask->id[nn] > 0)){ double vec_x = double(ii-2); double vec_y = double(jj-2); double vec_z = double(kk-2); @@ -358,7 +356,7 @@ void ScaLBL_ColorModel::Initialize(){ int idx=Map(i,j,k); int n = k*Nx*Ny+j*Nx+i; if (!(idx < 0)){ - if (Mask.id[n] == 1) + if (Mask->id[n] == 1) PhaseLabel[idx] = 1.0; else { PhaseLabel[idx] = -1.0; From 0b8c5f10530fa45b51a84584b8fa3ea402e9b472 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 18:08:14 -0400 Subject: [PATCH 058/124] dbg --- models/ColorModel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 8f862f21..6dbf2ac6 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -64,7 +64,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; for (int i=0; iid[i] = 1; // initialize this way - Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new TwoPhase(Dm.get()) ); // TwoPhase analysis object MPI_Barrier(comm); Dm->CommInit(comm); @@ -195,7 +195,7 @@ void ScaLBL_ColorModel::Create(){ if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask.get())); //Create a second communicator based on the regular data layout //ScaLBL_Communicator ScaLBL_Comm_Regular(Mask); From 5060f593238a7f817fa2f8a196a932f7b1eb1742 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 20:54:37 -0400 Subject: [PATCH 059/124] down the rabbit hole --- analysis/TwoPhase.cpp | 242 ++++++++++++++++---------------- analysis/TwoPhase.h | 4 +- analysis/runAnalysis.cpp | 24 ++-- analysis/runAnalysis.h | 4 +- common/ScaLBL.cpp | 294 +++++++++++++++++++-------------------- common/ScaLBL.h | 2 +- models/ColorModel.cpp | 133 ++++++++++-------- tests/CMakeLists.txt | 2 +- 8 files changed, 359 insertions(+), 346 deletions(-) diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index 492c295e..4028be50 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -54,7 +54,7 @@ #define PI 3.14159265359 // Constructor -TwoPhase::TwoPhase(Domain &dm): +TwoPhase::TwoPhase(Domain *dm): n_nw_pts(0), n_ns_pts(0), n_ws_pts(0), n_nws_pts(0), n_local_sol_pts(0), n_local_nws_pts(0), n_nw_tris(0), n_ns_tris(0), n_ws_tris(0), n_nws_seg(0), n_local_sol_tris(0), nc(0), kstart(0), kfinish(0), fluid_isovalue(0), solid_isovalue(0), Volume(0), @@ -69,8 +69,8 @@ TwoPhase::TwoPhase(Domain &dm): trRwn(0), trRwn_global(0), nwp_volume_global(0), wp_volume_global(0), As_global(0), wwndnw_global(0), wwnsdnwn_global(0), Jwnwwndnw_global(0), dEs(0), dAwn(0), dAns(0) { - Nx=dm.Nx; Ny=dm.Ny; Nz=dm.Nz; - Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm.nprocx()*Dm.nprocy()*Dm.nprocz()*1.0; + Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz; + Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0; TempID = new char[Nx*Ny*Nz]; @@ -135,7 +135,7 @@ TwoPhase::TwoPhase(Domain &dm): Gns_global.resize(6); Gws_global.resize(6); //......................................... - if (Dm.rank()==0){ + if (Dm->rank()==0){ TIMELOG = fopen("timelog.tcat","a+"); if (fseek(TIMELOG,0,SEEK_SET) == fseek(TIMELOG,0,SEEK_CUR)) { @@ -165,7 +165,7 @@ TwoPhase::TwoPhase(Domain &dm): } else{ char LocalRankString[8]; - sprintf(LocalRankString,"%05d",Dm.rank()); + sprintf(LocalRankString,"%05d",Dm->rank()); char LocalRankFilename[40]; sprintf(LocalRankFilename,"%s%s","timelog.tcat.",LocalRankString); TIMELOG = fopen(LocalRankFilename,"a+"); @@ -259,7 +259,7 @@ void TwoPhase::ComputeDelPhi() int i,j,k; double fx,fy,fz; - Dm.CommunicateMeshHalo(Phase); + Dm->CommunicateMeshHalo(Phase); for (k=1; kBoundaryCondition !=0 && Dm->kproc==0) kstart = 4; + if (Dm->BoundaryCondition !=0 && Dm->kproc==Dm->nprocz-1) kfinish = Nz-4; nc=0; for (k=kstart; kCommunicateMeshHalo(SDs); //........................................................................... // Gradient of the Signed Distance function //........................................................................... pmmc_MeshGradient(SDs,SDs_x,SDs_y,SDs_z,Nx,Ny,Nz); //........................................................................... - Dm.CommunicateMeshHalo(SDs_x); + Dm->CommunicateMeshHalo(SDs_x); //........................................................................... - Dm.CommunicateMeshHalo(SDs_y); + Dm->CommunicateMeshHalo(SDs_y); //........................................................................... - Dm.CommunicateMeshHalo(SDs_z); + Dm->CommunicateMeshHalo(SDs_z); //........................................................................... } @@ -346,27 +346,27 @@ void TwoPhase::UpdateMeshValues() { int i,j,k,n; //........................................................................... - Dm.CommunicateMeshHalo(SDn); + Dm->CommunicateMeshHalo(SDn); //........................................................................... // Compute the gradients of the phase indicator and signed distance fields pmmc_MeshGradient(SDn,SDn_x,SDn_y,SDn_z,Nx,Ny,Nz); //........................................................................... // Gradient of the phase indicator field //........................................................................... - Dm.CommunicateMeshHalo(SDn_x); + Dm->CommunicateMeshHalo(SDn_x); //........................................................................... - Dm.CommunicateMeshHalo(SDn_y); + Dm->CommunicateMeshHalo(SDn_y); //........................................................................... - Dm.CommunicateMeshHalo(SDn_z); + Dm->CommunicateMeshHalo(SDn_z); //........................................................................... - Dm.CommunicateMeshHalo(SDs); + Dm->CommunicateMeshHalo(SDs); pmmc_MeshGradient(SDs,SDs_x,SDs_y,SDs_z,Nx,Ny,Nz); //........................................................................... - Dm.CommunicateMeshHalo(SDs_x); + Dm->CommunicateMeshHalo(SDs_x); //........................................................................... - Dm.CommunicateMeshHalo(SDs_y); + Dm->CommunicateMeshHalo(SDs_y); //........................................................................... - Dm.CommunicateMeshHalo(SDs_z); + Dm->CommunicateMeshHalo(SDs_z); //........................................................................... // Compute the mesh curvature of the phase indicator field pmmc_MeshCurvature(SDn, MeanCurvature, GaussCurvature, Nx, Ny, Nz); @@ -375,26 +375,26 @@ void TwoPhase::UpdateMeshValues() // Map Phase_tplus and Phase_tminus for (int n=0; nCommunicateMeshHalo(Press); //........................................................................... - Dm.CommunicateMeshHalo(Vel_x); + Dm->CommunicateMeshHalo(Vel_x); //........................................................................... - Dm.CommunicateMeshHalo(Vel_y); + Dm->CommunicateMeshHalo(Vel_y); //........................................................................... - Dm.CommunicateMeshHalo(Vel_z); + Dm->CommunicateMeshHalo(Vel_z); //........................................................................... - Dm.CommunicateMeshHalo(MeanCurvature); + Dm->CommunicateMeshHalo(MeanCurvature); //........................................................................... - Dm.CommunicateMeshHalo(GaussCurvature); + Dm->CommunicateMeshHalo(GaussCurvature); //........................................................................... - Dm.CommunicateMeshHalo(DelPhi); + Dm->CommunicateMeshHalo(DelPhi); //........................................................................... // Initializing the blob ID for (k=0; kid[n] == 0){ // Solid phase PhaseID(i,j,k) = 0; } @@ -418,8 +418,8 @@ void TwoPhase::ComputeLocal() // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; - if (Dm.BoundaryCondition > 0 && Dm.kproc() == 0) kmin=4; - if (Dm.BoundaryCondition > 0 && Dm.kproc() == Dm.nprocz()-1) kmax=Nz-4; + if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; + if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; for (k=kmin; kid[n] != 0 ){ // 1-D index for this cube corner // compute the norm of the gradient of the phase indicator field // Compute the non-wetting phase volume contribution @@ -551,7 +551,7 @@ void TwoPhase::AssignComponentLabels() int LabelNWP=1; int LabelWP=2; // NOTE: labeling the wetting phase components is tricky! One sandstone media had over 800,000 components - // NumberComponents_WP = ComputeGlobalPhaseComponent(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.rank_info,PhaseID,LabelWP,Label_WP); + // NumberComponents_WP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelWP,Label_WP); // treat all wetting phase is connected NumberComponents_WP=1; for (int k=0; kNx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelNWP,Label_NWP); + NumberComponents_NWP = ComputeGlobalBlobIDs(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,SDs,SDn,solid_isovalue,fluid_isovalue,Label_NWP,Dm->Comm); } void TwoPhase::ComponentAverages() @@ -585,15 +585,15 @@ void TwoPhase::ComponentAverages() ComponentAverages_WP.fill(0.0); ComponentAverages_NWP.fill(0.0); - if (Dm.rank()==0){ + if (Dm->rank()==0){ printf("Number of wetting phase components is %i \n",NumberComponents_WP); printf("Number of non-wetting phase components is %i \n",NumberComponents_NWP); } // If external boundary conditions are set, do not average over the inlet kmin=1; kmax=Nz-1; - if (Dm.BoundaryCondition > 0 && Dm.kproc() == 0) kmin=4; - if (Dm.BoundaryCondition > 0 && Dm.kproc() == Dm.nprocz()-1) kmax=Nz-4; + if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4; + if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4; for (k=kmin; kid[n] != 0 ){ // 1-D index for this cube corner // compute the norm of the gradient of the phase indicator field // Compute the non-wetting phase volume contribution @@ -636,9 +636,9 @@ void TwoPhase::ComponentAverages() ComponentAverages_NWP(VY,LabelNWP) += 0.125*Vel_y(n); ComponentAverages_NWP(VZ,LabelNWP) += 0.125*Vel_z(n); // center of mass - ComponentAverages_NWP(CMX,LabelNWP) += 0.125*(i+cube[p][0]+Dm.iproc()*Nx); - ComponentAverages_NWP(CMY,LabelNWP) += 0.125*(j+cube[p][1]+Dm.jproc()*Ny); - ComponentAverages_NWP(CMZ,LabelNWP) += 0.125*(k+cube[p][2]+Dm.kproc()*Nz); + ComponentAverages_NWP(CMX,LabelNWP) += 0.125*(i+cube[p][0]+Dm->iproc()*Nx); + ComponentAverages_NWP(CMY,LabelNWP) += 0.125*(j+cube[p][1]+Dm->jproc()*Ny); + ComponentAverages_NWP(CMZ,LabelNWP) += 0.125*(k+cube[p][2]+Dm->kproc()*Nz); // twice the kinetic energy ComponentAverages_NWP(VSQ,LabelNWP) += 0.125*(Vel_x(n)*Vel_x(n)+Vel_y(n)*Vel_y(n)+Vel_z(n)*Vel_z(n)); @@ -656,9 +656,9 @@ void TwoPhase::ComponentAverages() ComponentAverages_WP(VY,LabelWP)+= 0.125*Vel_y(n); ComponentAverages_WP(VZ,LabelWP) += 0.125*Vel_z(n); // Center of mass - ComponentAverages_WP(CMX,LabelWP) += 0.125*(i+cube[p][0]+Dm.iproc()*Nx); - ComponentAverages_WP(CMY,LabelWP) += 0.125*(j+cube[p][1]+Dm.jproc()*Ny); - ComponentAverages_WP(CMZ,LabelWP) += 0.125*(k+cube[p][2]+Dm.kproc()*Nz); + ComponentAverages_WP(CMX,LabelWP) += 0.125*(i+cube[p][0]+Dm->iproc()*Nx); + ComponentAverages_WP(CMY,LabelWP) += 0.125*(j+cube[p][1]+Dm->jproc()*Ny); + ComponentAverages_WP(CMZ,LabelWP) += 0.125*(k+cube[p][2]+Dm->kproc()*Nz); // twice the kinetic energy ComponentAverages_WP(VSQ,LabelWP) += 0.125*(Vel_x(n)*Vel_x(n)+Vel_y(n)*Vel_y(n)+Vel_z(n)*Vel_z(n)); @@ -802,24 +802,24 @@ void TwoPhase::ComponentAverages() } } - MPI_Barrier(Dm.Comm); - if (Dm.rank()==0){ + MPI_Barrier(Dm->Comm); + if (Dm->rank()==0){ printf("Component averages computed locally -- reducing result... \n"); } // Globally reduce the non-wetting phase averages RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP); /* for (int b=0; bComm); + MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); for (int idx=0; idxComm); + MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm->Comm); + // MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); - if (Dm.rank()==0){ + if (Dm->rank()==0){ printf("rescaling... \n"); } @@ -907,15 +907,15 @@ void TwoPhase::ComponentAverages() } } - if (Dm.rank()==0){ + if (Dm->rank()==0){ printf("reduce WP averages... \n"); } // reduce the wetting phase averages for (int b=0; bComm); +// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm); for (int idx=0; idxiproc()*(Nx-2); + A.y += 1.0*Dm->jproc()*(Nx-2); + A.z += 1.0*Dm->kproc()*(Nx-2); + B.x += 1.0*Dm->iproc()*(Nx-2); + B.y += 1.0*Dm->jproc()*(Nx-2); + B.z += 1.0*Dm->kproc()*(Nx-2); + C.x += 1.0*Dm->iproc()*(Nx-2); + C.y += 1.0*Dm->jproc()*(Nx-2); + C.z += 1.0*Dm->kproc()*(Nx-2); wn_mesh->A.push_back(A); wn_mesh->B.push_back(B); wn_mesh->C.push_back(C); @@ -1070,15 +1070,15 @@ void TwoPhase::WriteSurfaces(int logcount) B = ws_pts(ws_tris(1,r)); C = ws_pts(ws_tris(2,r)); // Remap the points - A.x += 1.0*Dm.iproc()*(Nx-2); - A.y += 1.0*Dm.jproc()*(Nx-2); - A.z += 1.0*Dm.kproc()*(Nx-2); - B.x += 1.0*Dm.iproc()*(Nx-2); - B.y += 1.0*Dm.jproc()*(Nx-2); - B.z += 1.0*Dm.kproc()*(Nx-2); - C.x += 1.0*Dm.iproc()*(Nx-2); - C.y += 1.0*Dm.jproc()*(Nx-2); - C.z += 1.0*Dm.kproc()*(Nx-2); + A.x += 1.0*Dm->iproc()*(Nx-2); + A.y += 1.0*Dm->jproc()*(Nx-2); + A.z += 1.0*Dm->kproc()*(Nx-2); + B.x += 1.0*Dm->iproc()*(Nx-2); + B.y += 1.0*Dm->jproc()*(Nx-2); + B.z += 1.0*Dm->kproc()*(Nx-2); + C.x += 1.0*Dm->iproc()*(Nx-2); + C.y += 1.0*Dm->jproc()*(Nx-2); + C.z += 1.0*Dm->kproc()*(Nx-2); ws_mesh->A.push_back(A); ws_mesh->B.push_back(B); ws_mesh->C.push_back(C); @@ -1088,15 +1088,15 @@ void TwoPhase::WriteSurfaces(int logcount) B = ns_pts(ns_tris(1,r)); C = ns_pts(ns_tris(2,r)); // Remap the points - A.x += 1.0*Dm.iproc()*(Nx-2); - A.y += 1.0*Dm.jproc()*(Nx-2); - A.z += 1.0*Dm.kproc()*(Nx-2); - B.x += 1.0*Dm.iproc()*(Nx-2); - B.y += 1.0*Dm.jproc()*(Nx-2); - B.z += 1.0*Dm.kproc()*(Nx-2); - C.x += 1.0*Dm.iproc()*(Nx-2); - C.y += 1.0*Dm.jproc()*(Nx-2); - C.z += 1.0*Dm.kproc()*(Nx-2); + A.x += 1.0*Dm->iproc()*(Nx-2); + A.y += 1.0*Dm->jproc()*(Nx-2); + A.z += 1.0*Dm->kproc()*(Nx-2); + B.x += 1.0*Dm->iproc()*(Nx-2); + B.y += 1.0*Dm->jproc()*(Nx-2); + B.z += 1.0*Dm->kproc()*(Nx-2); + C.x += 1.0*Dm->iproc()*(Nx-2); + C.y += 1.0*Dm->jproc()*(Nx-2); + C.z += 1.0*Dm->kproc()*(Nx-2); ns_mesh->A.push_back(A); ns_mesh->B.push_back(B); ns_mesh->C.push_back(C); @@ -1112,7 +1112,7 @@ void TwoPhase::WriteSurfaces(int logcount) meshData[1].mesh = ws_mesh; meshData[2].meshName = "ns-tris"; meshData[2].mesh = ns_mesh; - IO::writeData( logcount, meshData, Dm.Comm ); + IO::writeData( logcount, meshData, Dm->Comm ); } @@ -1121,43 +1121,43 @@ void TwoPhase::Reduce() int i; double iVol_global=1.0/Volume; //........................................................................... - MPI_Barrier(Dm.Comm); - MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Barrier(Dm->Comm); + MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); // Phase averages - MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); + MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm); - MPI_Barrier(Dm.Comm); + MPI_Barrier(Dm->Comm); // Normalize the phase averages // (density of both components = 1.0) @@ -1224,7 +1224,7 @@ void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT) void TwoPhase::PrintAll(int timestep) { - if (Dm.rank()==0){ + if (Dm->rank()==0){ fprintf(TIMELOG,"%i %.5g ",timestep,dEs); // change in surface energy fprintf(TIMELOG,"%.5g %.5g %.5g ",sat_w,paw_global,pan_global); // saturation and pressure fprintf(TIMELOG,"%.5g %.5g %.5g ",awn_global,ans_global,aws_global); // interfacial areas @@ -1277,7 +1277,7 @@ void TwoPhase::PrintAll(int timestep) void TwoPhase::PrintComponents(int timestep) { - if (Dm.rank()==0){ + if (Dm->rank()==0){ printf("PRINT %i COMPONENT AVEREAGES: time = %i \n",(int)ComponentAverages_NWP.size(1),timestep); for (int b=0; b 0.0){ diff --git a/analysis/TwoPhase.h b/analysis/TwoPhase.h index db0537ac..ee764348 100644 --- a/analysis/TwoPhase.h +++ b/analysis/TwoPhase.h @@ -64,7 +64,7 @@ class TwoPhase{ public: //........................................................................... - Domain& Dm; + Domain* Dm; int NumberComponents_WP,NumberComponents_NWP; //........................................................................... // Averaging variables @@ -145,7 +145,7 @@ public: DoubleArray ComponentAverages_WP; DoubleArray ComponentAverages_NWP; //........................................................................... - TwoPhase(Domain &dm); + TwoPhase(Domain *dm); ~TwoPhase(); void Initialize(); // void SetupCubes(Domain &Dm); diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 2f76b38c..79208329 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -282,22 +282,22 @@ runAnalysis::commWrapper runAnalysis::getComm( ) * Constructor/Destructors * ******************************************************************/ runAnalysis::runAnalysis( std::shared_ptr db, - const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm, + const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain* Dm, int Np, bool pBC, double beta, IntArray Map ): d_Np( Np ), d_beta( beta ), - d_ScaLBL_Comm( ScaLBL_Comm ), d_rank_info( rank_info ), d_Map( Map ), - d_fillData(Dm.Comm,Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,1,1,1,0,1) + d_ScaLBL_Comm( ScaLBL_Comm ), + d_fillData(Dm->Comm,Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,1,1,1,0,1) { NULL_USE( pBC ); INSIST( db, "Input database is empty" ); char rankString[20]; - sprintf(rankString,"%05d",Dm.rank()); - d_N[0] = Dm.Nx; - d_N[1] = Dm.Ny; - d_N[2] = Dm.Nz; + sprintf(rankString,"%05d",Dm->rank()); + d_N[0] = Dm->Nx; + d_N[1] = Dm->Ny; + d_N[2] = Dm->Nz; d_restart_interval = db->getScalar( "restart_interval" ); d_analysis_interval = db->getScalar( "analysis_interval" ); d_blobid_interval = db->getScalar( "blobid_interval" ); @@ -309,7 +309,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, // Create the MeshDataStruct d_meshData.resize(1); d_meshData[0].meshName = "domain"; - d_meshData[0].mesh = std::make_shared( Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,Dm.Lx,Dm.Ly,Dm.Lz ); + d_meshData[0].mesh = std::make_shared( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz ); auto PhaseVar = std::make_shared(); auto PressVar = std::make_shared(); auto SignDistVar = std::make_shared(); @@ -317,22 +317,22 @@ runAnalysis::runAnalysis( std::shared_ptr db, PhaseVar->name = "phase"; PhaseVar->type = IO::VariableType::VolumeVariable; PhaseVar->dim = 1; - PhaseVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); + PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); d_meshData[0].vars.push_back(PhaseVar); PressVar->name = "Pressure"; PressVar->type = IO::VariableType::VolumeVariable; PressVar->dim = 1; - PressVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); + PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); d_meshData[0].vars.push_back(PressVar); SignDistVar->name = "SignDist"; SignDistVar->type = IO::VariableType::VolumeVariable; SignDistVar->dim = 1; - SignDistVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); + SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); d_meshData[0].vars.push_back(SignDistVar); BlobIDVar->name = "BlobID"; BlobIDVar->type = IO::VariableType::VolumeVariable; BlobIDVar->dim = 1; - BlobIDVar->data.resize(Dm.Nx-2,Dm.Ny-2,Dm.Nz-2); + BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2); d_meshData[0].vars.push_back(BlobIDVar); // Initialize the comms MPI_Comm_dup(MPI_COMM_WORLD,&d_comm); diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index b3bd83b5..b64256ec 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -24,13 +24,13 @@ public: //! Constructor runAnalysis( std::shared_ptr db, const RankInfoStruct& rank_info, - const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm, int Np, bool pBC, double beta, IntArray Map ); + const ScaLBL_Communicator &ScaLBL_Comm, const Domain* dm, int Np, bool pBC, double beta, IntArray Map ); //! Destructor ~runAnalysis(); //! Run the next analysis - void run( int timestep, TwoPhase& Averages, const double *Phi, + void run( int timestep, TwoPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den ); //! Finish all active analysis diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 59785a79..92c11766 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1,83 +1,83 @@ #include "common/ScaLBL.h" -ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ +ScaLBL_Communicator::ScaLBL_Communicator(Domain *Dm){ //...................................................................................... Lock=false; // unlock the communicator //...................................................................................... // Create a separate copy of the communicator for the device - //MPI_Comm_group(Dm.Comm,&Group); - //MPI_Comm_create(Dm.Comm,Group,&MPI_COMM_SCALBL); - MPI_Comm_dup(Dm.Comm,&MPI_COMM_SCALBL); + //MPI_Comm_group(Dm->Comm,&Group); + //MPI_Comm_create(Dm->Comm,Group,&MPI_COMM_SCALBL); + MPI_Comm_dup(Dm->Comm,&MPI_COMM_SCALBL); //...................................................................................... // Copy the domain size and communication information directly from Dm - Nx = Dm.Nx; - Ny = Dm.Ny; - Nz = Dm.Nz; + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; N = Nx*Ny*Nz; next=0; - rank=Dm.rank(); - rank_x=Dm.rank_x(); - rank_y=Dm.rank_y(); - rank_z=Dm.rank_z(); - rank_X=Dm.rank_X(); - rank_Y=Dm.rank_Y(); - rank_Z=Dm.rank_Z(); - rank_xy=Dm.rank_xy(); - rank_XY=Dm.rank_XY(); - rank_xY=Dm.rank_xY(); - rank_Xy=Dm.rank_Xy(); - rank_xz=Dm.rank_xz(); - rank_XZ=Dm.rank_XZ(); - rank_xZ=Dm.rank_xZ(); - rank_Xz=Dm.rank_Xz(); - rank_yz=Dm.rank_yz(); - rank_YZ=Dm.rank_YZ(); - rank_yZ=Dm.rank_yZ(); - rank_Yz=Dm.rank_Yz(); - sendCount_x=Dm.sendCount_x; - sendCount_y=Dm.sendCount_y; - sendCount_z=Dm.sendCount_z; - sendCount_X=Dm.sendCount_X; - sendCount_Y=Dm.sendCount_Y; - sendCount_Z=Dm.sendCount_Z; - sendCount_xy=Dm.sendCount_xy; - sendCount_yz=Dm.sendCount_yz; - sendCount_xz=Dm.sendCount_xz; - sendCount_Xy=Dm.sendCount_Xy; - sendCount_Yz=Dm.sendCount_Yz; - sendCount_xZ=Dm.sendCount_xZ; - sendCount_xY=Dm.sendCount_xY; - sendCount_yZ=Dm.sendCount_yZ; - sendCount_Xz=Dm.sendCount_Xz; - sendCount_XY=Dm.sendCount_XY; - sendCount_YZ=Dm.sendCount_YZ; - sendCount_XZ=Dm.sendCount_XZ; - recvCount_x=Dm.recvCount_x; - recvCount_y=Dm.recvCount_y; - recvCount_z=Dm.recvCount_z; - recvCount_X=Dm.recvCount_X; - recvCount_Y=Dm.recvCount_Y; - recvCount_Z=Dm.recvCount_Z; - recvCount_xy=Dm.recvCount_xy; - recvCount_yz=Dm.recvCount_yz; - recvCount_xz=Dm.recvCount_xz; - recvCount_Xy=Dm.recvCount_Xy; - recvCount_Yz=Dm.recvCount_Yz; - recvCount_xZ=Dm.recvCount_xZ; - recvCount_xY=Dm.recvCount_xY; - recvCount_yZ=Dm.recvCount_yZ; - recvCount_Xz=Dm.recvCount_Xz; - recvCount_XY=Dm.recvCount_XY; - recvCount_YZ=Dm.recvCount_YZ; - recvCount_XZ=Dm.recvCount_XZ; + rank=Dm->rank(); + rank_x=Dm->rank_x(); + rank_y=Dm->rank_y(); + rank_z=Dm->rank_z(); + rank_X=Dm->rank_X(); + rank_Y=Dm->rank_Y(); + rank_Z=Dm->rank_Z(); + rank_xy=Dm->rank_xy(); + rank_XY=Dm->rank_XY(); + rank_xY=Dm->rank_xY(); + rank_Xy=Dm->rank_Xy(); + rank_xz=Dm->rank_xz(); + rank_XZ=Dm->rank_XZ(); + rank_xZ=Dm->rank_xZ(); + rank_Xz=Dm->rank_Xz(); + rank_yz=Dm->rank_yz(); + rank_YZ=Dm->rank_YZ(); + rank_yZ=Dm->rank_yZ(); + rank_Yz=Dm->rank_Yz(); + sendCount_x=Dm->sendCount_x; + sendCount_y=Dm->sendCount_y; + sendCount_z=Dm->sendCount_z; + sendCount_X=Dm->sendCount_X; + sendCount_Y=Dm->sendCount_Y; + sendCount_Z=Dm->sendCount_Z; + sendCount_xy=Dm->sendCount_xy; + sendCount_yz=Dm->sendCount_yz; + sendCount_xz=Dm->sendCount_xz; + sendCount_Xy=Dm->sendCount_Xy; + sendCount_Yz=Dm->sendCount_Yz; + sendCount_xZ=Dm->sendCount_xZ; + sendCount_xY=Dm->sendCount_xY; + sendCount_yZ=Dm->sendCount_yZ; + sendCount_Xz=Dm->sendCount_Xz; + sendCount_XY=Dm->sendCount_XY; + sendCount_YZ=Dm->sendCount_YZ; + sendCount_XZ=Dm->sendCount_XZ; + recvCount_x=Dm->recvCount_x; + recvCount_y=Dm->recvCount_y; + recvCount_z=Dm->recvCount_z; + recvCount_X=Dm->recvCount_X; + recvCount_Y=Dm->recvCount_Y; + recvCount_Z=Dm->recvCount_Z; + recvCount_xy=Dm->recvCount_xy; + recvCount_yz=Dm->recvCount_yz; + recvCount_xz=Dm->recvCount_xz; + recvCount_Xy=Dm->recvCount_Xy; + recvCount_Yz=Dm->recvCount_Yz; + recvCount_xZ=Dm->recvCount_xZ; + recvCount_xY=Dm->recvCount_xY; + recvCount_yZ=Dm->recvCount_yZ; + recvCount_Xz=Dm->recvCount_Xz; + recvCount_XY=Dm->recvCount_XY; + recvCount_YZ=Dm->recvCount_YZ; + recvCount_XZ=Dm->recvCount_XZ; - iproc = Dm.iproc(); - jproc = Dm.jproc(); - kproc = Dm.kproc(); - nprocx = Dm.nprocx(); - nprocy = Dm.nprocy(); - nprocz = Dm.nprocz(); - BoundaryCondition = Dm.BoundaryCondition; + iproc = Dm->iproc(); + jproc = Dm->jproc(); + kproc = Dm->kproc(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); + BoundaryCondition = Dm->BoundaryCondition; //...................................................................................... ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double)); // Allocate device memory @@ -176,43 +176,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ ScaLBL_AllocateZeroCopy((void **) &dvcRecvDist_YZ, recvCount_YZ*sizeof(int)); // Allocate device memory //...................................................................................... - ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm.sendList_x,sendCount_x*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm.sendList_X,sendCount_X*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm.sendList_y,sendCount_y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm.sendList_Y,sendCount_Y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm.sendList_z,sendCount_z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm.sendList_Z,sendCount_Z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm.sendList_xy,sendCount_xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm.sendList_XY,sendCount_XY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm.sendList_xY,sendCount_xY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm.sendList_Xy,sendCount_Xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm.sendList_xz,sendCount_xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm.sendList_XZ,sendCount_XZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm.sendList_xZ,sendCount_xZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm.sendList_Xz,sendCount_Xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm.sendList_yz,sendCount_yz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm.sendList_YZ,sendCount_YZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm.sendList_yZ,sendCount_yZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm.sendList_Yz,sendCount_Yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_x,Dm->sendList_x,sendCount_x*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_X,Dm->sendList_X,sendCount_X*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_y,Dm->sendList_y,sendCount_y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Y,Dm->sendList_Y,sendCount_Y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_z,Dm->sendList_z,sendCount_z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Z,Dm->sendList_Z,sendCount_Z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xy,Dm->sendList_xy,sendCount_xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_XY,Dm->sendList_XY,sendCount_XY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xY,Dm->sendList_xY,sendCount_xY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Xy,Dm->sendList_Xy,sendCount_Xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xz,Dm->sendList_xz,sendCount_xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_XZ,Dm->sendList_XZ,sendCount_XZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_xZ,Dm->sendList_xZ,sendCount_xZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Xz,Dm->sendList_Xz,sendCount_Xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_yz,Dm->sendList_yz,sendCount_yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_YZ,Dm->sendList_YZ,sendCount_YZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_yZ,Dm->sendList_yZ,sendCount_yZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcSendList_Yz,Dm->sendList_Yz,sendCount_Yz*sizeof(int)); //...................................................................................... - ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm.recvList_x,recvCount_x*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm.recvList_X,recvCount_X*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm.recvList_y,recvCount_y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm.recvList_Y,recvCount_Y*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm.recvList_z,recvCount_z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm.recvList_Z,recvCount_Z*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm.recvList_xy,recvCount_xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm.recvList_XY,recvCount_XY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm.recvList_xY,recvCount_xY*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm.recvList_Xy,recvCount_Xy*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm.recvList_xz,recvCount_xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm.recvList_XZ,recvCount_XZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm.recvList_xZ,recvCount_xZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm.recvList_Xz,recvCount_Xz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm.recvList_yz,recvCount_yz*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm.recvList_YZ,recvCount_YZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm.recvList_yZ,recvCount_yZ*sizeof(int)); - ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm.recvList_Yz,recvCount_Yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_x,Dm->recvList_x,recvCount_x*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_X,Dm->recvList_X,recvCount_X*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_y,Dm->recvList_y,recvCount_y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Y,Dm->recvList_Y,recvCount_Y*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_z,Dm->recvList_z,recvCount_z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Z,Dm->recvList_Z,recvCount_Z*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xy,Dm->recvList_xy,recvCount_xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_XY,Dm->recvList_XY,recvCount_XY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xY,Dm->recvList_xY,recvCount_xY*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Xy,Dm->recvList_Xy,recvCount_Xy*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xz,Dm->recvList_xz,recvCount_xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_XZ,Dm->recvList_XZ,recvCount_XZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_xZ,Dm->recvList_xZ,recvCount_xZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Xz,Dm->recvList_Xz,recvCount_Xz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_yz,Dm->recvList_yz,recvCount_yz*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_YZ,Dm->recvList_YZ,recvCount_YZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_yZ,Dm->recvList_yZ,recvCount_yZ*sizeof(int)); + ScaLBL_CopyToZeroCopy(dvcRecvList_Yz,Dm->recvList_Yz,recvCount_Yz*sizeof(int)); //...................................................................................... MPI_Barrier(MPI_COMM_SCALBL); @@ -221,70 +221,70 @@ ScaLBL_Communicator::ScaLBL_Communicator(Domain &Dm){ // Set up the recieve distribution lists //................................................................................... //...Map recieve list for the X face: q=2,8,10,12,14 ................................. - D3Q19_MapRecv(-1,0,0,Dm.recvList_X,0,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,-1,0,Dm.recvList_X,recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,1,0,Dm.recvList_X,2*recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,0,-1,Dm.recvList_X,3*recvCount_X,recvCount_X,dvcRecvDist_X); - D3Q19_MapRecv(-1,0,1,Dm.recvList_X,4*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,0,Dm->recvList_X,0,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,-1,0,Dm->recvList_X,recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,1,0,Dm->recvList_X,2*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,-1,Dm->recvList_X,3*recvCount_X,recvCount_X,dvcRecvDist_X); + D3Q19_MapRecv(-1,0,1,Dm->recvList_X,4*recvCount_X,recvCount_X,dvcRecvDist_X); //................................................................................... //...Map recieve list for the x face: q=1,7,9,11,13.................................. - D3Q19_MapRecv(1,0,0,Dm.recvList_x,0,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,1,0,Dm.recvList_x,recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,-1,0,Dm.recvList_x,2*recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,0,1,Dm.recvList_x,3*recvCount_x,recvCount_x,dvcRecvDist_x); - D3Q19_MapRecv(1,0,-1,Dm.recvList_x,4*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,0,Dm->recvList_x,0,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,1,0,Dm->recvList_x,recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,-1,0,Dm->recvList_x,2*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,1,Dm->recvList_x,3*recvCount_x,recvCount_x,dvcRecvDist_x); + D3Q19_MapRecv(1,0,-1,Dm->recvList_x,4*recvCount_x,recvCount_x,dvcRecvDist_x); //................................................................................... //...Map recieve list for the y face: q=4,8,9,16,18 ................................... - D3Q19_MapRecv(0,-1,0,Dm.recvList_Y,0,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(-1,-1,0,Dm.recvList_Y,recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(1,-1,0,Dm.recvList_Y,2*recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(0,-1,-1,Dm.recvList_Y,3*recvCount_Y,recvCount_Y,dvcRecvDist_Y); - D3Q19_MapRecv(0,-1,1,Dm.recvList_Y,4*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,0,Dm->recvList_Y,0,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(-1,-1,0,Dm->recvList_Y,recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(1,-1,0,Dm->recvList_Y,2*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,-1,Dm->recvList_Y,3*recvCount_Y,recvCount_Y,dvcRecvDist_Y); + D3Q19_MapRecv(0,-1,1,Dm->recvList_Y,4*recvCount_Y,recvCount_Y,dvcRecvDist_Y); //................................................................................... //...Map recieve list for the Y face: q=3,7,10,15,17 .................................. - D3Q19_MapRecv(0,1,0,Dm.recvList_y,0,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(1,1,0,Dm.recvList_y,recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(-1,1,0,Dm.recvList_y,2*recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(0,1,1,Dm.recvList_y,3*recvCount_y,recvCount_y,dvcRecvDist_y); - D3Q19_MapRecv(0,1,-1,Dm.recvList_y,4*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,0,Dm->recvList_y,0,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(1,1,0,Dm->recvList_y,recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(-1,1,0,Dm->recvList_y,2*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,1,Dm->recvList_y,3*recvCount_y,recvCount_y,dvcRecvDist_y); + D3Q19_MapRecv(0,1,-1,Dm->recvList_y,4*recvCount_y,recvCount_y,dvcRecvDist_y); //................................................................................... //...Map recieve list for the z face<<<6,12,13,16,17).............................................. - D3Q19_MapRecv(0,0,-1,Dm.recvList_Z,0,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(-1,0,-1,Dm.recvList_Z,recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(1,0,-1,Dm.recvList_Z,2*recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(0,-1,-1,Dm.recvList_Z,3*recvCount_Z,recvCount_Z,dvcRecvDist_Z); - D3Q19_MapRecv(0,1,-1,Dm.recvList_Z,4*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,0,-1,Dm->recvList_Z,0,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(-1,0,-1,Dm->recvList_Z,recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(1,0,-1,Dm->recvList_Z,2*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,-1,-1,Dm->recvList_Z,3*recvCount_Z,recvCount_Z,dvcRecvDist_Z); + D3Q19_MapRecv(0,1,-1,Dm->recvList_Z,4*recvCount_Z,recvCount_Z,dvcRecvDist_Z); //...Map recieve list for the Z face<<<5,11,14,15,18).............................................. - D3Q19_MapRecv(0,0,1,Dm.recvList_z,0,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(1,0,1,Dm.recvList_z,recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(-1,0,1,Dm.recvList_z,2*recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(0,1,1,Dm.recvList_z,3*recvCount_z,recvCount_z,dvcRecvDist_z); - D3Q19_MapRecv(0,-1,1,Dm.recvList_z,4*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,0,1,Dm->recvList_z,0,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(1,0,1,Dm->recvList_z,recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(-1,0,1,Dm->recvList_z,2*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,1,1,Dm->recvList_z,3*recvCount_z,recvCount_z,dvcRecvDist_z); + D3Q19_MapRecv(0,-1,1,Dm->recvList_z,4*recvCount_z,recvCount_z,dvcRecvDist_z); //.................................................................................. //...Map recieve list for the xy edge <<<8)................................ - D3Q19_MapRecv(-1,-1,0,Dm.recvList_XY,0,recvCount_XY,dvcRecvDist_XY); + D3Q19_MapRecv(-1,-1,0,Dm->recvList_XY,0,recvCount_XY,dvcRecvDist_XY); //...Map recieve list for the Xy edge <<<9)................................ - D3Q19_MapRecv(1,-1,0,Dm.recvList_xY,0,recvCount_xY,dvcRecvDist_xY); + D3Q19_MapRecv(1,-1,0,Dm->recvList_xY,0,recvCount_xY,dvcRecvDist_xY); //...Map recieve list for the xY edge <<<10)................................ - D3Q19_MapRecv(-1,1,0,Dm.recvList_Xy,0,recvCount_Xy,dvcRecvDist_Xy); + D3Q19_MapRecv(-1,1,0,Dm->recvList_Xy,0,recvCount_Xy,dvcRecvDist_Xy); //...Map recieve list for the XY edge <<<7)................................ - D3Q19_MapRecv(1,1,0,Dm.recvList_xy,0,recvCount_xy,dvcRecvDist_xy); + D3Q19_MapRecv(1,1,0,Dm->recvList_xy,0,recvCount_xy,dvcRecvDist_xy); //...Map recieve list for the xz edge <<<12)................................ - D3Q19_MapRecv(-1,0,-1,Dm.recvList_XZ,0,recvCount_XZ,dvcRecvDist_XZ); + D3Q19_MapRecv(-1,0,-1,Dm->recvList_XZ,0,recvCount_XZ,dvcRecvDist_XZ); //...Map recieve list for the xZ edge <<<14)................................ - D3Q19_MapRecv(-1,0,1,Dm.recvList_Xz,0,recvCount_Xz,dvcRecvDist_Xz); + D3Q19_MapRecv(-1,0,1,Dm->recvList_Xz,0,recvCount_Xz,dvcRecvDist_Xz); //...Map recieve list for the Xz edge <<<13)................................ - D3Q19_MapRecv(1,0,-1,Dm.recvList_xZ,0,recvCount_xZ,dvcRecvDist_xZ); + D3Q19_MapRecv(1,0,-1,Dm->recvList_xZ,0,recvCount_xZ,dvcRecvDist_xZ); //...Map recieve list for the XZ edge <<<11)................................ - D3Q19_MapRecv(1,0,1,Dm.recvList_xz,0,recvCount_xz,dvcRecvDist_xz); + D3Q19_MapRecv(1,0,1,Dm->recvList_xz,0,recvCount_xz,dvcRecvDist_xz); //...Map recieve list for the yz edge <<<16)................................ - D3Q19_MapRecv(0,-1,-1,Dm.recvList_YZ,0,recvCount_YZ,dvcRecvDist_YZ); + D3Q19_MapRecv(0,-1,-1,Dm->recvList_YZ,0,recvCount_YZ,dvcRecvDist_YZ); //...Map recieve list for the yZ edge <<<18)................................ - D3Q19_MapRecv(0,-1,1,Dm.recvList_Yz,0,recvCount_Yz,dvcRecvDist_Yz); + D3Q19_MapRecv(0,-1,1,Dm->recvList_Yz,0,recvCount_Yz,dvcRecvDist_Yz); //...Map recieve list for the Yz edge <<<17)................................ - D3Q19_MapRecv(0,1,-1,Dm.recvList_yZ,0,recvCount_yZ,dvcRecvDist_yZ); + D3Q19_MapRecv(0,1,-1,Dm->recvList_yZ,0,recvCount_yZ,dvcRecvDist_yZ); //...Map recieve list for the YZ edge <<<15)................................ - D3Q19_MapRecv(0,1,1,Dm.recvList_yz,0,recvCount_yz,dvcRecvDist_yz); + D3Q19_MapRecv(0,1,1,Dm->recvList_yz,0,recvCount_yz,dvcRecvDist_yz); //................................................................................... //...................................................................................... diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 65e45338..503a4aeb 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -129,7 +129,7 @@ extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int class ScaLBL_Communicator{ public: //...................................................................................... - ScaLBL_Communicator(Domain &Dm); + ScaLBL_Communicator(Domain *Dm); //ScaLBL_Communicator(Domain &Dm, IntArray &Map); ~ScaLBL_Communicator(); diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 6dbf2ac6..56eebcfe 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -71,22 +71,17 @@ void ScaLBL_ColorModel::ReadParams(string filename){ MPI_Barrier(comm); } -ScaLBL_ColorModel::~ScaLBL_ColorModel(){ - -} - void ScaLBL_ColorModel::ReadInput(){ + int rank=Dm->rank(); + size_t readID; //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... - sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); // .......... READ THE INPUT FILE ....................................... id = new char[N]; - double sum, sum_local; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); //........................................................................... if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; //....................................................................... @@ -98,16 +93,15 @@ void ScaLBL_ColorModel::ReadInput(){ if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); sprintf(LocalRankFilename,"ID.%05i",rank); - size_t readID; FILE *IDFILE = fopen(LocalRankFilename,"rb"); if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: ID.xxxxx"); readID=fread(id,1,N,IDFILE); - if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading ID (rank=%i) \n",rank); + if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading ID (rank=%i) \n",Dm->rank()); fclose(IDFILE); // Read restart file if (Restart == true){ - if (rank==0){ + if (Dm->rank()==0){ printf("Reading restart file! \n"); ifstream restart("Restart.txt"); if (restart.is_open()){ @@ -136,8 +130,13 @@ void ScaLBL_ColorModel::Create(){ //....................................................................... // Compute the media porosity, assign phase labels and solid composition //....................................................................... - double porosity; - double sum_local=0.0; + int nprocs=nprocx*nprocy*nprocz; + int rank=Dm->rank(); + double sum; + double porosity; + double sum_local=0.0; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); Np=0; // number of local pore nodes //....................................................................... for (int k=1;kid,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); MPI_Barrier(comm); //........................................................................... @@ -232,6 +231,9 @@ void ScaLBL_ColorModel::Create(){ //........................................................................... // Update GPU data structures if (rank==0) printf ("Setting up device map and neighbor list \n"); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + int *TmpMap; TmpMap=new int[Np]; for (int k=1; krank(); // Compute the solid interaction potential and copy result to device if (rank==0) printf("Computing solid interaction potential \n"); + double *PhaseLabel; + PhaseLabel=new double [Nx*Ny*Nz]; + Mask->AssignComponentLabels(PhaseLabel); double *Tmp; Tmp=new double[3*Np]; //Averages->UpdateMeshValues(); // this computes the gradient of distance field (among other things) @@ -339,9 +344,9 @@ void ScaLBL_ColorModel::Initialize(){ DoubleArray Psy(Nx,Ny,Nz); DoubleArray Psz(Nx,Ny,Nz); DoubleArray Psnorm(Nx,Ny,Nz); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[0],Psx); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[Np],Psy); - ScaLBL_Comm.RegularLayout(Map,&SolidPotential[2*Np],Psz); + ScaLBL_Comm->RegularLayout(Map,&SolidPotential[0],Psx); + ScaLBL_Comm->RegularLayout(Map,&SolidPotential[Np],Psy); + ScaLBL_Comm->RegularLayout(Map,&SolidPotential[2*Np],Psz); for (int n=0; nlast_interior, Np); } void ScaLBL_ColorModel::Run(){ + int nprocs=nprocx*nprocy*nprocz; + int rank=Dm->rank(); + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + if (rank==0) printf("********************************************************\n"); if (rank==0) printf("No. of timesteps: %i \n", timestepMax); //.......create and start timer............ @@ -390,8 +397,14 @@ void ScaLBL_ColorModel::Run(){ starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ + // read the input database + auto db = std::make_shared( "input.db" ); + auto domain_db = db->getDatabase( "Domain" ); + auto color_db = db->getDatabase( "Color" ); + auto analysis_db = db->getDatabase( "Analysis" ); + PROFILE_START("Loop"); - runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); + // runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); while (timestep < timestepMax ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } PROFILE_START("Update"); @@ -399,83 +412,83 @@ void ScaLBL_ColorModel::Run(){ timestep++; // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set BCs if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } else if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ MPI_Barrier(comm); PROFILE_STOP("Update"); // Run the analysis - analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); + // analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); } - analysis.finish(); + // analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e43d7ee1..c80ca533 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,7 +11,7 @@ ADD_LBPM_EXECUTABLE( lbpm_random_pp ) ADD_LBPM_EXECUTABLE( lbpm_refine_pp ) ADD_LBPM_EXECUTABLE( lbpm_morphdrain_pp ) ADD_LBPM_EXECUTABLE( lbpm_morphopen_pp ) -ADD_LBPM_EXECUTABLE( lbpm_morph_pp ) +#ADD_LBPM_EXECUTABLE( lbpm_morph_pp ) ADD_LBPM_EXECUTABLE( lbpm_segmented_pp ) #ADD_LBPM_EXECUTABLE( lbpm_block_pp ) ADD_LBPM_EXECUTABLE( lbpm_segmented_decomp ) From 04854bdc4e225d778673d3f161f9907eab705241 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 21:04:02 -0400 Subject: [PATCH 060/124] removing models --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de984c16..d108e8ed 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,7 +158,7 @@ IF ( NOT ONLY_BUILD_DOCS ) ADD_PACKAGE_SUBDIRECTORY( analysis ) ADD_PACKAGE_SUBDIRECTORY( IO ) ADD_PACKAGE_SUBDIRECTORY( threadpool ) - ADD_PACKAGE_SUBDIRECTORY( models ) + #ADD_PACKAGE_SUBDIRECTORY( models ) IF ( USE_CUDA ) ADD_PACKAGE_SUBDIRECTORY( gpu ) ELSE() From 76cfb78880f59dac7bcd7f429ee043cd398c0417 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 21:32:30 -0400 Subject: [PATCH 061/124] fixed TestTorus --- tests/CMakeLists.txt | 2 +- tests/TestTorus.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e43d7ee1..c80ca533 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,7 +11,7 @@ ADD_LBPM_EXECUTABLE( lbpm_random_pp ) ADD_LBPM_EXECUTABLE( lbpm_refine_pp ) ADD_LBPM_EXECUTABLE( lbpm_morphdrain_pp ) ADD_LBPM_EXECUTABLE( lbpm_morphopen_pp ) -ADD_LBPM_EXECUTABLE( lbpm_morph_pp ) +#ADD_LBPM_EXECUTABLE( lbpm_morph_pp ) ADD_LBPM_EXECUTABLE( lbpm_segmented_pp ) #ADD_LBPM_EXECUTABLE( lbpm_block_pp ) ADD_LBPM_EXECUTABLE( lbpm_segmented_decomp ) diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index 5c2fa519..a2c4210b 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -12,7 +12,8 @@ std::shared_ptr loadInputs( int nprocs ) { - auto db = std::make_shared( "Domain.in" ); + //auto db = std::make_shared( "Domain.in" ); + auto db = std::make_shared(); db->putScalar( "BC", 0 ); db->putVector( "nproc", { 1, 1, 1 } ); db->putVector( "n", { 100, 100, 100 } ); From 6fdf5e8a290f9eaf988f104dd0f7c3b47f047fbc Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 21:46:29 -0400 Subject: [PATCH 062/124] debugging DFH tests --- tests/TestBubbleDFH.cpp | 317 +++++++++++++++---------------------- tests/TestColorGradDFH.cpp | 111 +++---------- tests/TestTorus.cpp | 30 ++-- 3 files changed, 167 insertions(+), 291 deletions(-) diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 040eba8a..b8819c30 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -20,6 +20,17 @@ using namespace std; +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared(); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 100, 100, 100 } ); + db->putScalar( "nspheres", 1 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} + //************************************************************************* // Implementation of Two-Phase Immiscible LBM //************************************************************************* @@ -56,208 +67,134 @@ int main(int argc, char **argv) PROFILE_START("Main"); Utilities::setErrorHandlers(); - int ANALYSIS_INTERVAL = 1000; - int BLOBID_INTERVAL = 1000; - std::string analysis_method = "independent"; - if (argc >= 3) { - ANALYSIS_INTERVAL = atoi(argv[1]); - BLOBID_INTERVAL = atoi(argv[2]); - } - if (argc >= 4) - analysis_method = std::string(argv[3]); + if ( argc < 2 ) { + std::cerr << "Invalid number of arguments, no input file specified\n"; + return -1; + } + auto filename = argv[1]; + auto db = std::make_shared( filename ); + auto domain_db = db->getDatabase( "Domain" ); + auto color_db = db->getDatabase( "Color" ); + auto analysis_db = db->getDatabase( "Analysis" ); - // Variables that specify the computational domain - string FILENAME; - int Nx,Ny,Nz,Np; // local sub-domain size - double Lx,Ly,Lz; // Domain length - // Color Model parameters - int timestepMax; - double tauA, tauB, rhoA,rhoB; - double Fx,Fy,Fz,tol,err; - double alpha, beta; - double bns,bws,cns,cws; - int BoundaryCondition; - int InitialCondition; - // bool pBC,Restart; - int i,j,k,n; - double din, dout, flux; - double inletA,inletB,outletA,outletB; - inletA=1.f; - inletB=0.f; - outletA=0.f; - outletB=1.f; - flux = 10.f; - dout=1.f; + if (rank == 0){ + printf("********************************************************\n"); + printf("Running Color LBM \n"); + printf("********************************************************\n"); + } + // Initialize compute device + // int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); - int RESTART_INTERVAL=20000; - //int ANALYSIS_)INTERVAL=1000; - int BLOB_ANALYSIS_INTERVAL=1000; - int timestep = 0; + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); - if (rank==0){ - //............................................................. - // READ SIMULATION PARMAETERS FROM INPUT FILE - //............................................................. - ifstream input("Color.in"); - if (input.is_open()){ - // Line 1: model parameters (tau, alpha, beta, das, dbs) - input >> tauA; // Viscosity non-wetting - input >> tauB; // Viscosity wetting - input >> rhoA; // density non-wetting - input >> rhoB; // density wetting - input >> alpha; // Surface Tension parameter - input >> beta; // Width of the interface - input >> cws; // solid interaction coefficients - input >> bws; // solid interaction coefficients - input >> cns; // solid interaction coefficients - input >> bns; // solid interaction coefficients - // Line 2: External force components (Fx,Fy, Fz) - input >> Fx; - input >> Fy; - input >> Fz; - // Line 4: Pressure Boundary conditions - input >> InitialCondition; - input >> BoundaryCondition; - input >> din; - input >> dout; - // Line 5: time-stepping criteria - input >> timestepMax; // max no. of timesteps - input >> RESTART_INTERVAL; // restart interval - input >> tol; // error tolerance - // Line 6: Analysis options - input >> BLOB_ANALYSIS_INTERVAL; // interval to analyze blob states - //............................................................. - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Color.in is missing)! Default parameters will be used. \n"); - tauA = tauB = 1.0; - rhoA = rhoB = 1.0; - alpha=0.005; - beta= 0.9; - Fx = Fy = Fz = 0.0; - InitialCondition=0; - BoundaryCondition=0; - din=dout=1.0; - timestepMax=0; - } + // Variables that specify the computational domain + string FILENAME; - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (input.is_open()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Domain.in is missing)! Default parameters will be used. \n"); - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=10; - Lx=Ly=Lz=1.0; - } - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&tauA,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&tauB,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&rhoA,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&rhoB,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&cns,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&cws,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&bns,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&bws,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm); - MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm); - MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm); - MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. + // Color Model parameters + int timestepMax = domain_db->getScalar( "timestepMax" ); + double tauA = domain_db->getScalar( "tauA" ); + double tauB = domain_db->getScalar( "tauB" ); + double rhoA = domain_db->getScalar( "rhoA" ); + double rhoB = domain_db->getScalar( "rhoB" ); + double Fx = domain_db->getVector( "F" )[0]; + double Fy = domain_db->getVector( "F" )[1]; + double Fz = domain_db->getVector( "F" )[2]; + double alpha = domain_db->getScalar( "alpha" ); + double beta = domain_db->getScalar( "beta" ); + bool Restart = domain_db->getScalar( "Restart" ); + double din = domain_db->getScalar( "din" ); + double dout = domain_db->getScalar( "dout" );; + double inletA=1.f; + double inletB=0.f; + double outletA=0.f; + double outletB=1.f; + double flux = 10.f; - flux = 0.f; - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details + // Read domain values + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + int BoundaryCondition = domain_db->getScalar( "BC" ); + int Nx = size[0]; + int Ny = size[1]; + int Nz = size[2]; + int Lx = L[0]; + int Ly = L[1]; + int Lz = L[2]; + int nprocx = nproc[0]; + int nprocy = nproc[1]; + int nprocz = nproc[2]; - // Get the rank info - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); + int timestep = 6; - MPI_Barrier(comm); + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + // Get the rank info + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - if (rank==0){ - printf("********************************************************\n"); - printf("tau (non-wetting) = %f \n", tauA); - printf("tau (wetting) = %f \n", tauB); - printf("density (non-wetting) = %f \n", rhoA); - printf("density (wetting) = %f \n", rhoB); - printf("alpha = %f \n", alpha); - printf("beta = %f \n", beta); - printf("gamma_{wn} = %f \n", 5.796*alpha); - printf("Force(x) = %f \n", Fx); - printf("Force(y) = %f \n", Fy); - printf("Force(z) = %f \n", Fz); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); - if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); - if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); - if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); - if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); - if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n"); - if (InitialCondition==0) printf("Initial conditions assigned from phase ID file \n"); - if (InitialCondition==1) printf("Initial conditions assigned from restart file \n"); - printf("********************************************************\n"); - } + MPI_Barrier(comm); - // Initialized domain and averaging framework for Two-Phase Flow - bool pBC,velBC; - if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4) - pBC=true; - else pBC=false; - if (BoundaryCondition==2) velBC=true; - else velBC=false; + if (nprocs != nprocx*nprocy*nprocz){ + printf("nprocx = %i \n",nprocx); + printf("nprocy = %i \n",nprocy); + printf("nprocz = %i \n",nprocz); + INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); + } - bool Restart; - if (InitialCondition==1) Restart=true; - else Restart=false; - NULL_USE(pBC); NULL_USE(velBC); + if (rank==0){ + printf("********************************************************\n"); + printf("tau (non-wetting) = %f \n", tauA); + printf("tau (wetting) = %f \n", tauB); + printf("density (non-wetting) = %f \n", rhoA); + printf("density (wetting) = %f \n", rhoB); + printf("alpha = %f \n", alpha); + printf("beta = %f \n", beta); + printf("gamma_{wn} = %f \n", 5.796*alpha); + printf("Force(x) = %f \n", Fx); + printf("Force(y) = %f \n", Fy); + printf("Force(z) = %f \n", Fz); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); + if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); + if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); + if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); + if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); + if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n"); + if (!Restart) printf("Initial conditions assigned from phase ID file \n"); + if (Restart) printf("Initial conditions assigned from restart file \n"); + printf("********************************************************\n"); + } - // Full domain used for averaging (do not use mask for analysis) - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + // Initialized domain and averaging framework for Two-Phase Flow + bool pBC; + if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4) + pBC=true; + else + pBC=false; + + // Full domain used for averaging (do not use mask for analysis) + Domain Dm(domain_db); + for (int i=0; i Averages( new TwoPhase(Dm) ); + // TwoPhase Averages(Dm); + Dm.CommInit(comm); + + // Mask that excludes the solid phase + Domain Mask(domain_db); + MPI_Barrier(comm); + + Nx+=2; Ny+=2; Nz += 2; + int N = Nx*Ny*Nz; + //....................................................................... + if (rank == 0) printf("Read input media... \n"); + //....................................................................... for (i=0; i Averages( new TwoPhase(Dm) ); // TwoPhase Averages(Dm); diff --git a/tests/TestColorGradDFH.cpp b/tests/TestColorGradDFH.cpp index 227d68ad..47ca9817 100644 --- a/tests/TestColorGradDFH.cpp +++ b/tests/TestColorGradDFH.cpp @@ -11,6 +11,16 @@ using namespace std; +std::shared_ptr loadInputs( int nprocs ) +{ + auto db = std::make_shared(); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 100, 100, 100 } ); + db->putScalar( "nspheres", 1 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} //*************************************************************************************** int main(int argc, char **argv) @@ -67,92 +77,24 @@ int main(int argc, char **argv) Fx = Fy = 0.f; Fz = 0.f; + + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=3; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=2; nprocy=1; - nprocz=1; - Nx=Ny=Nz=dim; - Nx = dim; Ny = dim; Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } - - MPI_Barrier(comm); - - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + if (rank==0){ + printf("********************************************************\n"); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("********************************************************\n"); + } + // Get the rank info + Domain Dm(db); Nx += 2; Ny += 2; Nz += 2; @@ -178,7 +120,6 @@ int main(int argc, char **argv) if (rank==0) printf ("Create ScaLBL_Communicator \n"); //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); ScaLBL_Communicator ScaLBL_Comm(Dm); // LBM variables diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index a2c4210b..d1ab142e 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -44,25 +44,23 @@ int main(int argc, char **argv) //....................................................................... int i,j,k,n; - // Load inputs - auto db = loadInputs( nprocs ); - int Nx = db->getVector( "n" )[0]; - int Ny = db->getVector( "n" )[1]; - int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; - - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } - + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + if (rank==0){ + printf("********************************************************\n"); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("********************************************************\n"); + } // Get the rank info - Domain Dm(db); + Domain Dm(db); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); TwoPhase Averages(Dm); Nx += 2; From 75b54773165623748fb3e9bf7f446fe6bd7e6601 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 22:32:26 -0400 Subject: [PATCH 063/124] debugging dfh --- tests/CMakeLists.txt | 2 +- tests/TestBubbleDFH.cpp | 83 ++++++++++++++--------------------------- 2 files changed, 29 insertions(+), 56 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c80ca533..fc3738d2 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,8 +43,8 @@ ADD_LBPM_TEST( TestFluxBC ) ADD_LBPM_TEST( TestMap ) ADD_LBPM_TEST( TestMRT ) ADD_LBPM_TEST( TestColorGrad ) -ADD_LBPM_TEST( TestBubbleDFH ) ADD_LBPM_TEST( TestColorGradDFH ) +ADD_LBPM_TEST( TestBubbleDFH ../example/Piston/input.db) ADD_LBPM_TEST( TestColorMassBounceback ) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index b8819c30..9716df80 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -20,17 +20,6 @@ using namespace std; -std::shared_ptr loadInputs( int nprocs ) -{ - auto db = std::make_shared(); - db->putScalar( "BC", 0 ); - db->putVector( "nproc", { 1, 1, 1 } ); - db->putVector( "n", { 100, 100, 100 } ); - db->putScalar( "nspheres", 1 ); - db->putVector( "L", { 1, 1, 1 } ); - return db; -} - //************************************************************************* // Implementation of Two-Phase Immiscible LBM //************************************************************************* @@ -44,11 +33,9 @@ int main(int argc, char **argv) MPI_Comm_dup(MPI_COMM_WORLD,&comm); int rank = comm_rank(comm); int nprocs = comm_size(comm); + int check=0; { // Limit scope so variables that contain communicators will free before MPI_Finialize - - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - + int i,j,k,n,Np; if (rank == 0){ printf("********************************************************\n"); printf("Running DFH/Color LBM \n"); @@ -98,19 +85,19 @@ int main(int argc, char **argv) string FILENAME; // Color Model parameters - int timestepMax = domain_db->getScalar( "timestepMax" ); - double tauA = domain_db->getScalar( "tauA" ); - double tauB = domain_db->getScalar( "tauB" ); - double rhoA = domain_db->getScalar( "rhoA" ); - double rhoB = domain_db->getScalar( "rhoB" ); - double Fx = domain_db->getVector( "F" )[0]; - double Fy = domain_db->getVector( "F" )[1]; - double Fz = domain_db->getVector( "F" )[2]; - double alpha = domain_db->getScalar( "alpha" ); - double beta = domain_db->getScalar( "beta" ); - bool Restart = domain_db->getScalar( "Restart" ); - double din = domain_db->getScalar( "din" ); - double dout = domain_db->getScalar( "dout" );; + int timestepMax = color_db->getScalar( "timestepMax" ); + double tauA = color_db->getScalar( "tauA" ); + double tauB = color_db->getScalar( "tauB" ); + double rhoA = color_db->getScalar( "rhoA" ); + double rhoB = color_db->getScalar( "rhoB" ); + double Fx = color_db->getVector( "F" )[0]; + double Fy = color_db->getVector( "F" )[1]; + double Fz = color_db->getVector( "F" )[2]; + double alpha = color_db->getScalar( "alpha" ); + double beta = color_db->getScalar( "beta" ); + bool Restart = color_db->getScalar( "Restart" ); + double din = color_db->getScalar( "din" ); + double dout = color_db->getScalar( "dout" );; double inletA=1.f; double inletB=0.f; double outletA=0.f; @@ -118,16 +105,16 @@ int main(int argc, char **argv) double flux = 10.f; // Read domain values - auto L = domain_db->getVector( "L" ); + auto L = domain_db->getVector( "L" ); auto size = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); int BoundaryCondition = domain_db->getScalar( "BC" ); int Nx = size[0]; int Ny = size[1]; int Nz = size[2]; - int Lx = L[0]; - int Ly = L[1]; - int Lz = L[2]; + double Lx = L[0]; + double Ly = L[1]; + double Lz = L[2]; int nprocx = nproc[0]; int nprocy = nproc[1]; int nprocz = nproc[2]; @@ -195,21 +182,6 @@ int main(int argc, char **argv) //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... - for (i=0; i Averages( new TwoPhase(Dm) ); - // TwoPhase Averages(Dm); - Dm.CommInit(comm); - - // Mask that excludes the solid phase - Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - MPI_Barrier(comm); - - Nx+=2; Ny+=2; Nz += 2; - int N = Nx*Ny*Nz; - //....................................................................... - if (rank == 0) printf("Read input media... \n"); - //....................................................................... - //....................................................................... // Filenames used char LocalRankString[8]; @@ -250,10 +222,10 @@ int main(int argc, char **argv) } } // Initialize the bubble - for (k=0;kUpdateMeshValues(); // this computes the gradient of distance field (among other things) double count_wet=0.f; + double cns,bns,cws,bws; + cns=bns=bws=cws=1.0; for (k=1; k analysis_db; + //std::shared_ptr analysis_db; runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); - analysis.createThreads( analysis_method, 4 ); + //analysis.createThreads( analysis_method, 4 ); while (timestep < timestepMax && err > tol ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } PROFILE_START("Update"); @@ -599,6 +571,7 @@ int main(int argc, char **argv) } // Limit scope so variables that contain communicators will free before MPI_Finialize MPI_Comm_free(&comm); MPI_Finalize(); + return check; } From 86c49f34777a04d353366c04557849aa4c635cb1 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 22:47:56 -0400 Subject: [PATCH 064/124] DFH test working --- example/Bubble/input.db | 38 ++++++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 +- tests/TestBubbleDFH.cpp | 2 +- tests/TestColorGradDFH.cpp | 8 +------- 4 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 example/Bubble/input.db diff --git a/example/Bubble/input.db b/example/Bubble/input.db new file mode 100644 index 00000000..72a9aecb --- /dev/null +++ b/example/Bubble/input.db @@ -0,0 +1,38 @@ +Color { + tauA = 1.0; + tauB = 1.0; + rhoA = 1.0; + rhoB = 1.0; + alpha = 1e-2; + beta = 0.95; + F = 0, 0, 0 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 200 + interval = 1000 + tol = 1e-5; + das = 0.1 + dbs = 0.9 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type +} + +Analysis { + blobid_interval = 1000 // Frequency to perform blob identification + analysis_interval = 1000 // Frequency to perform analysis + restart_interval = 20000 // Frequency to write restart data + visualization_interval = 20000 // Frequency to write visualization data + restart_file = "Restart" // Filename to use for restart file (will append rank) + N_threads = 4 // Number of threads to use + load_balance = "independent" // Load balance method to use: "none", "default", "independent" +} + + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fc3738d2..460fadf9 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -44,7 +44,7 @@ ADD_LBPM_TEST( TestMap ) ADD_LBPM_TEST( TestMRT ) ADD_LBPM_TEST( TestColorGrad ) ADD_LBPM_TEST( TestColorGradDFH ) -ADD_LBPM_TEST( TestBubbleDFH ../example/Piston/input.db) +ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db) ADD_LBPM_TEST( TestColorMassBounceback ) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 9716df80..f30d44e9 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -95,7 +95,7 @@ int main(int argc, char **argv) double Fz = color_db->getVector( "F" )[2]; double alpha = color_db->getScalar( "alpha" ); double beta = color_db->getScalar( "beta" ); - bool Restart = color_db->getScalar( "Restart" ); + bool Restart = color_db->getScalar( "Restart" ); double din = color_db->getScalar( "din" ); double dout = color_db->getScalar( "dout" );; double inletA=1.f; diff --git a/tests/TestColorGradDFH.cpp b/tests/TestColorGradDFH.cpp index 47ca9817..b0c4a695 100644 --- a/tests/TestColorGradDFH.cpp +++ b/tests/TestColorGradDFH.cpp @@ -34,13 +34,9 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); printf("Running Color Model: TestColorGradDFH \n"); @@ -49,13 +45,11 @@ int main(int argc, char **argv) // BGK Model parameters string FILENAME; - unsigned int nBlocks, nthreads; int timestepMax, interval; double Fx,Fy,Fz,tol; // Domain variables double Lx,Ly,Lz; int nspheres; - int Nx,Ny,Nz; int i,j,k,n; int dim = 3; //if (rank == 0) printf("dim=%d\n",dim); From 3a458c4a9cd44d9dff20889ec4a8d1d3c12ba053 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 22:53:38 -0400 Subject: [PATCH 065/124] db for dfh --- tests/lbpm_dfh_simulator.cpp | 37 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index 4b16040b..893d6671 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -20,7 +20,6 @@ using namespace std; - //************************************************************************* // Implementation of Two-Phase Immiscible LBM //************************************************************************* @@ -66,19 +65,19 @@ int main(int argc, char **argv) string FILENAME; // Color Model parameters - int timestepMax = domain_db->getScalar( "timestepMax" ); - double tauA = domain_db->getScalar( "tauA" ); - double tauB = domain_db->getScalar( "tauB" ); - double rhoA = domain_db->getScalar( "rhoA" ); - double rhoB = domain_db->getScalar( "rhoB" ); - double Fx = domain_db->getVector( "F" )[0]; - double Fy = domain_db->getVector( "F" )[1]; - double Fz = domain_db->getVector( "F" )[2]; - double alpha = domain_db->getScalar( "alpha" ); - double beta = domain_db->getScalar( "beta" ); - bool Restart = domain_db->getScalar( "Restart" ); - double din = domain_db->getScalar( "din" ); - double dout = domain_db->getScalar( "dout" );; + int timestepMax = color_db->getScalar( "timestepMax" ); + double tauA = color_db->getScalar( "tauA" ); + double tauB = color_db->getScalar( "tauB" ); + double rhoA = color_db->getScalar( "rhoA" ); + double rhoB = color_db->getScalar( "rhoB" ); + double Fx = color_db->getVector( "F" )[0]; + double Fy = color_db->getVector( "F" )[1]; + double Fz = color_db->getVector( "F" )[2]; + double alpha = color_db->getScalar( "alpha" ); + double beta = color_db->getScalar( "beta" ); + bool Restart = color_db->getScalar( "Restart" ); + double din = color_db->getScalar( "din" ); + double dout = color_db->getScalar( "dout" );; double inletA=1.f; double inletB=0.f; double outletA=0.f; @@ -86,16 +85,16 @@ int main(int argc, char **argv) double flux = 10.f; // Read domain values - auto L = domain_db->getVector( "L" ); + auto L = domain_db->getVector( "L" ); auto size = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); int BoundaryCondition = domain_db->getScalar( "BC" ); int Nx = size[0]; int Ny = size[1]; int Nz = size[2]; - int Lx = L[0]; - int Ly = L[1]; - int Lz = L[2]; + double Lx = L[0]; + double Ly = L[1]; + double Lz = L[2]; int nprocx = nproc[0]; int nprocy = nproc[1]; int nprocz = nproc[2]; @@ -155,7 +154,7 @@ int main(int argc, char **argv) Dm.CommInit(comm); // Mask that excludes the solid phase - Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + Domain Mask(domain_db); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; From c6aaa14e33b4e4bac5dbb5adef5a38a880ec0e06 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 23:08:36 -0400 Subject: [PATCH 066/124] dfh to mass bb test --- tests/TestColorMassBounceback.cpp | 423 +++++++++++++++--------------- 1 file changed, 208 insertions(+), 215 deletions(-) diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index 5c2150a2..edcda83e 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -29,135 +29,130 @@ int main(int argc, char **argv) // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; + auto filename = argv[1]; + auto db = std::make_shared( filename ); + auto domain_db = db->getDatabase( "Domain" ); + auto color_db = db->getDatabase( "Color" ); + auto analysis_db = db->getDatabase( "Analysis" ); + if (rank == 0){ + printf("********************************************************\n"); + printf("TestColorMassBounceback \n"); + printf("********************************************************\n"); + } + // Initialize compute device + // int device=ScaLBL_SetDevice(rank); + ScaLBL_DeviceBarrier(); + MPI_Barrier(comm); - if (rank == 0){ - printf("********************************************************\n"); - printf("Running Color Model: TestColor \n"); - printf("********************************************************\n"); - } + PROFILE_ENABLE(1); + //PROFILE_ENABLE_TRACE(); + //PROFILE_ENABLE_MEMORY(); + PROFILE_SYNCHRONIZE(); + PROFILE_START("Main"); + Utilities::setErrorHandlers(); - // BGK Model parameters - string FILENAME; - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double Fx,Fy,Fz,tol; - // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; - int i,j,k,n; - int dim = 3; - //if (rank == 0) printf("dim=%d\n",dim); - int timestep = 0; - int timesteps = 100; - int centralNode = 2; + // Variables that specify the computational domain + string FILENAME; - double tauA = 1.0; - double tauB = 1.0; - double rhoA = 1.0; - double rhoB = 1.0; - double alpha = 0.005; - double beta = 0.95; - - double tau = 1.0; - double mu=(tau-0.5)/3.0; - double rlx_setA=1.0/tau; - double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + // Color Model parameters + int timestepMax = color_db->getScalar( "timestepMax" ); + double tauA = color_db->getScalar( "tauA" ); + double tauB = color_db->getScalar( "tauB" ); + double rhoA = color_db->getScalar( "rhoA" ); + double rhoB = color_db->getScalar( "rhoB" ); + double Fx = color_db->getVector( "F" )[0]; + double Fy = color_db->getVector( "F" )[1]; + double Fz = color_db->getVector( "F" )[2]; + double alpha = color_db->getScalar( "alpha" ); + double beta = color_db->getScalar( "beta" ); + bool Restart = color_db->getScalar( "Restart" ); + double din = color_db->getScalar( "din" ); + double dout = color_db->getScalar( "dout" );; + double inletA=1.f; + double inletB=0.f; + double outletA=0.f; + double outletB=1.f; + double flux = 10.f; - Fx = Fy = 0.f; - Fz = 0.f; + // Read domain values + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + int BoundaryCondition = domain_db->getScalar( "BC" ); + int Nx = size[0]; + int Ny = size[1]; + int Nz = size[2]; + double Lx = L[0]; + double Ly = L[1]; + double Lz = L[2]; + int nprocx = nproc[0]; + int nprocy = nproc[1]; + int nprocz = nproc[2]; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=3; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=2; nprocy=1; - nprocz=1; - Nx=Ny=Nz=dim; - Nx = dim; Ny = dim; Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** + int timestep = 6; - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } + // Get the rank info + const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - MPI_Barrier(comm); + MPI_Barrier(comm); - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; + if (nprocs != nprocx*nprocy*nprocz){ + printf("nprocx = %i \n",nprocx); + printf("nprocy = %i \n",nprocy); + printf("nprocz = %i \n",nprocz); + INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); + } - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + if (rank==0){ + printf("********************************************************\n"); + printf("tau (non-wetting) = %f \n", tauA); + printf("tau (wetting) = %f \n", tauB); + printf("density (non-wetting) = %f \n", rhoA); + printf("density (wetting) = %f \n", rhoB); + printf("alpha = %f \n", alpha); + printf("beta = %f \n", beta); + printf("gamma_{wn} = %f \n", 5.796*alpha); + printf("Force(x) = %f \n", Fx); + printf("Force(y) = %f \n", Fy); + printf("Force(z) = %f \n", Fz); + printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); + if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); + if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); + if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); + if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); + if (BoundaryCondition==4) printf("Average flux boundary conditions will be applied \n"); + if (!Restart) printf("Initial conditions assigned from phase ID file \n"); + if (Restart) printf("Initial conditions assigned from restart file \n"); + printf("********************************************************\n"); + } - Nx += 2; - Ny += 2; - Nz += 2; - int N = Nx*Ny*Nz; + // Initialized domain and averaging framework for Two-Phase Flow + bool pBC; + if (BoundaryCondition==1 || BoundaryCondition==3 || BoundaryCondition == 4) + pBC=true; + else + pBC=false; + // Full domain used for averaging (do not use mask for analysis) + Domain Dm(domain_db); + for (int i=0; i Averages( new TwoPhase(Dm) ); + // TwoPhase Averages(Dm); + Dm.CommInit(comm); + + // Mask that excludes the solid phase + Domain Mask(domain_db); + MPI_Barrier(comm); + + Nx+=2; Ny+=2; Nz += 2; + int N = Nx*Ny*Nz; + //....................................................................... + if (rank == 0) printf("Read input media... \n"); + //....................................................................... int Np=0; // number of local pore nodes double *PhaseLabel; PhaseLabel = new double[N]; @@ -196,29 +191,25 @@ int main(int argc, char **argv) int neighborSize=18*Np*sizeof(int); int *neighborList; IntArray Map(Nx,Ny,Nz); - neighborList= new int[18*Np]; - - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Npad=Np+32; + neighborList= new int[18*Npad]; + Np=ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); MPI_Barrier(comm); //......................device distributions................................. int dist_mem_size = Np*sizeof(double); if (rank==0) printf ("Allocating distributions \n"); - - int *NeighborList; int *dvcMap; - // double *f_even,*f_odd; double *fq, *Aq, *Bq; double *Den, *Phi; - double *ColorGrad; + double *Gradient; double *Vel; double *Pressure; //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); @@ -226,7 +217,7 @@ int main(int argc, char **argv) ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures @@ -276,33 +267,34 @@ int main(int argc, char **argv) } ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); - if (rank==0) printf ("Initializing distributions \n"); - // Initialize the phase field and variables - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); - + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf ("Initializing phase field \n"); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); + + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; printf("Check after odd time \n"); @@ -341,29 +333,29 @@ int main(int argc, char **argv) } } } + // EVEN TIMESTEP + // Compute the Phase indicator field + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); - // *************EVEN TIMESTEP************* - // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + //************************************************************************ printf("Check after even time \n"); SIZE=2*Np*sizeof(double); @@ -415,36 +407,37 @@ int main(int argc, char **argv) } ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double)); - if (rank==0) printf ("Initializing distributions \n"); - // Initialize the phase field and variables - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf ("Initializing phase field \n"); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + // Read for Aq, Bq happens in this routine (requires communication) + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); + + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - printf("Check after odd time \n"); SIZE=2*Np*sizeof(double); ScaLBL_CopyToHost(&DENSITY[0],&Den[0],SIZE); @@ -482,28 +475,29 @@ int main(int argc, char **argv) } } + // EVEN TIMESTEP + // Compute the Phase indicator field + ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + + // compute the gradient + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm.RecvGrad(Phi,Gradient); - // *************EVEN TIMESTEP************* - // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; + // Perform the collision operation + ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + //************************************************************************ printf("Check after even time \n"); SIZE=2*Np*sizeof(double); @@ -551,7 +545,6 @@ int main(int argc, char **argv) MPI_Barrier(comm); MPI_Finalize(); // **************************************************** - return check; } From 4d5dbaa1d9344058ad9d39685e5cbe8ed65b4963 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 23:14:11 -0400 Subject: [PATCH 067/124] mass bounceback closer --- tests/TestColorMassBounceback.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index edcda83e..ea5505c7 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -27,8 +27,8 @@ int main(int argc, char **argv) int check; { // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; int iproc,jproc,kproc; + int i,j,k,n,Npad; auto filename = argv[1]; auto db = std::make_shared( filename ); auto domain_db = db->getDatabase( "Domain" ); @@ -44,12 +44,6 @@ int main(int argc, char **argv) // int device=ScaLBL_SetDevice(rank); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - - PROFILE_ENABLE(1); - //PROFILE_ENABLE_TRACE(); - //PROFILE_ENABLE_MEMORY(); - PROFILE_SYNCHRONIZE(); - PROFILE_START("Main"); Utilities::setErrorHandlers(); // Variables that specify the computational domain @@ -140,8 +134,6 @@ int main(int argc, char **argv) // Full domain used for averaging (do not use mask for analysis) Domain Dm(domain_db); for (int i=0; i Averages( new TwoPhase(Dm) ); - // TwoPhase Averages(Dm); Dm.CommInit(comm); // Mask that excludes the solid phase @@ -204,6 +196,7 @@ int main(int argc, char **argv) double *fq, *Aq, *Bq; double *Den, *Phi; double *Gradient; + double *SolidPotential; double *Vel; double *Pressure; @@ -218,6 +211,7 @@ int main(int argc, char **argv) ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np); ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidPotential, 3*sizeof(double)*Np); //........................................................................... // Update GPU data structures From 87ed8a218a5e2246ef1dbdf788a3cc4ae8b5e558 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 23:33:13 -0400 Subject: [PATCH 068/124] debigging TestColorMassBounceback --- tests/TestColorMassBounceback.cpp | 52 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index ea5505c7..5082f4de 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -264,30 +264,30 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_Comm.RecvGrad(Phi,Gradient); // Perform the collision operation ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -330,23 +330,23 @@ int main(int argc, char **argv) // EVEN TIMESTEP // Compute the Phase indicator field ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_Comm.RecvGrad(Phi,Gradient); // Perform the collision operation ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ @@ -404,31 +404,31 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) // Read for Aq, Bq happens in this routine (requires communication) ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_Comm.RecvGrad(Phi,Gradient); // Perform the collision operation ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -472,23 +472,23 @@ int main(int argc, char **argv) // EVEN TIMESTEP // Compute the Phase indicator field ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_Comm.RecvGrad(Phi,Gradient); // Perform the collision operation ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ From ffb6661367f63af3b3891e2da51a389903b02c6a Mon Sep 17 00:00:00 2001 From: James E McClure Date: Wed, 16 May 2018 23:33:59 -0400 Subject: [PATCH 069/124] ScaLBL --- common/ScaLBL.cpp | 9 +++++++++ common/ScaLBL.h | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 59785a79..b2a6b0ae 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -311,6 +311,15 @@ ScaLBL_Communicator::~ScaLBL_Communicator(){ // destrutor does nothing (bad idea) // -- note that there needs to be a way to free memory allocated on the device!!! } +int ScaLBL_Communicator::LastExterior(){ + return next; +} +int ScaLBL_Communicator::FirstInterior(){ + return first_interior; +} +int ScaLBL_Communicator::LastInterior(){ + return last_interior; +} void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist){ diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 65e45338..888f1849 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -136,9 +136,10 @@ public: //...................................................................................... unsigned long int CommunicationCount,SendCount,RecvCount; int Nx,Ny,Nz,N; + int BoundaryCondition; + int next; int first_interior,last_interior; - int BoundaryCondition; //...................................................................................... // Set up for D319 distributions // - determines how much memory is allocated @@ -153,6 +154,10 @@ public: double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ; //...................................................................................... + int LastExterior(); + int FirstInterior(); + int LastInterior(); + int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, char *id, int Np); // void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np); // void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np); @@ -195,6 +200,7 @@ private: bool Lock; // use Lock to make sure only one call at a time to protect data in transit // only one set of Send requests can be active at any time (per instance) int i,j,k,n; + int iproc,jproc,kproc; int nprocx,nprocy,nprocz; int sendtag,recvtag; From 15560317910999173b076df96f9f69a2839f6886 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 00:21:08 -0400 Subject: [PATCH 070/124] adding neighborlist check to TestMap --- tests/TestMap.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index 18e9e219..7106ec9a 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -175,6 +175,14 @@ int main(int argc, char **argv) Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); MPI_Barrier(comm); + + // Check the neighborlist + for (int idx=0; idxNp) printf("error at q=%i, idx=%i \n",q,idx); + } + } //......................device distributions................................. int dist_mem_size = Np*sizeof(double); From 37468440c2fbb495b4f598d4af399b55eb4af062 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 00:27:02 -0400 Subject: [PATCH 071/124] update TestMap --- tests/TestMap.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index 7106ec9a..a8008ce3 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -177,10 +177,19 @@ int main(int argc, char **argv) MPI_Barrier(comm); // Check the neighborlist - for (int idx=0; idxNp) printf("error at q=%i, idx=%i \n",q,idx); + if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); + + } + } + for (int idx=ScaLBL_Comm.FirstInterior(); idxNp) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); + } } From 0c4318091e6bd1890d20e5ef445c24e7aa9181f6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 00:44:59 -0400 Subject: [PATCH 072/124] domain chck in TestMap.cpp --- tests/TestMap.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index a8008ce3..3bb55ae6 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -159,7 +159,26 @@ int main(int argc, char **argv) //Create a second communicator based on the regular data layout ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - + // Check that domain information properly copied to ScaLBL communicator + if (ScaLBL_Comm.sendCount_x != Dm.sendCount_x) printf("domain error x \n"); + if (ScaLBL_Comm.sendCount_X != Dm.sendCount_X) printf("domain error X \n"); + if (ScaLBL_Comm.sendCount_y != Dm.sendCount_y) printf("domain error y \n"); + if (ScaLBL_Comm.sendCount_Y != Dm.sendCount_Y) printf("domain error Y \n"); + if (ScaLBL_Comm.sendCount_z != Dm.sendCount_z) printf("domain error z \n"); + if (ScaLBL_Comm.sendCount_Z != Dm.sendCount_Z) printf("domain error Z \n"); + if (ScaLBL_Comm.sendCount_xy != Dm.sendCount_xy) printf("domain error xy \n"); + if (ScaLBL_Comm.sendCount_xY != Dm.sendCount_xY) printf("domain error xY \n"); + if (ScaLBL_Comm.sendCount_Xy != Dm.sendCount_Xy) printf("domain error Xy \n"); + if (ScaLBL_Comm.sendCount_XY != Dm.sendCount_XY) printf("domain error XY \n"); + if (ScaLBL_Comm.sendCount_xz != Dm.sendCount_xz) printf("domain error xz \n"); + if (ScaLBL_Comm.sendCount_xZ != Dm.sendCount_xZ) printf("domain error xZ \n"); + if (ScaLBL_Comm.sendCount_Xz != Dm.sendCount_Xz) printf("domain error Xz \n"); + if (ScaLBL_Comm.sendCount_XZ != Dm.sendCount_XZ) printf("domain error XZ \n"); + if (ScaLBL_Comm.sendCount_yz != Dm.sendCount_yz) printf("domain error yz \n"); + if (ScaLBL_Comm.sendCount_yZ != Dm.sendCount_yZ) printf("domain error yZ \n"); + if (ScaLBL_Comm.sendCount_Yz != Dm.sendCount_Yz) printf("domain error Yz \n"); + if (ScaLBL_Comm.sendCount_YZ != Dm.sendCount_YZ) printf("domain error YZ \n"); + if (rank==0){ printf("Total domain size = %i \n",N); printf("Reduced domain size = %i \n",Np); From 5a686278bfc08e30b1b42e436eaca3ed6e32327e Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 01:00:03 -0400 Subject: [PATCH 073/124] Map test --- common/ScaLBL.h | 2 +- tests/TestMap.cpp | 21 --------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 888f1849..f92cdb40 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -221,12 +221,12 @@ private: int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //...................................................................................... - //...................................................................................... int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z; int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ; int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ; //...................................................................................... + int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z; int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ; int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ; diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index 3bb55ae6..4efb9424 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -158,27 +158,6 @@ int main(int argc, char **argv) ScaLBL_Communicator ScaLBL_Comm(Dm); //Create a second communicator based on the regular data layout ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - - // Check that domain information properly copied to ScaLBL communicator - if (ScaLBL_Comm.sendCount_x != Dm.sendCount_x) printf("domain error x \n"); - if (ScaLBL_Comm.sendCount_X != Dm.sendCount_X) printf("domain error X \n"); - if (ScaLBL_Comm.sendCount_y != Dm.sendCount_y) printf("domain error y \n"); - if (ScaLBL_Comm.sendCount_Y != Dm.sendCount_Y) printf("domain error Y \n"); - if (ScaLBL_Comm.sendCount_z != Dm.sendCount_z) printf("domain error z \n"); - if (ScaLBL_Comm.sendCount_Z != Dm.sendCount_Z) printf("domain error Z \n"); - if (ScaLBL_Comm.sendCount_xy != Dm.sendCount_xy) printf("domain error xy \n"); - if (ScaLBL_Comm.sendCount_xY != Dm.sendCount_xY) printf("domain error xY \n"); - if (ScaLBL_Comm.sendCount_Xy != Dm.sendCount_Xy) printf("domain error Xy \n"); - if (ScaLBL_Comm.sendCount_XY != Dm.sendCount_XY) printf("domain error XY \n"); - if (ScaLBL_Comm.sendCount_xz != Dm.sendCount_xz) printf("domain error xz \n"); - if (ScaLBL_Comm.sendCount_xZ != Dm.sendCount_xZ) printf("domain error xZ \n"); - if (ScaLBL_Comm.sendCount_Xz != Dm.sendCount_Xz) printf("domain error Xz \n"); - if (ScaLBL_Comm.sendCount_XZ != Dm.sendCount_XZ) printf("domain error XZ \n"); - if (ScaLBL_Comm.sendCount_yz != Dm.sendCount_yz) printf("domain error yz \n"); - if (ScaLBL_Comm.sendCount_yZ != Dm.sendCount_yZ) printf("domain error yZ \n"); - if (ScaLBL_Comm.sendCount_Yz != Dm.sendCount_Yz) printf("domain error Yz \n"); - if (ScaLBL_Comm.sendCount_YZ != Dm.sendCount_YZ) printf("domain error YZ \n"); - if (rank==0){ printf("Total domain size = %i \n",N); printf("Reduced domain size = %i \n",Np); From 80d4ec480790bdb4a478fefebabf5c6ac9bc3878 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 09:21:28 -0400 Subject: [PATCH 074/124] refactoring to std ptr --- analysis/runAnalysis.cpp | 4 ++-- analysis/runAnalysis.h | 2 +- common/ScaLBL.cpp | 2 +- common/ScaLBL.h | 2 +- models/ColorModel.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 79208329..27235212 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -282,13 +282,13 @@ runAnalysis::commWrapper runAnalysis::getComm( ) * Constructor/Destructors * ******************************************************************/ runAnalysis::runAnalysis( std::shared_ptr db, - const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain* Dm, + const RankInfoStruct& rank_info, std::shared_ptr ScaLBL_Comm, std::shared_ptr Dm, int Np, bool pBC, double beta, IntArray Map ): d_Np( Np ), d_beta( beta ), d_rank_info( rank_info ), d_Map( Map ), - d_ScaLBL_Comm( ScaLBL_Comm ), + d_ScaLBL_Comm( ScLBL_Comm ), d_fillData(Dm->Comm,Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,1,1,1,0,1) { NULL_USE( pBC ); diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index b64256ec..90050ab3 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -24,7 +24,7 @@ public: //! Constructor runAnalysis( std::shared_ptr db, const RankInfoStruct& rank_info, - const ScaLBL_Communicator &ScaLBL_Comm, const Domain* dm, int Np, bool pBC, double beta, IntArray Map ); + std::shared_ptr ScaLBL_Comm, std::shared_ptr dm, int Np, bool pBC, double beta, IntArray Map ); //! Destructor ~runAnalysis(); diff --git a/common/ScaLBL.cpp b/common/ScaLBL.cpp index 92c11766..25085b88 100644 --- a/common/ScaLBL.cpp +++ b/common/ScaLBL.cpp @@ -1,6 +1,6 @@ #include "common/ScaLBL.h" -ScaLBL_Communicator::ScaLBL_Communicator(Domain *Dm){ +ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr Dm){ //...................................................................................... Lock=false; // unlock the communicator //...................................................................................... diff --git a/common/ScaLBL.h b/common/ScaLBL.h index 503a4aeb..5cb48723 100644 --- a/common/ScaLBL.h +++ b/common/ScaLBL.h @@ -129,7 +129,7 @@ extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int class ScaLBL_Communicator{ public: //...................................................................................... - ScaLBL_Communicator(Domain *Dm); + ScaLBL_Communicator(std::shared_ptr Dm); //ScaLBL_Communicator(Domain &Dm, IntArray &Map); ~ScaLBL_Communicator(); diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 56eebcfe..083c73c8 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -64,7 +64,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; for (int i=0; iid[i] = 1; // initialize this way - Averages = std::shared_ptr ( new TwoPhase(Dm.get()) ); // TwoPhase analysis object + Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); Dm->CommInit(comm); From d335e634ed1b6840361d67f1abf353ddca670311 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 09:36:34 -0400 Subject: [PATCH 075/124] std ptr in runAnalysis --- analysis/runAnalysis.cpp | 10 +++++----- analysis/runAnalysis.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 27235212..dccf9c3c 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -288,7 +288,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, d_beta( beta ), d_rank_info( rank_info ), d_Map( Map ), - d_ScaLBL_Comm( ScLBL_Comm ), + d_ScaLBL_Comm( ScaLBL_Comm), d_fillData(Dm->Comm,Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,1,1,1,0,1) { NULL_USE( pBC ); @@ -548,10 +548,10 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi, PROFILE_STOP("Copy-Wait",1); PROFILE_START("Copy-State",1); memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double)); - d_ScaLBL_Comm.RegularLayout(d_Map,Pressure,Averages.Press); - d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[0],Averages.Vel_x); - d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y); - d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z); + d_ScaLBL_Comm->RegularLayout(d_Map,Pressure,Averages.Press); + d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[0],Averages.Vel_x); + d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y); + d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z); PROFILE_STOP("Copy-State",1); } std::shared_ptr cDen, cfq; diff --git a/analysis/runAnalysis.h b/analysis/runAnalysis.h index 90050ab3..c5644fe2 100644 --- a/analysis/runAnalysis.h +++ b/analysis/runAnalysis.h @@ -86,7 +86,6 @@ private: int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval; double d_beta; ThreadPool d_tpool; - ScaLBL_Communicator d_ScaLBL_Comm; RankInfoStruct d_rank_info; IntArray d_Map; BlobIDstruct d_last_ids; @@ -98,6 +97,7 @@ private: MPI_Comm d_comm; MPI_Comm d_comms[1024]; volatile bool d_comm_used[1024]; + std::shared_ptr d_ScaLBL_Comm; // Ids of work items to use for dependencies ThreadPool::thread_id_t d_wait_blobID; From 507ebc86e9539c09d2fda0747fc9ee5c840c171d Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 09:43:12 -0400 Subject: [PATCH 076/124] std ptr in TwoPhase --- analysis/TwoPhase.cpp | 6 +++--- analysis/TwoPhase.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index 4028be50..201bbed2 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -54,7 +54,7 @@ #define PI 3.14159265359 // Constructor -TwoPhase::TwoPhase(Domain *dm): +TwoPhase::TwoPhase(std::shared_ptr dm): n_nw_pts(0), n_ns_pts(0), n_ws_pts(0), n_nws_pts(0), n_local_sol_pts(0), n_local_nws_pts(0), n_nw_tris(0), n_ns_tris(0), n_ws_tris(0), n_nws_seg(0), n_local_sol_tris(0), nc(0), kstart(0), kfinish(0), fluid_isovalue(0), solid_isovalue(0), Volume(0), @@ -196,8 +196,8 @@ TwoPhase::~TwoPhase() void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData) { - double factor,temp,value; - /* factor=0.5/Beta; + /*double factor,temp,value; + factor=0.5/Beta; // Initialize to -1,1 (segmentation) for (int k=0; k Dm; int NumberComponents_WP,NumberComponents_NWP; //........................................................................... // Averaging variables @@ -145,7 +145,7 @@ public: DoubleArray ComponentAverages_WP; DoubleArray ComponentAverages_NWP; //........................................................................... - TwoPhase(Domain *dm); + TwoPhase(std::shared_ptr Dm); ~TwoPhase(); void Initialize(); // void SetupCubes(Domain &Dm); From c786ef29673c09df55531877531b78a40477a109 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 09:44:52 -0400 Subject: [PATCH 077/124] std ptr in ColorModel --- models/ColorModel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 083c73c8..963305a8 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -194,7 +194,7 @@ void ScaLBL_ColorModel::Create(){ if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask.get())); + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); //Create a second communicator based on the regular data layout //ScaLBL_Communicator ScaLBL_Comm_Regular(Mask); From 51e54f297156e0e023803ba1f2fbf1c2db50c857 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 09:55:28 -0400 Subject: [PATCH 078/124] design ColorModel --- models/ColorModel.cpp | 36 ++++++++++++++++++------------------ models/ColorModel.h | 8 +++++++- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 963305a8..f6ef66eb 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -16,26 +16,26 @@ ScaLBL_ColorModel::~ScaLBL_ColorModel(){ void ScaLBL_ColorModel::ReadParams(string filename){ // read the input database - auto db = std::make_shared( filename ); - auto domain_db = db->getDatabase( "Domain" ); - auto color_db = db->getDatabase( "Color" ); - auto analysis_db = db->getDatabase( "Analysis" ); + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + color_db = db->getDatabase( "Color" ); + analysis_db = db->getDatabase( "Analysis" ); // Color Model parameters - timestepMax = domain_db->getScalar( "timestepMax" ); - tauA = domain_db->getScalar( "tauA" ); - tauB = domain_db->getScalar( "tauB" ); - rhoA = domain_db->getScalar( "rhoA" ); - rhoB = domain_db->getScalar( "rhoB" ); - Fx = domain_db->getVector( "F" )[0]; - Fy = domain_db->getVector( "F" )[1]; - Fz = domain_db->getVector( "F" )[2]; - alpha = domain_db->getScalar( "alpha" ); - beta = domain_db->getScalar( "beta" ); - Restart = domain_db->getScalar( "Restart" ); - din = domain_db->getScalar( "din" ); - dout = domain_db->getScalar( "dout" ); - flux = domain_db->getScalar( "flux" );; + timestepMax = color_db->getScalar( "timestepMax" ); + tauA = color_db->getScalar( "tauA" ); + tauB = color_db->getScalar( "tauB" ); + rhoA = color_db->getScalar( "rhoA" ); + rhoB = color_db->getScalar( "rhoB" ); + Fx = color_db->getVector( "F" )[0]; + Fy = color_db->getVector( "F" )[1]; + Fz = color_db->getVector( "F" )[2]; + alpha = color_db->getScalar( "alpha" ); + beta = color_db->getScalar( "beta" ); + Restart = color_db->getScalar( "Restart" ); + din = color_db->getScalar( "din" ); + dout = color_db->getScalar( "dout" ); + flux = color_db->getScalar( "flux" );; inletA=1.f; inletB=0.f; outletA=0.f; diff --git a/models/ColorModel.h b/models/ColorModel.h index 8fcfa299..c371297a 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -45,7 +45,13 @@ private: std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; std::shared_ptr Averages; - + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr color_db; + std::shared_ptr analysis_db; + // filenames char LocalRankString[8]; char LocalRankFilename[40]; From b3626d3edd49c27b1a199974c9b10ca5a01668aa Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 10:14:49 -0400 Subject: [PATCH 079/124] refactor color bubble --- tests/TestColorBubble.cpp | 436 +------------------------------------- 1 file changed, 6 insertions(+), 430 deletions(-) diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 72161254..6f0438b1 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -98,442 +98,18 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); int check; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); printf("Running Color Model: TestColor \n"); printf("********************************************************\n"); - } - - // BGK Model parameters - string FILENAME; - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double Fx,Fy,Fz,tol; - // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; - int i,j,k,n; - int dim = 50; - //if (rank == 0) printf("dim=%d\n",dim); - int timestep = 0; - int timesteps = 100; - int centralNode = 2; - - double tauA = 1.0; - double tauB = 1.0; - double rhoA = 1.0; - double rhoB = 1.0; - double alpha = 0.001; - double beta = 0.95; - - double tau = 1.0; - double mu=(tau-0.5)/3.0; - double rlx_setA=1.0/tau; - double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - - Fx = Fy = 0.f; - Fz = 0.f; - - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=3; Ny = 1; - Nz = 1; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=2; nprocy=1; - nprocz=1; - Nx=Ny=Nz=dim; - Nx = dim; Ny = dim; Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } - MPI_Barrier(comm); - - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - int BoundaryCondition=0; - - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - - Nx += 2; - Ny += 2; - Nz += 2; - int N = Nx*Ny*Nz; - - //....................................................................... - // Assign the phase ID field - //....................................................................... - char LocalRankString[8]; - sprintf(LocalRankString,"%05d",rank); - char LocalRankFilename[40]; - sprintf(LocalRankFilename,"ID.%05i",rank); - - for (k=0;k 0){ - sum_local+=1.0; - Np++; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = sum*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - MPI_Barrier(comm); - - // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Dm); - //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - - //...........device phase ID................................................. - if (rank==0) printf ("Copying phase ID to device \n"); - char *ID; - ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory - // Copy to the device - ScaLBL_CopyToDevice(ID, Dm.id, N); - //........................................................................... - - if (rank==0){ - printf("Total domain size = %i \n",N); - printf("Reduced domain size = %i \n",Np); - } - - // LBM variables - if (rank==0) printf ("Set up the neighborlist \n"); - - int neighborSize=18*Np*sizeof(int); - int *neighborList; - IntArray Map(Nx,Ny,Nz); - neighborList= new int[18*Np]; - - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); - MPI_Barrier(comm); - - //......................device distributions................................. - int dist_mem_size = Np*sizeof(double); - if (rank==0) printf ("Allocating distributions \n"); - - int *NeighborList; - int *dvcMap; - // double *f_even,*f_odd; - double *fq, *Aq, *Bq; - double *Den, *Phi; - double *ColorGrad; - double *Vel; - double *Pressure; - - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); - - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); - int *TmpMap; - TmpMap=new int[Np]; - for (k=1; k 0){ - int idx = Map(i,j,k); - sum_local+=VEL[2*Np+idx]; - } - } - } - } - - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - double PoreVel = sum*iVol_global; - if (rank==0) printf("Velocity = %f \n",PoreVel); - - /* - double *PHASE; - PHASE= new double [Nx*Ny*Nz]; - SIZE=Nx*Ny*Nz*sizeof(double); - ScaLBL_CopyToHost(&PHASE[0],&Phi[0],SIZE); - - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PHASE,8,N,OUTFILE); - fclose(OUTFILE); - */ - + auto filename = argv[1]; + ScaLBL_ColorModel() ColorModel; + ColorModel.ReadParams(filename); } // **************************************************** MPI_Barrier(comm); From ebbccdaf7c94fee169bfe6c79985295e946826b2 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 10:18:57 -0400 Subject: [PATCH 080/124] refactor color bubble --- models/ColorModel.h | 1 + tests/TestColorBubble.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index c371297a..03f7fa32 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -23,6 +23,7 @@ public: // functions in they should be run void ReadParams(string filename); + void ReadParams(std::shared_ptr DB) void ReadInput(); void Create(); void Initialize(); diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 6f0438b1..fd0d21bd 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -8,6 +8,7 @@ #include #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" +#include "models/ColorModel.h" using namespace std; @@ -108,7 +109,7 @@ int main(int argc, char **argv) } } auto filename = argv[1]; - ScaLBL_ColorModel() ColorModel; + ScaLBL_ColorModel ColorModel; ColorModel.ReadParams(filename); } // **************************************************** From 0fb8f310efd6c58daaa324c6d7b8885ce62bf112 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 10:27:04 -0400 Subject: [PATCH 081/124] color bubble --- models/ColorModel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index 03f7fa32..ce0cb586 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -23,7 +23,7 @@ public: // functions in they should be run void ReadParams(string filename); - void ReadParams(std::shared_ptr DB) + void ReadParams(std::shared_ptr DB); void ReadInput(); void Create(); void Initialize(); From 7483b93e50020a29b9bfb1f7247090d3eb44b890 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 10:27:42 -0400 Subject: [PATCH 082/124] database initialization for color model --- models/ColorModel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index 03f7fa32..c507cd92 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -23,7 +23,7 @@ public: // functions in they should be run void ReadParams(string filename); - void ReadParams(std::shared_ptr DB) + void ReadParams(std::shared_ptr db0) void ReadInput(); void Create(); void Initialize(); @@ -71,6 +71,7 @@ private: double *Pressure; //int rank,nprocs; + void LoadParams(std::shared_ptr db0) }; From 232ca742ad903dd5182b3a8eb683075dfa44f0e2 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 10:34:36 -0400 Subject: [PATCH 083/124] color param init --- models/ColorModel.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index c507cd92..dd37d7cd 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -23,7 +23,7 @@ public: // functions in they should be run void ReadParams(string filename); - void ReadParams(std::shared_ptr db0) + void ReadParams(std::shared_ptr db0); void ReadInput(); void Create(); void Initialize(); @@ -71,7 +71,7 @@ private: double *Pressure; //int rank,nprocs; - void LoadParams(std::shared_ptr db0) + void LoadParams(std::shared_ptr db0); }; From 84be31aeafed823fd71fe8dfd3778ddc57088a60 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 11:01:21 -0400 Subject: [PATCH 084/124] changed colormodel vars from private to public --- models/ColorModel.h | 20 ++++++++++---------- tests/TestColorBubble.cpp | 4 ++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/models/ColorModel.h b/models/ColorModel.h index dd37d7cd..f99b683e 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -39,9 +39,7 @@ public: int Nx,Ny,Nz,N,Np; int nprocx,nprocy,nprocz; double Lx,Ly,Lz; - -private: - MPI_Comm comm; + std::shared_ptr Dm; // this domain is for analysis std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; @@ -52,14 +50,8 @@ private: std::shared_ptr domain_db; std::shared_ptr color_db; std::shared_ptr analysis_db; - - // filenames - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - - IntArray Map; + IntArray Map; char *id; int *NeighborList; int *dvcMap; @@ -69,7 +61,15 @@ private: double *Velocity; double *Gradient; double *Pressure; + +private: + MPI_Comm comm; + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + //int rank,nprocs; void LoadParams(std::shared_ptr db0); diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index fd0d21bd..6f6b211f 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -111,6 +111,10 @@ int main(int argc, char **argv) auto filename = argv[1]; ScaLBL_ColorModel ColorModel; ColorModel.ReadParams(filename); + ColorModel.ReadInput(); + ColorModel.Create(); + ColorModel.Initialize(); + ColorModel.Run(); } // **************************************************** MPI_Barrier(comm); From 195481f615ce49eb7a788f97aa6d60655a1b7fac Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 17 May 2018 12:07:20 -0400 Subject: [PATCH 085/124] Moving to a new signed distance function --- analysis/TwoPhase.cpp | 2 +- analysis/analysis.cpp | 4 +- analysis/distance.cpp | 213 ++++++++++++++++++ analysis/distance.h | 40 ++++ analysis/eikonal.cpp | 384 --------------------------------- analysis/eikonal.h | 53 ----- analysis/runAnalysis.cpp | 2 +- analysis/uCT.cpp | 22 +- common/Communication.h | 25 ++- common/Communication.hpp | 157 +++++++++----- tests/BlobIdentifyParallel.cpp | 2 +- tests/ComponentLabel.cpp | 2 +- tests/TestBlobIdentify.cpp | 8 +- tests/TestSegDist.cpp | 156 ++++++-------- tests/lbpm_morph_pp.cpp | 72 +++---- tests/lbpm_morphdrain_pp.cpp | 2 +- tests/lbpm_random_pp.cpp | 2 +- tests/lbpm_refine_pp.cpp | 2 +- tests/lbpm_segmented_pp.cpp | 11 +- tests/testCommunication.cpp | 2 +- 20 files changed, 502 insertions(+), 659 deletions(-) create mode 100644 analysis/distance.cpp create mode 100644 analysis/distance.h delete mode 100644 analysis/eikonal.cpp delete mode 100644 analysis/eikonal.h diff --git a/analysis/TwoPhase.cpp b/analysis/TwoPhase.cpp index 492c295e..b2abda0e 100644 --- a/analysis/TwoPhase.cpp +++ b/analysis/TwoPhase.cpp @@ -232,7 +232,7 @@ void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, Double } } - Eikonal(DistData,TempID,Dm,50); + CalcDist(DistData,TempID,Dm); for (int k=0; k fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true); + fillHalo fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1,{true,true,true}); fillData.fill(IDs); // Create a list of all neighbor ranks (excluding self) std::vector neighbors; @@ -420,7 +420,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_ } } // Fill the ghosts - fillHalo fillData2(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true); + fillHalo fillData2(comm,rank_info,{nx,ny,nz},{1,1,1},0,1,{true,true,true}); fillData2.fill(IDs); // Reorder based on size (and compress the id space int N_blobs_global = ReorderBlobIDs2(IDs,N_blobs_tot,ngx,ngy,ngz,comm); diff --git a/analysis/distance.cpp b/analysis/distance.cpp new file mode 100644 index 00000000..c5aa4d1e --- /dev/null +++ b/analysis/distance.cpp @@ -0,0 +1,213 @@ +#include "analysis/distance.h" + + +// Check if we need to recompute distance after updating ghose values +static bool checkUpdate( const Array &d, double dx, double dy, double dz ) +{ + auto s = d.size(); + bool test[3] = { false, false, false }; + // Check x-direction + Vec v1, v2; + for (size_t k=1; k +void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic ) +{ + ASSERT( Distance.size() == ID.size() ); + std::array n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 }; + fillHalo fillData( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,false,false}, periodic ); + Array id(ID.size()); + Array vecDist(Distance.size()); + for (size_t i=0; i &d, const Array &ID0, const Domain &Dm, const std::array& periodic ) +{ + const double dx = 1.0; + const double dy = 1.0; + const double dz = 1.0; + std::array N = { Dm.Nx, Dm.Ny, Dm.Nz }; + std::array n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 }; + // Create ID with ghosts + Array ID(N[0],N[1],N[2]); + fillHalo fillDataID( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,true,true}, periodic ); + fillDataID.copy( ID0, ID ); + // Fill ghosts with nearest neighbor + for (int k=1; k fillData( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,false,false}, periodic ); + // Initialize the vector distance + d.fill( Vec( 1e50, 1e50, 1e50 ) ); + const double dx0 = 0.5*dx; + const double dy0 = 0.5*dy; + const double dz0 = 0.5*dz; + //const double dxy0 = 0.25*sqrt( dx*dx + dy*dy ); + //const double dxz0 = 0.25*sqrt( dx*dx + dz*dz ); + //const double dyz0 = 0.25*sqrt( dy*dy + dz*dz ); + //const double dxyz0 = sqrt( dx*dx + dy*dy + dz*dz ); + for (int k=1; k=0; i--) { + auto v1 = d(i,j,k); + auto v2 = d(i+1,j,k); + v2.x -= dx; + if ( v2 < v1 ) + d(i,j,k) = v2; + } + } + } + // Propagate +/- y-direction + for (int k=0; k=0; j--) { + auto v1 = d(i,j,k); + auto v2 = d(i,j+1,k); + v2.y -= dy; + if ( v2 < v1 ) + d(i,j,k) = v2; + } + } + } + // Propagate +/- z-direction + for (int j=0; j=0; k--) { + auto v1 = d(i,j,k); + auto v2 = d(i,j,k+1); + v2.z -= dz; + if ( v2 < v1 ) + d(i,j,k) = v2; + } + } + } + fillData.fill( d ); + bool test = checkUpdate( d, dx, dy, dz ); + test = sumReduce( Dm.Comm, test ); + if ( !test ) + break; + } +} +template void CalcDist( Array&, const Array&, const Domain&, const std::array& ); +template void CalcDist( Array&, const Array&, const Domain&, const std::array& ); + + diff --git a/analysis/distance.h b/analysis/distance.h new file mode 100644 index 00000000..b1bb9af5 --- /dev/null +++ b/analysis/distance.h @@ -0,0 +1,40 @@ +#ifndef Distance_H_INC +#define Distance_H_INC + +#include "common/Domain.h" + + +struct Vec { + double x; + double y; + double z; + inline Vec(): x(0), y(0), z(0) {} + inline Vec( double x_, double y_, double z_ ): x(x_), y(y_), z(z_) {} + inline double norm() const { return sqrt(x*x+y*y+z*z); } +}; +inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; } + + +/*! + * @brief Calculate the distance using a simple method + * @details This routine calculates the vector distance to the nearest domain surface. + * @param[out] Distance Distance function + * @param[in] ID Segmentation id + * @param[in] Dm Domain information + * @param[in] periodic Directions that are periodic + */ +template +void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic = {true,true,true} ); + +/*! + * @brief Calculate the distance using a simple method + * @details This routine calculates the vector distance to the nearest domain surface. + * @param[out] Distance Distance function + * @param[in] ID Domain id + * @param[in] Dm Domain information + * @param[in] periodic Directions that are periodic + */ +void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic ); + + +#endif diff --git a/analysis/eikonal.cpp b/analysis/eikonal.cpp deleted file mode 100644 index 01e00405..00000000 --- a/analysis/eikonal.cpp +++ /dev/null @@ -1,384 +0,0 @@ -#include "analysis/eikonal.h" -#include "analysis/imfilter.h" - - - - -static inline float minmod(float &a, float &b) -{ - float value = a; - if ( a*b < 0.0) - value=0.0; - else if (fabs(a) > fabs(b)) - value = b; - return value; -} - - -static inline double minmod(double &a, double &b){ - - double value; - - value = a; - if ( a*b < 0.0) value=0.0; - else if (fabs(a) > fabs(b)) value = b; - - return value; -} - - -template -static inline MPI_Datatype getType( ); -template<> inline MPI_Datatype getType() { return MPI_FLOAT; } -template<> inline MPI_Datatype getType() { return MPI_DOUBLE; } - - -/****************************************************************** -* Solve the eikonal equation * -******************************************************************/ -template -TYPE Eikonal( Array &Distance, const Array &ID, const Domain &Dm, const int timesteps) -{ - - /* - * This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - * - * Reference: - * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 - */ - - int xdim = Dm.Nx-2; - int ydim = Dm.Ny-2; - int zdim = Dm.Nz-2; - fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); - - // Arrays to store the second derivatives - Array Dxx(Dm.Nx,Dm.Ny,Dm.Nz); - Array Dyy(Dm.Nx,Dm.Ny,Dm.Nz); - Array Dzz(Dm.Nx,Dm.Ny,Dm.Nz); - Array sign(ID.size()); - for (size_t i=0; i 0.f) - Dx = Dxp*Dxp; - else - Dx = Dxm*Dxm; - if (Dyp + Dym > 0.f) - Dy = Dyp*Dyp; - else - Dy = Dym*Dym; - if (Dzp + Dzm > 0.f) - Dz = Dzp*Dzp; - else - Dz = Dzm*Dzm; - } - else{ - if (Dxp + Dxm < 0.f) - Dx = Dxp*Dxp; - else - Dx = Dxm*Dxm; - if (Dyp + Dym < 0.f) - Dy = Dyp*Dyp; - else - Dy = Dym*Dym; - if (Dzp + Dzm < 0.f) - Dz = Dzp*Dzp; - else - Dz = Dzm*Dzm; - } - - //Dx = max(Dxp*Dxp,Dxm*Dxm); - //Dy = max(Dyp*Dyp,Dym*Dym); - //Dz = max(Dzp*Dzp,Dzm*Dzm); - - double norm = sqrt(Dx + Dy + Dz); - if (norm > 1.0) - norm = 1.0; - - Distance(i,j,k) += dt*s*(1.0 - norm); - LocalVar += dt*s*(1.0 - norm); - - if (fabs(dt*s*(1.0 - norm)) > LocalMax) - LocalMax = fabs(dt*s*(1.0 - norm)); - } - } - } - - double GlobalMax; - MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); - MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm); - GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx()*Dm.nprocy()*Dm.nprocz(); - count++; - - - if (count%50 == 0 && Dm.rank()==0 ){ - printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar); - fflush(stdout); - } - - if (fabs(GlobalMax) < 1e-5){ - if (Dm.rank()==0) - printf("Exiting with max tolerance of 1e-5 \n"); - count=timesteps; - } - } - return GlobalVar; -} -template float Eikonal( Array&, const Array&, const Domain&, int ); -template double Eikonal( Array&, const Array&, const Domain&, int ); - - -/****************************************************************** -* A fast distance calculation * -******************************************************************/ -bool CalcDist3DIteration( Array &Distance, const Domain & ) -{ - const float sq2 = sqrt(2.0f); - const float sq3 = sqrt(3.0f); - float dist0[27]; - dist0[0] = sq3; dist0[1] = sq2; dist0[2] = sq3; - dist0[3] = sq2; dist0[4] = 1; dist0[5] = sq2; - dist0[6] = sq3; dist0[7] = sq2; dist0[8] = sq3; - dist0[9] = sq2; dist0[10] = 1; dist0[11] = sq2; - dist0[12] = 1; dist0[13] = 0; dist0[14] = 1; - dist0[15] = sq2; dist0[16] = 1; dist0[17] = sq2; - dist0[18] = sq3; dist0[19] = sq2; dist0[20] = sq3; - dist0[21] = sq2; dist0[22] = 1; dist0[23] = sq2; - dist0[24] = sq3; dist0[25] = sq2; dist0[26] = sq3; - bool changed = false; - for (size_t k=1; k &Distance, const Array &ID, const Domain &Dm ) -{ - PROFILE_START("Calc Distance"); - // Initialize the distance to be 0 fore the cells adjacent to the interface - Distance.fill(1e100); - for (size_t k=1; k fillData(Dm.Comm, Dm.rank_info,Dm.Nx,Dm.Ny,Dm.Nz,1,1,1,0,1); - while ( true ) { - // Communicate the halo of values - fillData.fill(Distance); - // The distance of the cell is the minimum of the distance of the neighbors plus the distance to that node - bool changed = CalcDist3DIteration( Distance, Dm ); - changed = sumReduce(Dm.Comm,changed); - if ( !changed ) - break; - } - // Update the sign of the distance - for (size_t i=0; i0 ? 1:-1; - PROFILE_STOP("Calc Distance"); -} - - -/****************************************************************** -* A fast distance calculation * -******************************************************************/ -void CalcDistMultiLevelHelper( Array &Distance, const Domain &Dm ) -{ - size_t ratio = 4; - std::function&)> coarsen = [ratio]( const Array& data ) - { - float tmp = 1e100; - int nx = data.size(0); - int ny = data.size(1); - int nz = data.size(2); - for (int k=0; k fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); - if ( Nx%ratio==0 && Nx>8 && Ny%ratio==0 && Ny>8 && Nz%ratio==0 && Nz>8 ) { - // Use recursive version - int Nr = std::max(std::max(ratio,ratio),ratio); - // Run Nr iterations, communicate, run Nr iterations - for (int i=0; i dist(Nx,Ny,Nz); - fillData.copy(Distance,dist); - auto db = Dm.getDatabase()->cloneDatabase(); - auto n = db->getVector( "n" ); - db->putVector( "n", { n[0]/ratio, n[1]/ratio, n[2]/ratio } ); - Domain Dm2(db); - Dm2.CommInit(Dm.Comm); - fillHalo fillData2(Dm2.Comm,Dm2.rank_info,Nx/ratio,Ny/ratio,Nz/ratio,1,1,1,0,1); - auto dist2 = dist.coarsen( {ratio,ratio,ratio}, coarsen ); - Array Distance2(Nx/ratio+2,Ny/ratio+2,Nz/ratio+2); - fillData2.copy(dist2,Distance2); - // Solve - CalcDistMultiLevelHelper( Distance2, Dm2 ); - // Interpolate the coarse grid to the fine grid - fillData2.copy(Distance2,dist2); - for (int k=0; k &Distance, const Array &ID, const Domain &Dm ) -{ - PROFILE_START("Calc Distance Multilevel"); - int Nx = Dm.Nx-2; - int Ny = Dm.Ny-2; - int Nz = Dm.Nz-2; - ASSERT(int(Distance.size(0))==Nx+2&&int(Distance.size(1))==Ny+2&&int(Distance.size(2))==Nz+2); - fillHalo fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); - // Initialize the distance to be 0 fore the cells adjacent to the interface - Distance.fill(1e100); - for (size_t k=1; k0 ? 1:-1; - fillData.fill(Distance); - // Run a quick filter to smooth the data - float sigma = 0.6; - Array H = imfilter::create_filter( { 1 }, "gaussian", &sigma ); - std::vector BC(3,imfilter::BC::replicate); - Distance = imfilter::imfilter_separable( Distance, {H,H,H}, BC ); - PROFILE_STOP("Calc Distance Multilevel"); -} diff --git a/analysis/eikonal.h b/analysis/eikonal.h deleted file mode 100644 index d89a7c0e..00000000 --- a/analysis/eikonal.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef Eikonal_H_INC -#define Eikonal_H_INC - -#include "common/Domain.h" - - -/*! - * @brief Calculate the distance solving the Eikonal equation - * @details This routine converts the data in the Distance array to a signed distance - * by solving the equation df/dt = sign*(1-|grad f|), where Distance provides - * the values of f on the mesh associated with domain Dm - * It has been tested with segmented data initialized to values [-1,1] - * and will converge toward the signed distance to the surface bounding the associated phases - * - * Reference: - * Min C (2010) On reinitializing level set functions, Journal of Computational Physics 229 - * - * @param[in/out] Distance Distance function - * @param[in] ID Segmentation id - * @param[in] DM Domain information - * @param[in] timesteps Maximum number of timesteps to process - * @return Returns the global variation - */ -template -TYPE Eikonal( Array &Distance, const Array &ID, const Domain &Dm, int timesteps); - - -/*! - * @brief Calculate the distance using a simple method - * @details This routine calculates the distance using a very simple method working off the segmentation id. - * - * @param[in/out] Distance Distance function - * @param[in] ID Segmentation id - * @param[in] DM Domain information - * @return Returns the global variation - */ -void CalcDist3D( Array &Distance, const Array &ID, const Domain &Dm ); - - -/*! - * @brief Calculate the distance using a multi-level method - * @details This routine calculates the distance using a multi-grid method - * - * @return Returns the number of cubes in the blob - * @param[in/out] Distance Distance function - * @param[in] ID Segmentation id - * @param[in] DM Domain information - * @return Returns the global variation - */ -void CalcDistMultiLevel( Array &Distance, const Array &ID, const Domain &Dm ); - - -#endif diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index 2f76b38c..f28697ff 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -289,7 +289,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, d_ScaLBL_Comm( ScaLBL_Comm ), d_rank_info( rank_info ), d_Map( Map ), - d_fillData(Dm.Comm,Dm.rank_info,Dm.Nx-2,Dm.Ny-2,Dm.Nz-2,1,1,1,0,1) + d_fillData(Dm.Comm,Dm.rank_info,{Dm.Nx-2,Dm.Ny-2,Dm.Nz-2},{1,1,1},0,1) { NULL_USE( pBC ); INSIST( db, "Input database is empty" ); diff --git a/analysis/uCT.cpp b/analysis/uCT.cpp index 4ee879af..172de263 100644 --- a/analysis/uCT.cpp +++ b/analysis/uCT.cpp @@ -1,6 +1,6 @@ #include "analysis/uCT.h" #include "analysis/analysis.h" -#include "analysis/eikonal.h" +#include "analysis/distance.h" #include "analysis/filters.h" #include "analysis/imfilter.h" @@ -149,7 +149,7 @@ void solve( const Array& VOL, Array& Mean, Array& ID, fillFloat.fill( Mean ); segment( Mean, ID, 0.01 ); // Compute the distance using the segmented volume - Eikonal( Dist, ID, Dm, ID.size(0)*nprocx ); + CalcDist( Dist, ID, Dm ); fillFloat.fill(Dist); smooth( VOL, Dist, 2.0, MultiScaleSmooth, fillFloat ); // Compute non-local mean @@ -210,9 +210,7 @@ void refine( const Array& Dist_coarse, //removeDisconnected( ID, Dm ); // Compute the distance using the segmented volume if ( level > 0 ) { - //Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx ); - //CalcDist3D( Dist, ID, Dm ); - CalcDistMultiLevel( Dist, ID, Dm ); + CalcDist( Dist, ID, Dm ); fillFloat.fill(Dist); } } @@ -232,7 +230,7 @@ void filter_final( Array& ID, Array& Dist, int Ny = Dm.Ny-2; int Nz = Dm.Nz-2; // Calculate the distance - CalcDistMultiLevel( Dist, ID, Dm ); + CalcDist( Dist, ID, Dm ); fillFloat.fill(Dist); // Compute the range to shrink the volume based on the L2 norm of the distance Array Dist0(Nx,Ny,Nz); @@ -256,8 +254,8 @@ void filter_final( Array& ID, Array& Dist, } //Array Dist1 = Dist; //Array Dist2 = Dist; - CalcDistMultiLevel( Dist1, ID1, Dm ); - CalcDistMultiLevel( Dist2, ID2, Dm ); + CalcDist( Dist1, ID1, Dm ); + CalcDist( Dist2, ID2, Dm ); fillFloat.fill(Dist1); fillFloat.fill(Dist2); // Keep those regions that are within dx2 of the new volumes @@ -272,8 +270,8 @@ void filter_final( Array& ID, Array& Dist, } } // Find regions of uncertainty that are entirely contained within another region - fillHalo fillDouble(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); - fillHalo fillInt(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); + fillHalo fillDouble(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1); + fillHalo fillInt(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1); BlobIDArray GlobalBlobID; DoubleArray SignDist(ID.size()); for (size_t i=0; i& ID, Array& Dist, // Perform the final segmentation and update the distance fillFloat.fill(Mean); segment( Mean, ID, 0.01 ); - CalcDistMultiLevel( Dist, ID, Dm ); + CalcDist( Dist, ID, Dm ); fillFloat.fill(Dist); } @@ -356,7 +354,7 @@ void filter_src( const Domain& Dm, Array& src ) int Nx = Dm.Nx-2; int Ny = Dm.Ny-2; int Nz = Dm.Nz-2; - fillHalo fillFloat(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); + fillHalo fillFloat(Dm.Comm,Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1); // Perform a hot-spot filter on the data std::vector BC = { imfilter::BC::replicate, imfilter::BC::replicate, imfilter::BC::replicate }; std::function&)> filter_3D = []( const Array& data ) diff --git a/common/Communication.h b/common/Communication.h index 12241be3..17a6f042 100644 --- a/common/Communication.h +++ b/common/Communication.h @@ -5,6 +5,8 @@ #include "common/Utilities.h" #include "common/Array.h" +#include + // ********** COMMUNICTION ************************************** /* //.............................................................. @@ -43,18 +45,17 @@ public: /*! * @brief Default constructor * @param[in] info Rank and neighbor rank info - * @param[in] nx Number of local cells in the x direction - * @param[in] ny Number of local cells in the y direction - * @param[in] nz Number of local cells in the z direction - * @param[in] ngx Number of ghost cells in the x direction - * @param[in] ngy Number of ghost cells in the y direction - * @param[in] ngz Number of ghost cells in the z direction + * @param[in] n Number of local cells + * @param[in] ng Number of ghost cells * @param[in] tag Initial tag to use for the communication (we will require tag:tag+26) * @param[in] depth Maximum depth to support + * @param[in] fill Fill {faces,edges,corners} + * @param[in] periodic Periodic dimensions */ - fillHalo( MPI_Comm comm, const RankInfoStruct& info, int nx, int ny, int nz, - int ngx, int ngy, int ngz, int tag, int depth, - bool fill_face=true, bool fill_edge=true, bool fill_corner=true ); + fillHalo( MPI_Comm comm, const RankInfoStruct& info, + std::array n, std::array ng, int tag, int depth, + std::array fill = {true,true,true}, + std::array periodic = {true,true,true} ); //! Destructor ~fillHalo( ); @@ -75,15 +76,17 @@ public: private: + MPI_Comm comm; RankInfoStruct info; - int nx, ny, nz, ngx, ngy, ngz, depth; + std::array n, ng; + int depth; bool fill_pattern[3][3][3]; int tag[3][3][3]; int N_send_recv[3][3][3]; TYPE *mem; TYPE *send[3][3][3], *recv[3][3][3]; MPI_Request send_req[3][3][3], recv_req[3][3][3]; - MPI_Comm comm; + size_t N_type; MPI_Datatype datatype; fillHalo(); // Private empty constructor fillHalo(const fillHalo&); // Private copy constructor diff --git a/common/Communication.hpp b/common/Communication.hpp index 26139c66..a8eff9ee 100644 --- a/common/Communication.hpp +++ b/common/Communication.hpp @@ -11,16 +11,30 @@ * Structure to store the rank info * ********************************************************/ template -fillHalo::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, int ny0, int nz0, - int ngx0, int ngy0, int ngz0, int tag0, int depth0, - bool fill_face, bool fill_edge, bool fill_corner ): - info(info0), nx(nx0), ny(ny0), nz(nz0), ngx(ngx0), ngy(ngy0), ngz(ngz0), depth(depth0) +fillHalo::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_, + std::array n_, std::array ng_, int tag0, int depth_, + std::array fill, std::array periodic ): + comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_) { - comm = comm0; - datatype = getMPItype(); + if ( std::is_same() ) { + N_type = 1; + datatype = MPI_DOUBLE; + } else if ( std::is_same() ) { + N_type = 1; + datatype = MPI_FLOAT; + } else if ( sizeof(TYPE)%sizeof(double)==0 ) { + N_type = sizeof(TYPE) / sizeof(double); + datatype = MPI_DOUBLE; + } else if ( sizeof(TYPE)%sizeof(float)==0 ) { + N_type = sizeof(TYPE) / sizeof(float); + datatype = MPI_FLOAT; + } else { + N_type = sizeof(TYPE); + datatype = MPI_BYTE; + } // Set the fill pattern memset(fill_pattern,0,sizeof(fill_pattern)); - if ( fill_face ) { + if ( fill[0] ) { fill_pattern[0][1][1] = true; fill_pattern[2][1][1] = true; fill_pattern[1][0][1] = true; @@ -28,7 +42,7 @@ fillHalo::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, fill_pattern[1][1][0] = true; fill_pattern[1][1][2] = true; } - if ( fill_edge ) { + if ( fill[1] ) { fill_pattern[0][0][1] = true; fill_pattern[0][2][1] = true; fill_pattern[2][0][1] = true; @@ -42,7 +56,7 @@ fillHalo::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, fill_pattern[1][2][0] = true; fill_pattern[1][2][2] = true; } - if ( fill_corner ) { + if ( fill[2] ) { fill_pattern[0][0][0] = true; fill_pattern[0][0][2] = true; fill_pattern[0][2][0] = true; @@ -52,13 +66,50 @@ fillHalo::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, fill_pattern[2][2][0] = true; fill_pattern[2][2][2] = true; } + // Remove communication for non-perioidic directions + if ( !periodic[0] && info.ix==0 ) { + for (int j=0; j<3; j++) { + for (int k=0; k<3; k++) + fill_pattern[0][j][k] = false; + } + } + if ( !periodic[0] && info.ix==info.nx-1 ) { + for (int j=0; j<3; j++) { + for (int k=0; k<3; k++) + fill_pattern[2][j][k] = false; + } + } + if ( !periodic[1] && info.jy==0 ) { + for (int i=0; i<3; i++) { + for (int k=0; k<3; k++) + fill_pattern[i][0][k] = false; + } + } + if ( !periodic[1] && info.jy==info.ny-1 ) { + for (int i=0; i<3; i++) { + for (int k=0; k<3; k++) + fill_pattern[i][2][k] = false; + } + } + if ( !periodic[2] && info.kz==0 ) { + for (int i=0; i<3; i++) { + for (int j=0; j<3; j++) + fill_pattern[i][j][0] = false; + } + } + if ( !periodic[2] && info.kz==info.nz-1 ) { + for (int i=0; i<3; i++) { + for (int j=0; j<3; j++) + fill_pattern[i][j][2] = false; + } + } // Determine the number of elements for each send/recv for (int i=0; i<3; i++) { - int ni = (i-1)==0 ? nx:ngx; + int ni = (i-1)==0 ? n[0]:ng[0]; for (int j=0; j<3; j++) { - int nj = (j-1)==0 ? ny:ngy; + int nj = (j-1)==0 ? n[1]:ng[1]; for (int k=0; k<3; k++) { - int nk = (k-1)==0 ? nz:ngz; + int nk = (k-1)==0 ? n[2]:ng[2]; if ( fill_pattern[i][j][k] ) N_send_recv[i][j][k] = ni*nj*nk; else @@ -106,9 +157,9 @@ void fillHalo::fill( Array& data ) { //PROFILE_START("fillHalo::fill",1); int depth2 = data.size(3); - ASSERT((int)data.size(0)==nx+2*ngx); - ASSERT((int)data.size(1)==ny+2*ngy); - ASSERT((int)data.size(2)==nz+2*ngz); + ASSERT((int)data.size(0)==n[0]+2*ng[0]); + ASSERT((int)data.size(1)==n[1]+2*ng[1]); + ASSERT((int)data.size(2)==n[2]+2*ng[2]); ASSERT(depth2<=depth); ASSERT(data.ndim()==3||data.ndim()==4); // Start the recieves @@ -117,7 +168,7 @@ void fillHalo::fill( Array& data ) for (int k=0; k<3; k++) { if ( !fill_pattern[i][j][k] ) continue; - MPI_Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k], datatype, + MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype, info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] ); } } @@ -129,7 +180,7 @@ void fillHalo::fill( Array& data ) if ( !fill_pattern[i][j][k] ) continue; pack( data, i-1, j-1, k-1, send[i][j][k] ); - MPI_Isend( send[i][j][k], depth2*N_send_recv[i][j][k], datatype, + MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype, info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] ); } } @@ -162,12 +213,12 @@ template void fillHalo::pack( const Array& data, int i0, int j0, int k0, TYPE *buffer ) { int depth2 = data.size(3); - int ni = i0==0 ? nx:ngx; - int nj = j0==0 ? ny:ngy; - int nk = k0==0 ? nz:ngz; - int is = i0==0 ? ngx:((i0==-1)?ngx:nx); - int js = j0==0 ? ngy:((j0==-1)?ngy:ny); - int ks = k0==0 ? ngz:((k0==-1)?ngz:nz); + int ni = i0==0 ? n[0]:ng[0]; + int nj = j0==0 ? n[1]:ng[1]; + int nk = k0==0 ? n[2]:ng[2]; + int is = i0==0 ? ng[0]:((i0==-1)?ng[0]:n[0]); + int js = j0==0 ? ng[1]:((j0==-1)?ng[1]:n[1]); + int ks = k0==0 ? ng[2]:((k0==-1)?ng[2]:n[2]); for (int d=0; d void fillHalo::unpack( Array& data, int i0, int j0, int k0, const TYPE *buffer ) { int depth2 = data.size(3); - int ni = i0==0 ? nx:ngx; - int nj = j0==0 ? ny:ngy; - int nk = k0==0 ? nz:ngz; - int is = i0==0 ? ngx:((i0==-1)?0:nx+ngx); - int js = j0==0 ? ngy:((j0==-1)?0:ny+ngy); - int ks = k0==0 ? ngz:((k0==-1)?0:nz+ngz); + int ni = i0==0 ? n[0]:ng[0]; + int nj = j0==0 ? n[1]:ng[1]; + int nk = k0==0 ? n[2]:ng[2]; + int is = i0==0 ? ng[0]:((i0==-1)?0:n[0]+ng[0]); + int js = j0==0 ? ng[1]:((j0==-1)?0:n[1]+ng[1]); + int ks = k0==0 ? ng[2]:((k0==-1)?0:n[2]+ng[2]); for (int d=0; d void fillHalo::copy( const Array& src, Array& dst ) { //PROFILE_START("fillHalo::copy",1); - ASSERT( (int)src.size(0)==nx || (int)src.size(0)==nx+2*ngx ); - ASSERT( (int)dst.size(0)==nx || (int)dst.size(0)==nx+2*ngx ); - bool src_halo = (int)src.size(0)==nx+2*ngx; - bool dst_halo = (int)dst.size(0)==nx+2*ngx; + ASSERT( (int)src.size(0)==n[0] || (int)src.size(0)==n[0]+2*ng[0] ); + ASSERT( (int)dst.size(0)==n[0] || (int)dst.size(0)==n[0]+2*ng[0] ); + bool src_halo = (int)src.size(0)==n[0]+2*ng[0]; + bool dst_halo = (int)dst.size(0)==n[0]+2*ng[0]; if ( src_halo ) { - ASSERT((int)src.size(0)==nx+2*ngx); - ASSERT((int)src.size(1)==ny+2*ngy); - ASSERT((int)src.size(2)==nz+2*ngz); + ASSERT((int)src.size(0)==n[0]+2*ng[0]); + ASSERT((int)src.size(1)==n[1]+2*ng[1]); + ASSERT((int)src.size(2)==n[2]+2*ng[2]); } else { - ASSERT((int)src.size(0)==nx); - ASSERT((int)src.size(1)==ny); - ASSERT((int)src.size(2)==nz); + ASSERT((int)src.size(0)==n[0]); + ASSERT((int)src.size(1)==n[1]); + ASSERT((int)src.size(2)==n[2]); } if ( dst_halo ) { - ASSERT((int)dst.size(0)==nx+2*ngx); - ASSERT((int)dst.size(1)==ny+2*ngy); - ASSERT((int)dst.size(2)==nz+2*ngz); + ASSERT((int)dst.size(0)==n[0]+2*ng[0]); + ASSERT((int)dst.size(1)==n[1]+2*ng[1]); + ASSERT((int)dst.size(2)==n[2]+2*ng[2]); } else { - ASSERT((int)dst.size(0)==nx); - ASSERT((int)dst.size(1)==ny); - ASSERT((int)dst.size(2)==nz); + ASSERT((int)dst.size(0)==n[0]); + ASSERT((int)dst.size(1)==n[1]); + ASSERT((int)dst.size(2)==n[2]); } if ( src_halo == dst_halo ) { // Src and dst halos match @@ -236,19 +287,19 @@ void fillHalo::copy( const Array& src, Array& dst ) dst(i) = src(i); } else if ( src_halo && !dst_halo ) { // Src has halos - for (int k=0; k fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1); + fillHalo fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1); fillData.fill(Phase); fillData.fill(SignDist); diff --git a/tests/ComponentLabel.cpp b/tests/ComponentLabel.cpp index 16587699..f350a5be 100644 --- a/tests/ComponentLabel.cpp +++ b/tests/ComponentLabel.cpp @@ -431,7 +431,7 @@ int main(int argc, char **argv) Averages.PrintComponents(timestep); // Create the MeshDataStruct - fillHalo fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1); + fillHalo fillData(Dm.Comm,Dm.rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1); std::vector meshData(1); meshData[0].meshName = "domain"; meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) ); diff --git a/tests/TestBlobIdentify.cpp b/tests/TestBlobIdentify.cpp index 71e6f1cf..ccfc6afc 100644 --- a/tests/TestBlobIdentify.cpp +++ b/tests/TestBlobIdentify.cpp @@ -133,8 +133,8 @@ void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct int ngy = ny+2*abs(sy); int ngz = nz+2*abs(sz); Array tmp1(nx,ny,nz), tmp2(ngx,ngy,ngz), tmp3(ngx,ngy,ngz); - fillHalo fillData1(comm,rank_info,nx,ny,nz,1,1,1,0,1); - fillHalo fillData2(comm,rank_info,nx,ny,nz,abs(sx),abs(sy),abs(sz),0,1); + fillHalo fillData1(comm,rank_info,{nx,ny,nz},{1,1,1},0,1); + fillHalo fillData2(comm,rank_info,{nx,ny,nz},{abs(sx),abs(sy),abs(sz)},0,1); fillData1.copy(data,tmp1); fillData2.copy(tmp1,tmp2); fillData2.fill(tmp2); @@ -189,7 +189,7 @@ int main(int argc, char **argv) fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info ); // Communication the halos - fillHalo fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1); + fillHalo fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,1); fillData.fill(Phase); fillData.fill(SignDist); @@ -205,7 +205,7 @@ int main(int argc, char **argv) // Create the MeshDataStruct std::vector meshData(1); meshData[0].meshName = "domain"; - meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(rank_info,nx,ny,nz,Lx,Ly,Lz) ); + meshData[0].mesh = std::make_shared(rank_info,nx,ny,nz,Lx,Ly,Lz); std::shared_ptr PhaseVar( new IO::Variable() ); std::shared_ptr SignDistVar( new IO::Variable() ); std::shared_ptr BlobIDVar( new IO::Variable() ); diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index 18405907..f2b8b631 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -11,8 +11,8 @@ #include #include "common/Array.h" #include "common/Domain.h" -#include "analysis/eikonal.h" #include "IO/Writer.h" +#include "analysis/distance.h" std::shared_ptr loadInputs( int nprocs ) @@ -21,7 +21,7 @@ std::shared_ptr loadInputs( int nprocs ) auto db = std::make_shared( ); db->putScalar( "BC", 0 ); db->putVector( "nproc", { 2, 2, 2 } ); - db->putVector( "n", { 50, 50, 50 } ); + db->putVector( "n", { 100, 100, 100 } ); db->putScalar( "nspheres", 0 ); db->putVector( "L", { 1, 1, 1 } ); return db; @@ -31,13 +31,13 @@ std::shared_ptr loadInputs( int nprocs ) //*************************************************************************************** int main(int argc, char **argv) { - // Initialize MPI - int rank, nprocs; - MPI_Init(&argc,&argv); + // Initialize MPI + int rank, nprocs; + MPI_Init(&argc,&argv); MPI_Comm comm = MPI_COMM_WORLD; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&nprocs); - { + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&nprocs); + { // Load inputs @@ -49,99 +49,81 @@ int main(int argc, char **argv) // Get the rank info Domain Dm(db); - for (int k=0;k id(nx,ny,nz); id.fill(0); - for (int k=1;k ID0(id.size()); @@ -149,17 +131,17 @@ int main(int argc, char **argv) Array ID(Nx,Ny,Nz); Array dist1(Nx,Ny,Nz); Array dist2(Nx,Ny,Nz); - fillHalo fillData(Dm.Comm, Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1); + fillHalo fillData(Dm.Comm, Dm.rank_info,{Nx,Ny,Nz},{1,1,1},0,1); fillData.copy( ID0, ID ); - fillData.copy( Distance, dist1 ); - fillData.copy( TrueDist, dist2 ); + fillData.copy( TrueDist, dist1 ); + fillData.copy( Distance, dist2 ); std::vector data(1); data[0].meshName = "mesh"; data[0].mesh.reset( new IO::DomainMesh( Dm.rank_info, Nx, Ny, Nz, Dm.Lx, Dm.Ly, Dm.Lz ) ); data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "ID", ID ) ); - data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "Distance", dist1 ) ); - data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "TrueDist", dist2 ) ); - data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "error", dist1-dist2 ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "TrueDist", dist1 ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "Distance", dist2 ) ); + data[0].vars.emplace_back( new IO::Variable( 1, IO::VariableType::VolumeVariable, "error", dist2-dist1 ) ); IO::initialize( "", "silo", false ); IO::writeData( "testSegDist", data, MPI_COMM_WORLD ); diff --git a/tests/lbpm_morph_pp.cpp b/tests/lbpm_morph_pp.cpp index 515e3afd..270e0142 100644 --- a/tests/lbpm_morph_pp.cpp +++ b/tests/lbpm_morph_pp.cpp @@ -343,42 +343,42 @@ int main(int argc, char **argv) PackID(Dm.sendList_yZ, Dm.sendCount_yZ ,sendID_yZ, id); PackID(Dm.sendList_YZ, Dm.sendCount_YZ ,sendID_YZ, id); //...................................................................................... - MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x,sendtag, - recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X,sendtag, - recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y,sendtag, - recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y,sendtag, - recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z,sendtag, - recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z,sendtag, - recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy,sendtag, - recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY,sendtag, - recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy,sendtag, - recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY,sendtag, - recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz,sendtag, - recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ,sendtag, - recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz,sendtag, - recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ,sendtag, - recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz,sendtag, - recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ,sendtag, - recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz,sendtag, - recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ,recvtag,comm,MPI_STATUS_IGNORE); - MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ,sendtag, - recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz,recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_x,Dm.sendCount_x,MPI_CHAR,Dm.rank_x(),sendtag, + recvID_X,Dm.recvCount_X,MPI_CHAR,Dm.rank_X(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_X,Dm.sendCount_X,MPI_CHAR,Dm.rank_X(),sendtag, + recvID_x,Dm.recvCount_x,MPI_CHAR,Dm.rank_x(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_y,Dm.sendCount_y,MPI_CHAR,Dm.rank_y(),sendtag, + recvID_Y,Dm.recvCount_Y,MPI_CHAR,Dm.rank_Y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Y,Dm.sendCount_Y,MPI_CHAR,Dm.rank_Y(),sendtag, + recvID_y,Dm.recvCount_y,MPI_CHAR,Dm.rank_y(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_z,Dm.sendCount_z,MPI_CHAR,Dm.rank_z(),sendtag, + recvID_Z,Dm.recvCount_Z,MPI_CHAR,Dm.rank_Z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Z,Dm.sendCount_Z,MPI_CHAR,Dm.rank_Z(),sendtag, + recvID_z,Dm.recvCount_z,MPI_CHAR,Dm.rank_z(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xy,Dm.sendCount_xy,MPI_CHAR,Dm.rank_xy(),sendtag, + recvID_XY,Dm.recvCount_XY,MPI_CHAR,Dm.rank_XY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XY,Dm.sendCount_XY,MPI_CHAR,Dm.rank_XY(),sendtag, + recvID_xy,Dm.recvCount_xy,MPI_CHAR,Dm.rank_xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xy,Dm.sendCount_Xy,MPI_CHAR,Dm.rank_Xy(),sendtag, + recvID_xY,Dm.recvCount_xY,MPI_CHAR,Dm.rank_xY(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, + recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, + recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, + recvID_xZ,Dm.recvCount_xZ,MPI_CHAR,Dm.rank_xZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_xZ,Dm.sendCount_xZ,MPI_CHAR,Dm.rank_xZ(),sendtag, + recvID_Xz,Dm.recvCount_Xz,MPI_CHAR,Dm.rank_Xz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yz,Dm.sendCount_yz,MPI_CHAR,Dm.rank_yz(),sendtag, + recvID_YZ,Dm.recvCount_YZ,MPI_CHAR,Dm.rank_YZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_YZ,Dm.sendCount_YZ,MPI_CHAR,Dm.rank_YZ(),sendtag, + recvID_yz,Dm.recvCount_yz,MPI_CHAR,Dm.rank_yz(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_Yz,Dm.sendCount_Yz,MPI_CHAR,Dm.rank_Yz(),sendtag, + recvID_yZ,Dm.recvCount_yZ,MPI_CHAR,Dm.rank_yZ(),recvtag,comm,MPI_STATUS_IGNORE); + MPI_Sendrecv(sendID_yZ,Dm.sendCount_yZ,MPI_CHAR,Dm.rank_yZ(),sendtag, + recvID_Yz,Dm.recvCount_Yz,MPI_CHAR,Dm.rank_Yz(),recvtag,comm,MPI_STATUS_IGNORE); UnpackID(Dm.recvList_x, Dm.recvCount_x ,recvID_x, id); UnpackID(Dm.recvList_X, Dm.recvCount_X ,recvID_X, id); diff --git a/tests/lbpm_morphdrain_pp.cpp b/tests/lbpm_morphdrain_pp.cpp index 9a9f2f20..42250dcf 100644 --- a/tests/lbpm_morphdrain_pp.cpp +++ b/tests/lbpm_morphdrain_pp.cpp @@ -155,7 +155,7 @@ int main(int argc, char **argv) xdim=Dm.Nx-2; ydim=Dm.Ny-2; zdim=Dm.Nz-2; - fillHalo fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1); + fillHalo fillData(Dm.Comm, Dm.rank_info,{xdim,ydim,zdim},{1,1,1},0,1); DoubleArray SignDist(nx,ny,nz); // Read the signed distance from file diff --git a/tests/lbpm_random_pp.cpp b/tests/lbpm_random_pp.cpp index 8426396b..b6b75e0a 100644 --- a/tests/lbpm_random_pp.cpp +++ b/tests/lbpm_random_pp.cpp @@ -367,7 +367,7 @@ int main(int argc, char **argv) MPI_Sendrecv(sendID_xY,Dm.sendCount_xY,MPI_CHAR,Dm.rank_xY(),sendtag, recvID_Xy,Dm.recvCount_Xy,MPI_CHAR,Dm.rank_Xy(),recvtag,comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_xz,Dm.sendCount_xz,MPI_CHAR,Dm.rank_xz(),sendtag, - recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE); + recvID_XZ,Dm.recvCount_XZ,MPI_CHAR,Dm.rank_XZ(),recvtag,comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_XZ,Dm.sendCount_XZ,MPI_CHAR,Dm.rank_XZ(),sendtag, recvID_xz,Dm.recvCount_xz,MPI_CHAR,Dm.rank_xz(),recvtag,comm,MPI_STATUS_IGNORE); MPI_Sendrecv(sendID_Xz,Dm.sendCount_Xz,MPI_CHAR,Dm.rank_Xz(),sendtag, diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 951d708e..01c12464 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -87,7 +87,7 @@ int main(int argc, char **argv) // Communication the halos const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - fillHalo fillData(comm,rank_info,rnx,rny,rnz,1,1,1,0,1); + fillHalo fillData(comm,rank_info,{rnx,rny,rnz},{1,1,1},0,1); nx+=2; ny+=2; nz+=2; rnx+=2; rny+=2; rnz+=2; diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 92f3100d..a14b0c32 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -12,7 +12,7 @@ #include "common/Array.h" #include "common/Domain.h" #include "analysis/TwoPhase.h" -#include "analysis/eikonal.h" +#include "analysis/distance.h" inline void MeanFilter(DoubleArray &Mesh){ for (int k=1; k<(int)Mesh.size(2)-1; k++){ @@ -251,15 +251,8 @@ int main(int argc, char **argv) } MeanFilter(Averages.SDs); - double LocalVar, TotalVar; if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); - int Maxtime=10*max(max(Dm.Nx*Dm.nprocx(),Dm.Ny*Dm.nprocy()),Dm.Nz*Dm.nprocz()); - Maxtime=min(Maxtime,MAXTIME); - LocalVar = Eikonal(Averages.SDs,id,Dm,Maxtime); - - MPI_Allreduce(&LocalVar,&TotalVar,1,MPI_DOUBLE,MPI_SUM,comm); - TotalVar /= nprocs; - if (rank==0) printf("Final variation in signed distance function %f \n",TotalVar); + CalcDist(Averages.SDs,id,Dm); sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"wb"); diff --git a/tests/testCommunication.cpp b/tests/testCommunication.cpp index 04b83719..57ce0959 100644 --- a/tests/testCommunication.cpp +++ b/tests/testCommunication.cpp @@ -221,7 +221,7 @@ int testHalo( MPI_Comm comm, int nprocx, int nprocy, int nprocz, int depth ) } // Communicate the halo - fillHalo fillData(comm,rank_info,nx,ny,nz,1,1,1,0,depth); + fillHalo fillData(comm,rank_info,{nx,ny,nz},{1,1,1},0,depth); fillData.fill(array); // Check the results From fa42b83efd7ade465eec5b46d60dc517dde67717 Mon Sep 17 00:00:00 2001 From: Mark Berrill Date: Thu, 17 May 2018 16:03:30 -0400 Subject: [PATCH 086/124] Improving CalcDist --- IO/Reader.cpp | 3 - IO/silo.hpp | 2 - analysis/distance.cpp | 288 +++++++++++++++++++----------------------- analysis/distance.h | 8 +- common/MPI_Helpers.h | 18 +++ tests/TestSegDist.cpp | 62 +++++---- 6 files changed, 184 insertions(+), 197 deletions(-) diff --git a/IO/Reader.cpp b/IO/Reader.cpp index c9595a8b..dedb9b82 100644 --- a/IO/Reader.cpp +++ b/IO/Reader.cpp @@ -44,7 +44,6 @@ std::vector IO::readTimesteps( const std::string& filename ) FILE *fid= fopen(filename.c_str(),"rb"); if ( fid==NULL ) ERROR("Error opening file"); - auto pos = std::min(filename.find_last_of(47),filename.find_last_of(90)); std::vector timesteps; char buf[1000]; while (fgets(buf,sizeof(buf),fid) != NULL) { @@ -160,7 +159,6 @@ std::shared_ptr IO::getMesh( const std::string& path, const std::strin #ifdef USE_SILO const DatabaseEntry& database = meshDatabase.domains[domain]; std::string filename = path + "/" + timestep + "/" + database.file; - int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str()); auto fid = silo::open( filename, silo::READ ); if ( meshDatabase.meshClass=="PointList" ) { Array coords = silo::readPointMesh( fid, database.name ); @@ -262,7 +260,6 @@ std::shared_ptr IO::getVariable( const std::string& path, const st const auto& database = meshDatabase.domains[domain]; auto variableDatabase = meshDatabase.getVariableDatabase( variable ); std::string filename = path + "/" + timestep + "/" + database.file; - int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str()); auto fid = silo::open( filename, silo::READ ); var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) ); if ( meshDatabase.meshClass=="PointList" ) { diff --git a/IO/silo.hpp b/IO/silo.hpp index d1ecee0c..312f32d8 100644 --- a/IO/silo.hpp +++ b/IO/silo.hpp @@ -322,9 +322,7 @@ void readTriMesh( DBfile* fid, const std::string& meshname, Array& coords, } auto zones = mesh->zones; int N_zones = zones->nzones; - int ndim_zones = zones->ndims; ASSERT( zones->nshapes==1 ); - int shape_type = zones->shapetype[0]; int shapesize = zones->shapesize[0]; tri.resize(N_zones,shapesize); for (int i=0; i &d, double dx, double dy, double dz ) -{ - auto s = d.size(); - bool test[3] = { false, false, false }; - // Check x-direction - Vec v1, v2; - for (size_t k=1; k -void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic ) +void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, + const std::array& periodic, const std::array& dx ) { ASSERT( Distance.size() == ID.size() ); std::array n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 }; @@ -64,15 +16,121 @@ void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, c for (size_t i=0; i &d, const Array &ID0, const Domain &Dm, const std::array& periodic ) + + +/****************************************************************** +* Vector-based distance calculation * +* Initialize cells adjacent to boundaries * +******************************************************************/ +static void calcVecInitialize( Array &d, const Array &ID, double dx, double dy, double dz ) +{ + d.fill( Vec( 1e50, 1e50, 1e50 ) ); + const double dx0 = 0.5*dx; + const double dy0 = 0.5*dy; + const double dz0 = 0.5*dz; + //const double dxy0 = 0.25*sqrt( dx*dx + dy*dy ); + //const double dxz0 = 0.25*sqrt( dx*dx + dz*dz ); + //const double dyz0 = 0.25*sqrt( dy*dy + dz*dz ); + //const double dxyz0 = sqrt( dx*dx + dy*dy + dz*dz ); + int Nx = d.size(0); + int Ny = d.size(1); + int Nz = d.size(2); + for (int k=1; k &d, double dx, double dy, double dz ) +{ + double err = 0; + int Nx = d.size(0); + int Ny = d.size(1); + int Nz = d.size(2); + // Propagate (+,+,+) + for (int k=1; k=0; k--) { + for (int j=Ny-2; j>=0; j--) { + for (int i=Nx-2; i>=0; i--) { + auto vx = d(i+1,j,k); + auto vy = d(i,j+1,k); + auto vz = d(i,j,k+1); + vx.x -= dx; + vy.y -= dy; + vz.z -= dz; + auto v = std::min( std::min(vx,vy), vz ); + double d1 = v.norm2(); + double d2 = d(i,j,k).norm2(); + if ( d1 < d2 ) { + d(i,j,k) = v; + err = std::max( err, sqrt(d2)-sqrt(d1) ); + } + } + } + } + return err; +} + + +/****************************************************************** +* Vector-based distance calculation * +******************************************************************/ +void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, + const std::array& periodic, const std::array& dx ) { - const double dx = 1.0; - const double dy = 1.0; - const double dz = 1.0; std::array N = { Dm.Nx, Dm.Ny, Dm.Nz }; std::array n = { Dm.Nx-2, Dm.Ny-2, Dm.Nz-2 }; // Create ID with ghosts @@ -102,112 +160,30 @@ void CalcVecDist( Array &d, const Array &ID0, const Domain &Dm, const fillDataID.fill( ID ); // Create communicator for distance fillHalo fillData( Dm.Comm, Dm.rank_info, n, {1,1,1}, 50, 1, {true,false,false}, periodic ); - // Initialize the vector distance - d.fill( Vec( 1e50, 1e50, 1e50 ) ); - const double dx0 = 0.5*dx; - const double dy0 = 0.5*dy; - const double dz0 = 0.5*dz; - //const double dxy0 = 0.25*sqrt( dx*dx + dy*dy ); - //const double dxz0 = 0.25*sqrt( dx*dx + dz*dz ); - //const double dyz0 = 0.25*sqrt( dy*dy + dz*dz ); - //const double dxyz0 = sqrt( dx*dx + dy*dy + dz*dz ); - for (int k=1; ktol; it++) { + err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] ); } - int N_it = Dm.nprocx() + Dm.nprocy() + Dm.nprocz() + 3; + // Calculate the global distances + int N_it = Dm.nprocx() + Dm.nprocy() + Dm.nprocz() + 100; for ( int it=0; it=0; i--) { - auto v1 = d(i,j,k); - auto v2 = d(i+1,j,k); - v2.x -= dx; - if ( v2 < v1 ) - d(i,j,k) = v2; - } - } - } - // Propagate +/- y-direction - for (int k=0; k=0; j--) { - auto v1 = d(i,j,k); - auto v2 = d(i,j+1,k); - v2.y -= dy; - if ( v2 < v1 ) - d(i,j,k) = v2; - } - } - } - // Propagate +/- z-direction - for (int j=0; j=0; k--) { - auto v1 = d(i,j,k); - auto v2 = d(i,j,k+1); - v2.z -= dz; - if ( v2 < v1 ) - d(i,j,k) = v2; - } - } - } + // Update ghosts fillData.fill( d ); - bool test = checkUpdate( d, dx, dy, dz ); - test = sumReduce( Dm.Comm, test ); - if ( !test ) + // Update distance + double err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] ); + // Check if we are finished + err = maxReduce( Dm.Comm, err ); + if ( err < tol ) break; } } -template void CalcDist( Array&, const Array&, const Domain&, const std::array& ); -template void CalcDist( Array&, const Array&, const Domain&, const std::array& ); + + +// Explicit instantiations +template void CalcDist( Array&, const Array&, const Domain&, const std::array&, const std::array& ); +template void CalcDist( Array&, const Array&, const Domain&, const std::array&, const std::array& ); diff --git a/analysis/distance.h b/analysis/distance.h index b1bb9af5..b84a93f8 100644 --- a/analysis/distance.h +++ b/analysis/distance.h @@ -11,6 +11,7 @@ struct Vec { inline Vec(): x(0), y(0), z(0) {} inline Vec( double x_, double y_, double z_ ): x(x_), y(y_), z(z_) {} inline double norm() const { return sqrt(x*x+y*y+z*z); } + inline double norm2() const { return x*x+y*y+z*z; } }; inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; } @@ -24,7 +25,8 @@ inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l. * @param[in] periodic Directions that are periodic */ template -void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic = {true,true,true} ); +void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, + const std::array& periodic = {true,true,true}, const std::array& dx = {1,1,1} ); /*! * @brief Calculate the distance using a simple method @@ -34,7 +36,7 @@ void CalcDist( Array &Distance, const Array &ID, const Domain &Dm, c * @param[in] Dm Domain information * @param[in] periodic Directions that are periodic */ -void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, const std::array& periodic ); - +void CalcVecDist( Array &Distance, const Array &ID, const Domain &Dm, + const std::array& periodic = {true,true,true}, const std::array& dx = {1,1,1} ); #endif diff --git a/common/MPI_Helpers.h b/common/MPI_Helpers.h index 5a02397c..0cae743b 100644 --- a/common/MPI_Helpers.h +++ b/common/MPI_Helpers.h @@ -170,6 +170,12 @@ void unpack( std::set& data, const char *buffer ); // Helper functions +inline double sumReduce( MPI_Comm comm, double x ) +{ + double y = 0; + MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,comm); + return y; +} inline float sumReduce( MPI_Comm comm, float x ) { float y = 0; @@ -199,12 +205,24 @@ inline std::vector sumReduce( MPI_Comm comm, const std::vector& x ) MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm); return y; } +inline double maxReduce( MPI_Comm comm, double x ) +{ + double y = 0; + MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_MAX,comm); + return y; +} inline float maxReduce( MPI_Comm comm, float x ) { float y = 0; MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm); return y; } +inline int maxReduce( MPI_Comm comm, int x ) +{ + int y = 0; + MPI_Allreduce(&x,&y,1,MPI_INT,MPI_MAX,comm); + return y; +} #endif diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index f2b8b631..5866101d 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -17,11 +17,18 @@ std::shared_ptr loadInputs( int nprocs ) { - INSIST(nprocs==8, "TestSegDist: Number of MPI processes must be equal to 8"); + std::vector nproc; + if ( nprocs == 1 ) { + nproc = { 1, 1, 1 }; + } else if ( nprocs == 8 ) { + nproc = { 2, 2, 2 }; + } else { + ERROR("TestSegDist: Unsupported number of processors"); + } auto db = std::make_shared( ); db->putScalar( "BC", 0 ); - db->putVector( "nproc", { 2, 2, 2 } ); - db->putVector( "n", { 100, 100, 100 } ); + db->putVector( "nproc", nproc ); + db->putVector( "n", { 200, 200, 200 } ); db->putScalar( "nspheres", 0 ); db->putVector( "L", { 1, 1, 1 } ); return db; @@ -63,35 +70,30 @@ int main(int argc, char **argv) int ny = Ny+2; int nz = Nz+2; - double BubbleRadius = 0.3*nx; - // Initialize the bubble - double Cx = 1.0*nx; - double Cy = 1.0*ny; - double Cz = 1.0*nz; + double BubbleRadius = 0.15*Nx*Dm.nprocx(); + double Cx = 0.40*Nx*Dm.nprocx(); + double Cy = 0.45*Nx*Dm.nprocy(); + double Cz = 0.50*Nx*Dm.nprocy(); DoubleArray TrueDist(nx,ny,nz); Array id(nx,ny,nz); id.fill(0); - for (int k=1;k ID0(id.size()); From 9e5282fde34f29143711fa90419f69a23e3c0b47 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 21:03:11 -0400 Subject: [PATCH 087/124] eat the cake --- tests/CMakeLists.txt | 18 ++++---- tests/TestColorSquareTube.cpp | 36 +++++++-------- tests/TestFluxBC.cpp | 46 ++++++++++---------- tests/TestForceMoments.cpp | 49 ++++++++++----------- tests/TestMap.cpp | 15 +++---- tests/TestPressVel.cpp | 16 +++---- tests/TestTorus.cpp | 82 +++++++++++++++++------------------ tests/lbpm_captube_pp.cpp | 24 +++++----- tests/lbpm_inkbottle_pp.cpp | 23 +++++----- tests/lbpm_plates_pp.cpp | 16 ++++--- tests/lbpm_porenetwork_pp.cpp | 22 ++++++---- tests/lbpm_segmented_pp.cpp | 30 +++++++------ 12 files changed, 188 insertions(+), 189 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c80ca533..b31b582a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,10 +2,10 @@ #ADD_LBPM_EXECUTABLE( lbpm_nonnewtonian_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) -ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) +#ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) +#ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) +#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) +#ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator ) ADD_LBPM_EXECUTABLE( lbpm_sphere_pp ) ADD_LBPM_EXECUTABLE( lbpm_random_pp ) ADD_LBPM_EXECUTABLE( lbpm_refine_pp ) @@ -41,10 +41,10 @@ ADD_LBPM_TEST( pmmc_cylinder ) ADD_LBPM_TEST( TestTorus ) ADD_LBPM_TEST( TestFluxBC ) ADD_LBPM_TEST( TestMap ) -ADD_LBPM_TEST( TestMRT ) -ADD_LBPM_TEST( TestColorGrad ) -ADD_LBPM_TEST( TestBubbleDFH ) -ADD_LBPM_TEST( TestColorGradDFH ) +#ADD_LBPM_TEST( TestMRT ) +#ADD_LBPM_TEST( TestColorGrad ) +#ADD_LBPM_TEST( TestBubbleDFH ) +#ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestColorMassBounceback ) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) @@ -59,7 +59,7 @@ ADD_LBPM_TEST_1_2_4( TestBlobIdentify ) #ADD_LBPM_TEST_PARALLEL( TestTwoPhase 8 ) ADD_LBPM_TEST_PARALLEL( TestBlobAnalyze 8 ) ADD_LBPM_TEST_PARALLEL( TestSegDist 8 ) -ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) +#ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) #ADD_LBPM_TEST_PARALLEL( TestMassConservationD3Q7 1 ) ADD_LBPM_TEST_1_2_4( testCommunication ) ADD_LBPM_TEST_1_2_4( testUtilities ) diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index be72c0e7..dc63f9d6 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -115,9 +115,9 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; + std::shared_ptr Dm (new Domain(db)); + Dm->CommInit(comm); - Domain Dm(db); - Nx += 2; Ny += 2; Nz += 2; @@ -136,7 +136,7 @@ int main(int argc, char **argv) for (j=0;jid[n]=0; } } } @@ -155,16 +155,16 @@ int main(int argc, char **argv) int kglobal= k+(Nz-2)*kproc; // Initialize phase position field for parallel bubble test - if (iglobal < 2) Dm.id[n]=0; - else if (iglobal > (Nx-2)*nprocx-2) Dm.id[n]=0; - else if (jglobal < 2) Dm.id[n]=0; - else if (jglobal > (Ny-2)*nprocy-2) Dm.id[n]=0; - else if (kglobal < 20) Dm.id[n]=1; - else Dm.id[n]=2; + if (iglobal < 2) Dm->id[n]=0; + else if (iglobal > (Nx-2)*nprocx-2) Dm->id[n]=0; + else if (jglobal < 2) Dm->id[n]=0; + else if (jglobal > (Ny-2)*nprocy-2) Dm->id[n]=0; + else if (kglobal < 20) Dm->id[n]=1; + else Dm->id[n]=2; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); //....................................................................... // Compute the media porosity, assign phase labels and solid composition @@ -174,13 +174,13 @@ int main(int argc, char **argv) int Np=0; // number of local pore nodes double *PhaseLabel; PhaseLabel = new double[N]; - Dm.AssignComponentLabels(PhaseLabel); + Dm->AssignComponentLabels(PhaseLabel); //....................................................................... for (k=1;k 0){ + if (Dm->id[n] > 0){ sum_local+=1.0; Np++; } @@ -203,7 +203,7 @@ int main(int argc, char **argv) char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm.id, N); + ScaLBL_CopyToDevice(ID, Dm->id, N); //........................................................................... if (rank==0){ @@ -219,7 +219,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Np]; - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -277,12 +277,12 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); // Initialize the phase field and variables ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); - if (Dm.kproc()==0){ + if (Dm->kproc()==0){ ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); } - if (Dm.kproc() == nprocz-1){ + if (Dm->kproc() == nprocz-1){ ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); @@ -409,7 +409,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); sum_local+=VEL[2*Np+idx]; } @@ -426,7 +426,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); double vz = VEL[2*Np+idx]; printf("%f ",vz); diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index c9401fa8..1ee1740c 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -31,7 +31,7 @@ int main (int argc, char **argv) double din,dout; int BC=1; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); Nz += 2; Nx = Ny = Nz; // Cubic domain @@ -67,8 +67,8 @@ int main (int argc, char **argv) id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; //......................................................... // Initialize communication structures in averaging domain - for (i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = id[i]; + Dm->CommInit(comm); Np=0; // number of local pore nodes //....................................................................... @@ -76,7 +76,7 @@ int main (int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ Np++; } } @@ -85,7 +85,7 @@ int main (int argc, char **argv) //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device - ScaLBL_Communicator ScaLBL_Comm(Dm); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); if (rank==0) printf ("Set up the neighborlist \n"); int Npad=Np+32; @@ -94,7 +94,7 @@ int main (int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -142,7 +142,7 @@ int main (int argc, char **argv) double flux = 1.0; int timestep=0; - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); printf("Computed pressure for flux = %f\n",din); @@ -165,7 +165,7 @@ int main (int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); Q += VEL[2*Np+idx]; } @@ -194,21 +194,21 @@ int main (int argc, char **argv) while (timestep < 2000) { - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -226,7 +226,7 @@ int main (int argc, char **argv) for (k=1;k 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); double vz = VEL[2*Np+idx]; printf("%f ",vz); @@ -242,7 +242,7 @@ int main (int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); double vz = VEL[2*Np+idx]; printf("%f ",vz); @@ -256,7 +256,7 @@ int main (int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); Q += VEL[2*Np+idx]; } diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 1f9196ae..94f4c101 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -13,7 +13,6 @@ using namespace std; extern void PrintNeighborList(int * neighborList, int Np, int rank) { if (rank == 0) { - int n; int neighbor; for (int i = 0; i < Np; i++) { @@ -32,7 +31,7 @@ extern void PrintNeighborList(int * neighborList, int Np, int rank) { std::shared_ptr loadInputs( int nprocs ) { auto db = std::make_shared( "Domain.in" ); - const int dim = 50; + db->putScalar( "BC", 0 ); if ( nprocs == 1 ){ db->putVector( "nproc", { 1, 1, 1 } ); @@ -118,7 +117,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(domain_db); + std::shared_ptr Dm(new Domain(domain_db)); Nx += 2; Ny += 2; @@ -135,7 +134,7 @@ int main(int argc, char **argv) /* FILE *IDFILE = fopen(LocalRankFilename,"rb"); if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - fread(Dm.id,1,N,IDFILE); + fread(Dm->id,1,N,IDFILE); fclose(IDFILE); */ @@ -144,11 +143,11 @@ int main(int argc, char **argv) for (j=0;jid[n]=1; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; @@ -157,7 +156,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ Np++; } } @@ -165,16 +164,14 @@ int main(int argc, char **argv) } if (rank==0) printf ("Create ScaLBL_Communicator \n"); - - // Create a communicator for the device - ScaLBL_Communicator ScaLBL_Comm(Dm); + auto ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Dm)); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm.id, N); + ScaLBL_CopyToDevice(ID, Dm->id, N); //........................................................................... if (rank==0){ @@ -191,7 +188,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Np]; - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); if (rank == 0) PrintNeighborList(neighborList,Np, rank); @@ -225,8 +222,8 @@ int main(int argc, char **argv) starttime = MPI_Wtime(); /************ MAIN ITERATION LOOP (timing communications)***************************************/ - //ScaLBL_Comm.SendD3Q19(dist, &dist[10*Np]); - //ScaLBL_Comm.RecvD3Q19(dist, &dist[10*Np]); + //ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]); + //ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); if (rank==0) printf("Beginning AA timesteps...\n"); @@ -235,17 +232,17 @@ int main(int argc, char **argv) while (timestep < 2) { - ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(dist); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -293,7 +290,7 @@ int main(int argc, char **argv) for (j=1;jid[n]); n = Map(i,j,k); //printf("%i,%i,%i; %i :",i,j,k,n); if (n<0) vz =0.f; @@ -318,7 +315,7 @@ int main(int argc, char **argv) for (k=1;kid[n]); n = Map(i,j,k); double fa = DIST[(2*q+1)*Np+n]; double fb = DIST[2*(q+1)*Np+n]; @@ -339,7 +336,7 @@ int main(int argc, char **argv) for (k=1;kid[n]); n = Map(i,j,k); double fa = DIST[(2*q+1)*Np+n]; double fb = DIST[2*(q+1)*Np+n]; @@ -359,7 +356,7 @@ int main(int argc, char **argv) for (j=1;jid[n]); n = Map(i,j,k); double fa = DIST[(2*q+1)*Np+n]; double fb = DIST[2*(q+1)*Np+n]; diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index 18e9e219..943073f2 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -134,7 +134,7 @@ int main(int argc, char **argv) MPI_Barrier(comm); int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition)); Nx += 2; Ny += 2; @@ -147,18 +147,17 @@ int main(int argc, char **argv) for (j=1;jid[n] = 1; Np++; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Dm); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - + std::shared_ptr ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm)); if (rank==0){ printf("Total domain size = %i \n",N); @@ -173,7 +172,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -236,7 +235,7 @@ int main(int argc, char **argv) // Loop over the distributions for interior lattice sites if (rank==0) printf ("Loop over distributions \n"); - for (int idx=ScaLBL_Comm.first_interior; idxfirst_interior; idxlast_interior; idx++){ n = TmpMap[idx]; k = n/(Nx*Ny); j = (n-Nx*Ny*k)/Nx; diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index d2aebeba..1bbdd3c6 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -76,7 +76,8 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(domain_db); + std::shared_ptr Dm (new Domain(domain_db)); + Dm->CommInit(comm); Nx += 2; Ny += 2; @@ -91,7 +92,7 @@ int main(int argc, char **argv) char LocalRankFilename[40]; sprintf(LocalRankFilename,"ID.%05i",rank); - Dm.CommInit(comm); + Dm->CommInit(comm); //....................................................................... // Compute the media porosity @@ -103,8 +104,8 @@ int main(int argc, char **argv) for (j=1;j 0){ + Dm->id[n] = 1; + if (Dm->id[n] > 0){ sum_local+=1.0; Np++; } @@ -120,14 +121,13 @@ int main(int argc, char **argv) if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device - ScaLBL_Communicator ScaLBL_Comm(Dm); - + auto ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Dm)); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm.id, N); + ScaLBL_CopyToDevice(ID, Dm->id, N); //........................................................................... if (rank==0){ @@ -142,7 +142,7 @@ int main(int argc, char **argv) int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. diff --git a/tests/TestTorus.cpp b/tests/TestTorus.cpp index 5c2fa519..a17db389 100644 --- a/tests/TestTorus.cpp +++ b/tests/TestTorus.cpp @@ -61,9 +61,9 @@ int main(int argc, char **argv) // Get the rank info - Domain Dm(db); + std::shared_ptr Dm(new Domain(db)); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - TwoPhase Averages(Dm); + std::shared_ptr Averages(new TwoPhase(Dm)); Nx += 2; Ny += 2; Nz += 2; @@ -73,12 +73,12 @@ int main(int argc, char **argv) for ( j=1;jid[n] = 1; } } } //....................................................................... - Dm.CommInit(comm); // Initialize communications for domains + Dm->CommInit(comm); // Initialize communications for domains //....................................................................... //....................................................................... @@ -103,42 +103,42 @@ int main(int argc, char **argv) n = k*Nx*Ny+j*Nx+i; // global position relative to center - x = Dm.iproc()*Nx+i - CX; - y = Dm.jproc()*Ny+j - CY; - z = Dm.kproc()*Nz+k - CZ; + x = Dm->iproc()*Nx+i - CX; + y = Dm->jproc()*Ny+j - CY; + z = Dm->kproc()*Nz+k - CZ; // Shrink the sphere sizes by two voxels to make sure they don't touch - Averages.SDs(i,j,k) = 100.0; + Averages->SDs(i,j,k) = 100.0; //.............................................................................. // Single torus - Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; + Averages->Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; // Double torus - /* y = Dm.jproc()*Ny+j - CY1; - //z = Dm.kproc()*Nz+k - CZ +R1; - Averages.Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; + /* y = Dm->jproc()*Ny+j - CY1; + //z = Dm->kproc()*Nz+k - CZ +R1; + Averages->Phase(i,j,k) = sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2; - y = Dm.jproc()*Ny+j - CY2; - //z = Dm.kproc()*Nz+k - CZ-R1; - Averages.Phase(i,j,k) = min(Averages.Phase(i,j,k), + y = Dm->jproc()*Ny+j - CY2; + //z = Dm->kproc()*Nz+k - CZ-R1; + Averages->Phase(i,j,k) = min(Averages->Phase(i,j,k), sqrt((sqrt(x*x+y*y) - R1)*(sqrt(x*x+y*y) - R1) + z*z) - R2); *///.............................................................................. - //Averages.Phase(i,j,k) = - Averages.Phase(i,j,k); - if (Averages.Phase(i,j,k) > 0.0){ - Dm.id[n] = 2; + //Averages->Phase(i,j,k) = - Averages->Phase(i,j,k); + if (Averages->Phase(i,j,k) > 0.0){ + Dm->id[n] = 2; } else{ - Dm.id[n] = 1; + Dm->id[n] = 1; } - Averages.SDn(i,j,k) = Averages.Phase(i,j,k); - Averages.Phase(i,j,k) = Averages.SDn(i,j,k); - Averages.Phase_tplus(i,j,k) = Averages.SDn(i,j,k); - Averages.Phase_tminus(i,j,k) = Averages.SDn(i,j,k); - Averages.DelPhi(i,j,k) = 0.0; - Averages.Press(i,j,k) = 0.0; - Averages.Vel_x(i,j,k) = 0.0; - Averages.Vel_y(i,j,k) = 0.0; - Averages.Vel_z(i,j,k) = 0.0; + Averages->SDn(i,j,k) = Averages->Phase(i,j,k); + Averages->Phase(i,j,k) = Averages->SDn(i,j,k); + Averages->Phase_tplus(i,j,k) = Averages->SDn(i,j,k); + Averages->Phase_tminus(i,j,k) = Averages->SDn(i,j,k); + Averages->DelPhi(i,j,k) = 0.0; + Averages->Press(i,j,k) = 0.0; + Averages->Vel_x(i,j,k) = 0.0; + Averages->Vel_y(i,j,k) = 0.0; + Averages->Vel_z(i,j,k) = 0.0; } } } @@ -146,25 +146,25 @@ int main(int argc, char **argv) double beta = 0.95; if (rank==0) printf("initializing the system \n"); - Averages.UpdateSolid(); - Dm.CommunicateMeshHalo(Averages.Phase); - Dm.CommunicateMeshHalo(Averages.SDn); + Averages->UpdateSolid(); + Dm->CommunicateMeshHalo(Averages->Phase); + Dm->CommunicateMeshHalo(Averages->SDn); - Averages.Initialize(); - Averages.UpdateMeshValues(); + Averages->Initialize(); + Averages->UpdateMeshValues(); if (rank==0) printf("computing local averages \n"); - Averages.AssignComponentLabels(); - Averages.ComponentAverages(); - Averages.PrintComponents(int(5)); + Averages->AssignComponentLabels(); + Averages->ComponentAverages(); + Averages->PrintComponents(int(5)); if (rank==0) printf("reducing averages \n"); - Averages.Initialize(); - Averages.ComputeLocal(); - Averages.Reduce(); - Averages.PrintAll(int(5)); - // Averages.Reduce(); + Averages->Initialize(); + Averages->ComputeLocal(); + Averages->Reduce(); + Averages->PrintAll(int(5)); + // Averages->Reduce(); } // Limit scope so variables that contain communicators will free before MPI_Finialize MPI_Barrier(comm); diff --git a/tests/lbpm_captube_pp.cpp b/tests/lbpm_captube_pp.cpp index ebe5480f..97acf8ec 100644 --- a/tests/lbpm_captube_pp.cpp +++ b/tests/lbpm_captube_pp.cpp @@ -34,8 +34,6 @@ int main(int argc, char **argv) MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); { - // parallel domain size (# of sub-domains) - int sendtag,recvtag; //***************************************** // MPI ranks for all 18 neighbors //********************************** @@ -44,8 +42,6 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** - MPI_Request req1[18],req2[18]; - MPI_Status stat1[18],stat2[18]; double TubeRadius =15.0; int BC; @@ -88,10 +84,12 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(db); - Dm.CommInit(comm); - TwoPhase Averages(Dm); + double Lx=1.f; + double Ly=1.f; + double Lz=1.f; + std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, @@ -133,18 +131,18 @@ int main(int argc, char **argv) for (i=0;iSDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2) + (j-Ny/2)*(j-Ny/2))); // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.kproc()*Nz+kkproc()*Nz+kkproc()*Nz+kSDs.data(),8,Averages->SDs.length(),DIST); fclose(DIST); sprintf(LocalRankFilename,"ID.%05i",rank); diff --git a/tests/lbpm_inkbottle_pp.cpp b/tests/lbpm_inkbottle_pp.cpp index dec4e8ee..012dac55 100644 --- a/tests/lbpm_inkbottle_pp.cpp +++ b/tests/lbpm_inkbottle_pp.cpp @@ -26,7 +26,6 @@ int main(int argc, char **argv) // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; - int sendtag,recvtag; //***************************************** // MPI ranks for all 18 neighbors //********************************** @@ -35,8 +34,6 @@ int main(int argc, char **argv) int rank_xz,rank_XZ,rank_xZ,rank_Xz; int rank_yz,rank_YZ,rank_yZ,rank_Yz; //********************************** - MPI_Request req1[18],req2[18]; - MPI_Status stat1[18],stat2[18]; double UpperTubeRadius =15.0; double LowerTubeRadius =15.0; @@ -117,9 +114,9 @@ int main(int argc, char **argv) } // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - Dm.CommInit(comm); - TwoPhase Averages(Dm); + std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, @@ -163,25 +160,25 @@ int main(int argc, char **argv) n = k*Nx*Ny + j*Nz + i; // Cylindrical capillary tube aligned with the z direction if (kSDs(i,j,k) = LowerTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2) + (j-Ny/2)*(j-Ny/2))); else - Averages.SDs(i,j,k) = UpperTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2) + Averages->SDs(i,j,k) = UpperTubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2) + (j-Ny/2)*(j-Ny/2))); BulbDist = BulbRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2)+ (j-Ny/2)*(j-Ny/2) + (k-Nz/2)*(k-Nz/2))); - if (BulbDist > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = BulbDist; + if (BulbDist > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = BulbDist; // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.kproc()*Nz+kkproc()*Nz+kkproc()*Nz+kSDs.data(),8,Averages->SDs.length(),DIST); fclose(DIST); sprintf(LocalRankFilename,"ID.%05i",rank); diff --git a/tests/lbpm_plates_pp.cpp b/tests/lbpm_plates_pp.cpp index c43f6781..74c68d5c 100644 --- a/tests/lbpm_plates_pp.cpp +++ b/tests/lbpm_plates_pp.cpp @@ -110,9 +110,11 @@ int main(int argc, char **argv) } // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - Dm.CommInit(comm); - TwoPhase Averages(Dm); + + std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); + MPI_Barrier(comm); @@ -149,13 +151,13 @@ int main(int argc, char **argv) for (i=0;iSDs(i,j,k) = TubeRadius-sqrt(1.0*((i-Nx/2)*(i-Nx/2))); // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Averages.SDs(i,j,k) < WIDTH){ + else if (Averages->SDs(i,j,k) < WIDTH){ id[n] = 2; sum++; } @@ -188,7 +190,7 @@ int main(int argc, char **argv) sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"wb"); - fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST); + fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST); fclose(DIST); sprintf(LocalRankFilename,"ID.%05i",rank); diff --git a/tests/lbpm_porenetwork_pp.cpp b/tests/lbpm_porenetwork_pp.cpp index 6e9de20f..616a3a8d 100644 --- a/tests/lbpm_porenetwork_pp.cpp +++ b/tests/lbpm_porenetwork_pp.cpp @@ -100,10 +100,14 @@ int main(int argc, char **argv) } // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - Dm.CommInit(comm); - TwoPhase Averages(Dm); + // Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + //Dm.CommInit(comm); + //TwoPhase Averages(Dm); + std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); + MPI_Barrier(comm); Nx += 2; Ny += 2; Nz += 2; @@ -180,7 +184,7 @@ int main(int argc, char **argv) for (k=0;kSDs(i,j,k) = -2.f*Nx; } } } @@ -208,7 +212,7 @@ int main(int argc, char **argv) double zk = double(k) - z; // value of s along center line {x=alpha*s, y = beta*s, z=gamma*s} that is closest to i,j,k double s = (alpha*xi + beta*yj + gamma*zk)/(alpha*alpha + beta*beta + gamma*gamma); - double distance=Averages.SDs(i,j,k); + double distance=Averages->SDs(i,j,k); if (s > length){ //distance = radius - sqrt((xi-X)*(xi-X) + (yj-Y)*(yj-Y) + (zk-Z)*(zk-Z)); } @@ -222,7 +226,7 @@ int main(int argc, char **argv) distance = radius - sqrt((xi-xs)*(xi-xs) + (yj-ys)*(yj-ys) + (zk-zs)*(zk-zs)); //if (distance>0)printf("s=%f,alpha=%f,beta=%f,gamma=%f,distance=%f\n",s,alpha,beta,gamma,distance); } - if (distance > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = distance; + if (distance > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = distance; } // Compute the distance to each sphere @@ -237,7 +241,7 @@ int main(int argc, char **argv) double distance = radius - sqrt((xi-x)*(xi-x) + (yj-y)*(yj-y) + (zk-z)*(zk-z)); - if (distance > Averages.SDs(i,j,k)) Averages.SDs(i,j,k) = distance; + if (distance > Averages->SDs(i,j,k)) Averages->SDs(i,j,k) = distance; } } } @@ -247,7 +251,7 @@ int main(int argc, char **argv) for (i=0;iSDs(i,j,k) < 0.0){ id[n] = 0; } else{ @@ -279,7 +283,7 @@ int main(int argc, char **argv) sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"wb"); - fwrite(Averages.SDs.data(),8,Averages.SDs.length(),DIST); + fwrite(Averages->SDs.data(),8,Averages->SDs.length(),DIST); fclose(DIST); sprintf(LocalRankFilename,"ID.%05i",rank); diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index 92f3100d..d8671766 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -189,9 +189,11 @@ int main(int argc, char **argv) char LocalRankFilename[40]; int N = (nx+2)*(ny+2)*(nz+2); - Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - for (n=0; n Dm (new Domain(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + for (n=0; nid[n]=1; + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); nx+=2; ny+=2; nz+=2; // Read the phase ID @@ -200,7 +202,7 @@ int main(int argc, char **argv) if (READ_FROM_BLOCK == 0){ size_t readID; FILE *ID = fopen(LocalRankFilename,"rb"); - readID=fread(Dm.id,1,N,ID); + readID=fread(Dm->id,1,N,ID); if (readID != size_t(N)) printf("lbpm_segmented_pp: Error reading ID \n"); fclose(ID); } @@ -208,13 +210,13 @@ int main(int argc, char **argv) // Read from the large block and write the local ID file if (rank==0) printf("Reading ID file from blocks \n"); fflush(stdout); - porosity = ReadFromBlock(Dm.id,Dm.iproc(),Dm.jproc(),Dm.kproc(),nx,ny,nz); + porosity = ReadFromBlock(Dm->id,Dm->iproc(),Dm->jproc(),Dm->kproc(),nx,ny,nz); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("Writing local ID files (poros=%f) \n",porosity); fflush(stdout); FILE *ID = fopen(LocalRankFilename,"wb"); - fwrite(Dm.id,1,N,ID); + fwrite(Dm->id,1,N,ID); fclose(ID); if (rank==0) printf("Succeeded! \n"); fflush(stdout); @@ -222,8 +224,8 @@ int main(int argc, char **argv) } // Initialize the domain and communication - Array id(nx,ny,nz); - TwoPhase Averages(Dm); + Array id(nx,ny,nz); + //TwoPhase Averages(Dm); // DoubleArray Distance(nx,ny,nz); // DoubleArray Phase(nx,ny,nz); @@ -234,7 +236,7 @@ int main(int argc, char **argv) for (i=0;iid[n] == 0) id(i,j,k) = 0; else id(i,j,k) = 1; } } @@ -245,17 +247,17 @@ int main(int argc, char **argv) for (i=0;iSDs(i,j,k) = 2.0*double(id(i,j,k))-1.0; } } } - MeanFilter(Averages.SDs); + MeanFilter(Averages->SDs); double LocalVar, TotalVar; if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); - int Maxtime=10*max(max(Dm.Nx*Dm.nprocx(),Dm.Ny*Dm.nprocy()),Dm.Nz*Dm.nprocz()); + int Maxtime=10*max(max(Dm->Nx*Dm->nprocx(),Dm->Ny*Dm->nprocy()),Dm->Nz*Dm->nprocz()); Maxtime=min(Maxtime,MAXTIME); - LocalVar = Eikonal(Averages.SDs,id,Dm,Maxtime); + LocalVar = Eikonal(Averages->SDs,id,*Dm,Maxtime); MPI_Allreduce(&LocalVar,&TotalVar,1,MPI_DOUBLE,MPI_SUM,comm); TotalVar /= nprocs; @@ -263,7 +265,7 @@ int main(int argc, char **argv) sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"wb"); - fwrite(Averages.SDs.data(),8,N,DIST); + fwrite(Averages->SDs.data(),8,N,DIST); fclose(DIST); } From 81be5a647e9bdfc97957d8f49624810f70057ab2 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 21:09:54 -0400 Subject: [PATCH 088/124] grow bigger --- tests/TestInterfaceSpeed.cpp | 66 ++++++++++++++++++------------------ tests/TestPoiseuille.cpp | 53 +++++++++++++---------------- tests/lbpm_squaretube_pp.cpp | 38 ++++++++++----------- 3 files changed, 76 insertions(+), 81 deletions(-) diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 3432fc26..84942b85 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -39,13 +39,13 @@ int main (int argc, char *argv[]) int nprocy = domain_db->getVector( "nproc" )[1]; int nprocz = domain_db->getVector( "nproc" )[2]; - Domain Dm(domain_db); + std::shared_ptr Dm(new Domain(domain_db)); - for (i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; - Dm.CommInit(comm); + Dm->CommInit(comm); - TwoPhase Averages(Dm); + std::shared_ptr Averages(new TwoPhase(Dm)); int timestep=0; double Cx,Cy,Cz; @@ -58,7 +58,7 @@ int main (int argc, char *argv[]) dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD; dist2 = fabs(Cz-k)-HEIGHT; - Averages.Phase_tminus(i,j,k) = dist2; + Averages->Phase_tminus(i,j,k) = dist2; } } } @@ -71,9 +71,9 @@ int main (int argc, char *argv[]) dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD; dist2 = fabs(Cz-k)-HEIGHT; - Averages.SDs(i,j,k) = -dist1; - Averages.Phase(i,j,k) = dist2; - Averages.SDn(i,j,k) = dist2; + Averages->SDs(i,j,k) = -dist1; + Averages->Phase(i,j,k) = dist2; + Averages->SDn(i,j,k) = dist2; } } } @@ -84,7 +84,7 @@ int main (int argc, char *argv[]) dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD; dist2 = fabs(Cz-k)-HEIGHT; - Averages.Phase_tplus(i,j,k) = dist2; + Averages->Phase_tplus(i,j,k) = dist2; } } } @@ -93,58 +93,58 @@ int main (int argc, char *argv[]) //.................................................................... // The following only need to be done once - //Averages.SetupCubes(Dm); - Averages.UpdateSolid(); // unless the solid is deformable! + //Averages->SetupCubes(Dm); + Averages->UpdateSolid(); // unless the solid is deformable! //.................................................................... // The following need to be called each time new averages are computed - Averages.Initialize(); - Averages.UpdateMeshValues(); - Averages.ComputeLocal(); - Averages.Reduce(); - Averages.PrintAll(timestep); + Averages->Initialize(); + Averages->UpdateMeshValues(); + Averages->ComputeLocal(); + Averages->Reduce(); + Averages->PrintAll(timestep); //.................................................................... printf("-------------------------------- \n"); - printf("NWP volume = %f \n", Averages.nwp_volume); - printf("Area wn = %f, Analytical = %f \n", Averages.awn,2*PI*RADIUS*RADIUS); - printf("Area ns = %f, Analytical = %f \n", Averages.ans, 2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT); - printf("Area ws = %f, Analytical = %f \n", Averages.aws, 4*PI*RADIUS*HEIGHT); - printf("Area s = %f, Analytical = %f \n", Averages.As, 2*PI*RADIUS*(Nz-2)); - printf("Length wns = %f, Analytical = %f \n", Averages.lwns, 4*PI*RADIUS); - printf("Geodesic curvature (wns) = %f, Analytical = %f \n", Averages.KGwns_global, 0.0); - printf("Normal curvature (wns) = %f, Analytical = %f \n", Averages.KNwns_global, 1.0/RADIUS); + printf("NWP volume = %f \n", Averages->nwp_volume); + printf("Area wn = %f, Analytical = %f \n", Averages->awn,2*PI*RADIUS*RADIUS); + printf("Area ns = %f, Analytical = %f \n", Averages->ans, 2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT); + printf("Area ws = %f, Analytical = %f \n", Averages->aws, 4*PI*RADIUS*HEIGHT); + printf("Area s = %f, Analytical = %f \n", Averages->As, 2*PI*RADIUS*(Nz-2)); + printf("Length wns = %f, Analytical = %f \n", Averages->lwns, 4*PI*RADIUS); + printf("Geodesic curvature (wns) = %f, Analytical = %f \n", Averages->KGwns_global, 0.0); + printf("Normal curvature (wns) = %f, Analytical = %f \n", Averages->KNwns_global, 1.0/RADIUS); // printf("Cos(theta_wns) = %f, Analytical = %f \n",efawns/lwns,1.0*RADIUS/CAPRAD); - printf("Interface Velocity = %f,%f,%f \n",Averages.vawn_global(0),Averages.vawn_global(1),Averages.vawn_global(2)); - printf("Common Curve Velocity = %f,%f,%f \n",Averages.vawns_global(0),Averages.vawns_global(1),Averages.vawns_global(2)); + printf("Interface Velocity = %f,%f,%f \n",Averages->vawn_global(0),Averages->vawn_global(1),Averages->vawn_global(2)); + printf("Common Curve Velocity = %f,%f,%f \n",Averages->vawns_global(0),Averages->vawns_global(1),Averages->vawns_global(2)); printf("-------------------------------- \n"); //......................................................................... int toReturn = 0; - if (fabs(Averages.awn - 2*PI*RADIUS*RADIUS)/(2*PI*RADIUS*RADIUS) > 0.02){ + if (fabs(Averages->awn - 2*PI*RADIUS*RADIUS)/(2*PI*RADIUS*RADIUS) > 0.02){ toReturn = 1; printf("TestCylinderArea.cpp: error tolerance exceeded for wn area \n"); } - if (fabs(Averages.ans - (2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT))/(2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT)> 0.02 ){ + if (fabs(Averages->ans - (2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT))/(2*PI*RADIUS*(Nz-2)-4*PI*RADIUS*HEIGHT)> 0.02 ){ toReturn = 2; printf("TestCylinderArea.cpp: error tolerance exceeded for ns area \n"); } - if (fabs(Averages.aws - 4*PI*RADIUS*HEIGHT)/(4*PI*RADIUS*HEIGHT) > 0.02 ){ + if (fabs(Averages->aws - 4*PI*RADIUS*HEIGHT)/(4*PI*RADIUS*HEIGHT) > 0.02 ){ toReturn = 3; printf("TestCylinderArea.cpp: error tolerance exceeded for ws area \n"); } - if (fabs(Averages.As - 2*PI*RADIUS*(Nz-2))/(2*PI*RADIUS*(Nz-2)) > 0.02 ){ + if (fabs(Averages->As - 2*PI*RADIUS*(Nz-2))/(2*PI*RADIUS*(Nz-2)) > 0.02 ){ toReturn = 4; printf("TestCylinderArea.cpp: error tolerance exceeded for solid area \n"); } - if (fabs(Averages.lwns - 4*PI*RADIUS)/(4*PI*RADIUS) > 0.02 ){ + if (fabs(Averages->lwns - 4*PI*RADIUS)/(4*PI*RADIUS) > 0.02 ){ toReturn = 5; printf("TestCylinderArea.cpp: error tolerance exceeded for common curve length \n"); } - if ( fabs(Averages.vawn_global(2)+0.2) > 0.01){ + if ( fabs(Averages->vawn_global(2)+0.2) > 0.01){ printf("TestInterfaceSpeed: Error too high for kinematic velocity of wn interface \n"); toReturn = 6; } - if ( fabs(Averages.vawns_global(2)+0.2) > 0.01){ + if ( fabs(Averages->vawns_global(2)+0.2) > 0.01){ printf("TestInterfaceSpeed: Error too high for kinematic velocity of common curve \n"); toReturn = 7; } diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 6c3f7de9..580486c5 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -9,10 +9,6 @@ #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" - - - - //*************************************************************************************** int main(int argc, char **argv) { @@ -34,7 +30,6 @@ int main(int argc, char **argv) } // BGK Model parameters - unsigned int nBlocks, nthreads; int timestepMax, interval; double tau,Fx,Fy,Fz,tol; // Domain variables @@ -80,7 +75,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - Domain Dm(domain_db); + std::shared_ptr Dm( new Domain(domain_db)); Nx += 2; Ny += 2; @@ -97,7 +92,7 @@ int main(int argc, char **argv) /* FILE *IDFILE = fopen(LocalRankFilename,"rb"); if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - fread(Dm.id,1,N,IDFILE); + fread(Dm->id,1,N,IDFILE); fclose(IDFILE); */ @@ -106,14 +101,14 @@ int main(int argc, char **argv) for (j=0;jNx-3) Dm.id[n] = 0; - else Dm.id[n]=1; + if (i<2) Dm->id[n] = 0; + else if (i>Nx-3) Dm->id[n] = 0; + else Dm->id[n]=1; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); MPI_Barrier(comm); //....................................................................... @@ -126,7 +121,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ sum_local+=1.0; Np++; } @@ -142,14 +137,14 @@ int main(int argc, char **argv) if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device - ScaLBL_Communicator ScaLBL_Comm(Dm); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); //...........device phase ID................................................. if (rank==0) printf ("Copying phase ID to device \n"); char *ID; ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory // Copy to the device - ScaLBL_CopyToDevice(ID, Dm.id, N); + ScaLBL_CopyToDevice(ID, Dm->id, N); //........................................................................... if (rank==0){ @@ -165,7 +160,7 @@ int main(int argc, char **argv) int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -195,8 +190,8 @@ int main(int argc, char **argv) starttime = MPI_Wtime(); /************ MAIN ITERATION LOOP (timing communications)***************************************/ - // ScaLBL_Comm.SendD3Q19(dist, &dist[10*Np]); - // ScaLBL_Comm.RecvD3Q19(dist, &dist[10*Np]); + // ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]); + // ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]); // ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // @@ -205,17 +200,17 @@ int main(int argc, char **argv) while (timestep < 2000) { - ScaLBL_Comm.SendD3Q19AA(dist); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(dist); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; - ScaLBL_Comm.SendD3Q19AA(dist); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm.RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm.next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************/ @@ -241,8 +236,8 @@ int main(int argc, char **argv) // if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9/cputime); // Report bandwidth in Gigabits per second // communication bandwidth includes both send and recieve - //if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime); - // if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime); + //if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm->CommunicationCount*64*timestep/1e9/cputime); + // if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm->CommunicationCount*64*timestep/1e9/cputime); double *Vz; Vz= new double [Np]; @@ -263,7 +258,7 @@ int main(int argc, char **argv) if (rank == 0) { for (i=0;iid[n]); n = Map(i,j,k); //printf("%i,%i,%i; %i :",i,j,k,n); @@ -283,7 +278,7 @@ int main(int argc, char **argv) if (rank == 1) { for (i=0;iid[n]); n = Map(i,j,k); //printf("%i,%i,%i; %i :",i,j,k,n); diff --git a/tests/lbpm_squaretube_pp.cpp b/tests/lbpm_squaretube_pp.cpp index 93836e17..691ada62 100644 --- a/tests/lbpm_squaretube_pp.cpp +++ b/tests/lbpm_squaretube_pp.cpp @@ -116,9 +116,9 @@ int main(int argc, char **argv) } // Initialized domain and averaging framework for Two-Phase Flow - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - Dm.CommInit(comm); - TwoPhase Averages(Dm); + std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); + Dm->CommInit(comm); + std::shared_ptr Averages(new TwoPhase(Dm)); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, @@ -163,17 +163,17 @@ int main(int argc, char **argv) if (ORIENTATION==0){ // square capillary tube aligned with the x direction - Averages.SDs(i,j,k) = TubeWidth/2 - fabs(j-0.5*Ny); - Averages.SDs(i,j,k) = min(Averages.SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz)); + Averages->SDs(i,j,k) = TubeWidth/2 - fabs(j-0.5*Ny); + Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz)); // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.iproc()*Nx+kiproc()*Nx+kiproc()*Nx+kSDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx); + Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(k-0.5*Nz)); // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.jproc()*Ny+kjproc()*Ny+kjproc()*Ny+kSDs(i,j,k) = TubeWidth/2 - fabs(i-0.5*Nx); + Averages->SDs(i,j,k) = min(Averages->SDs(i,j,k),TubeWidth/2-fabs(j-0.5*Ny)); // Initialize phase positions - if (Averages.SDs(i,j,k) < 0.0){ + if (Averages->SDs(i,j,k) < 0.0){ id[n] = 0; } - else if (Dm.kproc()*Nz+kkproc()*Nz+kkproc()*Nz+kSDs.data(),8,Averages->SDs.length(),DIST); fclose(DIST); sprintf(LocalRankFilename,"ID.%05i",rank); From 9fa19f62f1f6f14d28a0b924edde9ce9cfe30bd6 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 17 May 2018 21:21:24 -0400 Subject: [PATCH 089/124] drink me --- tests/TestColorMassBounceback.cpp | 108 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index 5c2150a2..ad522963 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -151,7 +151,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; int BoundaryCondition=0; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition)); Nx += 2; Ny += 2; @@ -166,7 +166,7 @@ int main(int argc, char **argv) for (j=0;jid[n]=0; } } } @@ -174,21 +174,21 @@ int main(int argc, char **argv) for (j=1;jid[n]=1; Np++; // constant color PhaseLabel[n]= -1.0; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - ScaLBL_Communicator ScaLBL_Comm(Dm); + std::shared_ptr ScaLBL_Comm_Regular(new ScaLBL_Communicator(Dm)); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); // LBM variables if (rank==0) printf ("Set up the neighborlist \n"); @@ -198,7 +198,7 @@ int main(int argc, char **argv) IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Np]; - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -280,28 +280,29 @@ int main(int argc, char **argv) // Initialize the phase field and variables ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_Comm_Regular->RecvHalo(Phi); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -317,7 +318,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); nA=DENSITY[idx]; nB=DENSITY[Np+idx]; @@ -344,23 +345,23 @@ int main(int argc, char **argv) // *************EVEN TIMESTEP************* // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_Comm_Regular->RecvHalo(Phi); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -377,7 +378,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); nA=DENSITY[idx]; nB=DENSITY[Np+idx]; @@ -419,28 +420,29 @@ int main(int argc, char **argv) // Initialize the phase field and variables ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np); + ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_Comm_Regular->RecvHalo(Phi); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -457,7 +459,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); nA=DENSITY[idx]; nB=DENSITY[Np+idx]; @@ -485,23 +487,23 @@ int main(int argc, char **argv) // *************EVEN TIMESTEP************* // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); + ScaLBL_Comm_Regular->SendHalo(Phi); + ScaLBL_Comm_Regular->RecvHalo(Phi); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; printf("Check after even time \n"); @@ -517,7 +519,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); nA=DENSITY[idx]; nB=DENSITY[Np+idx]; From 8e16de5ae7a1ea921f17bb711dde7c6aa3f5175c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 11:19:17 -0400 Subject: [PATCH 090/124] bubble test --- tests/TestColorBubble.cpp | 126 ++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 73 deletions(-) diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 6f6b211f..550ba8db 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -11,80 +11,58 @@ #include "models/ColorModel.h" using namespace std; +inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){ + // initialize a bubble + int i,j,k,n; + int Np=0; + double sum=0.f; + int rank = ColorModel.Mask->rank(); + int nprocx = ColorModel.Mask->nprocx(); + int nprocy = ColorModel.Mask->nprocy(); + int nprocz = ColorModel.Mask->nprocz(); + int Nx = ColorModel.Mask->Nx; + int Ny = ColorModel.Mask->Ny; + int Nz = ColorModel.Mask->Nz; + if (rank == 0) cout << "Setting up bubble..." << endl; + sum=0; Np=0; + for (k=0;kSDs(i,j,k) = 100.f; + // Initialize phase positions field + if (ColorModel.Averages->SDs(i,j,k) < 0.0){ + ColorModel.Mask->id[n] = 0; + } + else { + sum++; + Np++; + } + } + } + } + // Initialize the bubble + for (k=0;kiproc(); + int jglobal= j+(Ny-2)*ColorModel.Dm->jproc(); + int kglobal= k+(Nz-2)*ColorModel.Dm->kproc(); + // Initialize phase position field for parallel bubble test + if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) + +(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy) + +(kglobal-0.5*(Nz-2)*nprocz)*(kglobal-0.5*(Nz-2)*nprocz) < BubbleRadius*BubbleRadius){ + ColorModel.Mask->id[n] = 2; + } + else{ + ColorModel.Mask->id[n]=1; + } + } + } + } -inline void AssignComponentLabels(char *id, double *phase, int Nx, int Ny, int Nz, int rank, MPI_Comm comm) -{ - int NLABELS=0; - char VALUE=0; - double AFFINITY=0.f; - - vector Label; - vector Affinity; - // Read the labels - if (rank==0){ - printf("Component labels:\n"); - ifstream iFILE("ComponentLabels.csv");\ - if (iFILE.good()){ - while (!iFILE.eof()){ - iFILE>>VALUE; - iFILE>>AFFINITY; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - printf("%i %f\n",VALUE,AFFINITY); - } - } - else{ - printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n"); - // Set default values - VALUE=0; AFFINITY=-1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - printf("%i %f\n",VALUE,AFFINITY); - VALUE=1; AFFINITY=1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - printf("%i %f\n",VALUE,AFFINITY); - VALUE=2; AFFINITY=-1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - printf("%i %f\n",VALUE,AFFINITY); - } - } - // Broadcast the list - MPI_Bcast(&NLABELS,1,MPI_INT,0,comm); - - // Copy into contiguous buffers - char *LabelList; - double * AffinityList; - LabelList=new char[NLABELS]; - AffinityList=new double[NLABELS]; - MPI_Bcast(&LabelList,NLABELS,MPI_CHAR,0,comm); - MPI_Bcast(&AffinityList,NLABELS,MPI_DOUBLE,0,comm); - - // Assign the labels - for (int k=0;k Date: Fri, 18 May 2018 11:19:42 -0400 Subject: [PATCH 091/124] tests/lbpm_dfh_simulator.cpp --- tests/lbpm_dfh_simulator.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index 4b16040b..532b2acc 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -147,15 +147,13 @@ int main(int argc, char **argv) else pBC=false; - // Full domain used for averaging (do not use mask for analysis) - Domain Dm(domain_db); - for (int i=0; i Averages( new TwoPhase(Dm) ); - // TwoPhase Averages(Dm); - Dm.CommInit(comm); + std::shared_ptr Dm (new Domain(domain_db)); + for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; + Dm->CommInit(comm); + std::shared_ptr Averages( new TwoPhase(Dm) ); // Mask that excludes the solid phase - Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); + std::shared_ptr Mask (new Domain(domain_db)); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; @@ -286,7 +284,7 @@ int main(int argc, char **argv) if (rank==0) printf("Media porosity = %f \n",porosity); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm.kproc() == 0){ + if (BoundaryCondition > 0 && Dm->kproc() == 0){ for (int k=0; k<3; k++){ for (int j=0;j 0 && Dm.kproc() == nprocz-1){ + if (BoundaryCondition > 0 && Dm->kproc() == nprocz-1){ for (int k=Nz-3; kNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; + Mask->CommInit(comm); double *PhaseLabel; PhaseLabel = new double[N]; - Mask.AssignComponentLabels(PhaseLabel); + Mask->AssignComponentLabels(PhaseLabel); //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Mask); + //ScaLBL_Communicator ScaLBL_Comm(Mask); + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + //Create a second communicator based on the regular data layout ScaLBL_Communicator ScaLBL_Comm_Regular(Mask); @@ -332,7 +332,7 @@ int main(int argc, char **argv) if (rank==0) printf ("Set up memory efficient layout \n"); IntArray Map(Nx,Ny,Nz); auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask.id,Np); + Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); MPI_Barrier(comm); //........................................................................... @@ -429,7 +429,7 @@ int main(int argc, char **argv) if (!(idk < Nz)) idk=Nz-1; int nn = idk*Nx*Ny + idj*Nx + idi; - if (!(Mask.id[nn] > 0)){ + if (!(Mask->id[nn] > 0)){ double vec_x = double(ii-2); double vec_y = double(jj-2); double vec_z = double(kk-2); @@ -489,7 +489,7 @@ int main(int argc, char **argv) int idx=Map(i,j,k); int n = k*Nx*Ny+j*Nx+i; if (!(idx < 0)){ - if (Mask.id[n] == 1) + if (Mask->id[n] == 1) PhaseLabel[idx] = 1.0; else { PhaseLabel[idx] = -1.0; @@ -513,9 +513,9 @@ int main(int argc, char **argv) //....................................................................... // Once phase has been initialized, map solid to account for 'smeared' interface - //for (i=0; iSDs(i) -= (1.0); // Make sure the id match for the two domains - for (int i=0; iid[i] = Mask->id[i]; //....................................................................... // Finalize setup for averaging domain Averages->UpdateSolid(); From 2e38569a837a85319bce5b6c2dec59aa9e1fb490 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 16:07:26 -0400 Subject: [PATCH 092/124] it all builds again --- CMakeLists.txt | 2 +- tests/CMakeLists.txt | 2 +- tests/TestBlobAnalyze.cpp | 18 ++--- tests/TestBubbleDFH.cpp | 122 ++++++++++++++--------------- tests/TestColorMassBounceback.cpp | 123 +++++++++++++----------------- tests/TestMap.cpp | 20 ++--- tests/lbpm_segmented_pp.cpp | 7 +- 7 files changed, 132 insertions(+), 162 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d108e8ed..de984c16 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -158,7 +158,7 @@ IF ( NOT ONLY_BUILD_DOCS ) ADD_PACKAGE_SUBDIRECTORY( analysis ) ADD_PACKAGE_SUBDIRECTORY( IO ) ADD_PACKAGE_SUBDIRECTORY( threadpool ) - #ADD_PACKAGE_SUBDIRECTORY( models ) + ADD_PACKAGE_SUBDIRECTORY( models ) IF ( USE_CUDA ) ADD_PACKAGE_SUBDIRECTORY( gpu ) ELSE() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index df3afe1f..77212b80 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,7 @@ ADD_LBPM_EXECUTABLE( lbpm_porenetwork_pp ) ADD_LBPM_EXECUTABLE( lbpm_plates_pp ) ADD_LBPM_EXECUTABLE( lbpm_squaretube_pp ) ADD_LBPM_EXECUTABLE( GenerateSphereTest ) -ADD_LBPM_EXECUTABLE( ComponentLabel ) +#ADD_LBPM_EXECUTABLE( ComponentLabel ) ADD_LBPM_EXECUTABLE( ColorToBinary ) ADD_LBPM_EXECUTABLE( DataAggregator ) ADD_LBPM_EXECUTABLE( BlobAnalysis ) diff --git a/tests/TestBlobAnalyze.cpp b/tests/TestBlobAnalyze.cpp index dc39cb3d..3adbcafe 100644 --- a/tests/TestBlobAnalyze.cpp +++ b/tests/TestBlobAnalyze.cpp @@ -56,7 +56,7 @@ inline void WriteBlobs(TwoPhase Averages){ inline void WriteBlobStates(TwoPhase TCAT, double D, double porosity){ int a; - double iVol=1.0/TCAT.Dm.Volume; + double iVol=1.0/TCAT.Dm->Volume; double PoreVolume; double nwp_volume,vol_n,pan,pn,pw,pawn,pwn,awn,ans,aws,Jwn,Kwn,lwns,cwns,clwns; double sw,awnD,awsD,ansD,lwnsDD,JwnD,pc; @@ -216,7 +216,7 @@ int main(int argc, char **argv) // Set up the domain int BC=0; // Get the rank info - Domain Dm(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); + std::shared_ptr Dm(new Domain(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); TwoPhase Averages(Dm); @@ -229,12 +229,12 @@ int main(int argc, char **argv) for ( j=1;jid[n] = 1; } } } //....................................................................... - Dm.CommInit(comm); // Initialize communications for domains + Dm->CommInit(comm); // Initialize communications for domains //....................................................................... // Read in sphere pack (initialize the non-wetting phase as inside of spheres) // @@ -264,7 +264,7 @@ int main(int argc, char **argv) MPI_Barrier(comm); //....................................................................... SignedDistance(Averages.Phase.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz, - Dm.iproc(),Dm.jproc(),Dm.kproc(),Dm.nprocx(),Dm.nprocy(),Dm.nprocz()); + Dm->iproc(),Dm->jproc(),Dm->kproc(),Dm->nprocx(),Dm->nprocy(),Dm->nprocz()); //....................................................................... // Assign the phase ID field based on the signed distance //....................................................................... @@ -277,10 +277,10 @@ int main(int argc, char **argv) Averages.SDs(i,j,k) = 100.0; Averages.Phase(i,j,k) += 2.0; if (Averages.Phase(i,j,k) > 0.0){ - Dm.id[n] = 2; + Dm->id[n] = 2; } else{ - Dm.id[n] = 1; + Dm->id[n] = 1; } Averages.SDn(i,j,k) = -Averages.Phase(i,j,k); Averages.Phase(i,j,k) = Averages.SDn(i,j,k); @@ -297,8 +297,8 @@ int main(int argc, char **argv) if (rank==0) printf("initializing the system \n"); Averages.UpdateSolid(); - Dm.CommunicateMeshHalo(Averages.Phase); - Dm.CommunicateMeshHalo(Averages.SDn); + Dm->CommunicateMeshHalo(Averages.Phase); + Dm->CommunicateMeshHalo(Averages.SDn); Averages.Initialize(); Averages.UpdateMeshValues(); diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index f30d44e9..39bd6b68 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -167,14 +167,14 @@ int main(int argc, char **argv) pBC=false; // Full domain used for averaging (do not use mask for analysis) - Domain Dm(domain_db); - for (int i=0; i Dm(new Domain(domain_db)); + for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; std::shared_ptr Averages( new TwoPhase(Dm) ); // TwoPhase Averages(Dm); - Dm.CommInit(comm); + Dm->CommInit(comm); // Mask that excludes the solid phase - Domain Mask(domain_db); + std::shared_ptr Mask(new Domain(domain_db)); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; @@ -226,9 +226,9 @@ int main(int argc, char **argv) for (j=0;jiproc(); + int jglobal= j+(Ny-2)*Dm->jproc(); + int kglobal= k+(Nz-2)*Dm->kproc(); // Initialize phase position field for parallel bubble test if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) +(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy) @@ -249,22 +249,22 @@ int main(int argc, char **argv) //......................................................... // Initialize communication structures in averaging domain - for (i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; + Mask->CommInit(comm); double *PhaseLabel; PhaseLabel = new double[N]; //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Mask); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Mask)); int Npad=(Np/16 + 2)*16; if (rank==0) printf ("Set up memory efficient layout Npad=%i \n",Npad); int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Mask.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); MPI_Barrier(comm); //........................................................................... @@ -339,8 +339,8 @@ int main(int argc, char **argv) Tmp[idx+Np] = value*dy; Tmp[idx+2*Np] = value*dz; // initialize fluid phases - if (Mask.id[n] == 1) PhaseLabel[idx] = 1.0; - else if (Mask.id[n] == 2){ + if (Mask->id[n] == 1) PhaseLabel[idx] = 1.0; + else if (Mask->id[n] == 2){ PhaseLabel[idx] = -1.0; count_wet +=1.0; } @@ -365,13 +365,13 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.last_interior, Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->last_interior, Np); //....................................................................... // Once phase has been initialized, map solid to account for 'smeared' interface //for (i=0; iid[i] = Mask->id[i]; //....................................................................... // Finalize setup for averaging domain Averages->UpdateSolid(); @@ -389,10 +389,10 @@ int main(int argc, char **argv) //........................................................................... ScaLBL_DeviceBarrier(); ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); - ScaLBL_Comm.RegularLayout(Map,Pressure,Averages->Press); - ScaLBL_Comm.RegularLayout(Map,&Velocity[0],Averages->Vel_x); - ScaLBL_Comm.RegularLayout(Map,&Velocity[Np],Averages->Vel_y); - ScaLBL_Comm.RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); + ScaLBL_Comm->RegularLayout(Map,Pressure,Averages->Press); + ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Averages->Vel_x); + ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Averages->Vel_y); + ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Averages->Vel_z); //........................................................................... if (rank==0) printf("********************************************************\n"); @@ -422,73 +422,73 @@ int main(int argc, char **argv) timestep++; // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set BCs if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); // *************EVEN TIMESTEP************* timestep++; // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->next, Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.next, Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->next, Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE // Set boundary conditions if (BoundaryCondition > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); + ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB); + ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); } if (BoundaryCondition == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } else if (BoundaryCondition == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); + din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); + ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); } ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.next, Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); //************************************************************************ MPI_Barrier(comm); @@ -522,7 +522,7 @@ int main(int argc, char **argv) // Copy back final phase indicator field and convert to regular layout DoubleArray PhaseField(Nx,Ny,Nz); - ScaLBL_Comm.RegularLayout(Map,Phi,PhaseField); + ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); FILE *OUTFILE; sprintf(LocalRankFilename,"Phase.raw",rank); OUTFILE = fopen(LocalRankFilename,"wb"); @@ -533,9 +533,9 @@ int main(int argc, char **argv) DoubleArray Cy(Nx,Ny,Nz); DoubleArray Cz(Nx,Ny,Nz); DoubleArray GradNorm(Nx,Ny,Nz); - ScaLBL_Comm.RegularLayout(Map,&Gradient[0],Cx); - ScaLBL_Comm.RegularLayout(Map,&Gradient[Np],Cy); - ScaLBL_Comm.RegularLayout(Map,&Gradient[2*Np],Cz); + ScaLBL_Comm->RegularLayout(Map,&Gradient[0],Cx); + ScaLBL_Comm->RegularLayout(Map,&Gradient[Np],Cy); + ScaLBL_Comm->RegularLayout(Map,&Gradient[2*Np],Cz); for (k=1; kRegularLayout(Map,&Den[0],Rho1); + ScaLBL_Comm->RegularLayout(Map,&Den[Np],Rho2); FILE *RFILE1; sprintf(LocalRankFilename,"Rho1.raw",rank); RFILE1 = fopen(LocalRankFilename,"wb"); diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index 3d9cabd8..e8df5ae8 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -27,7 +27,6 @@ int main(int argc, char **argv) int check; { // parallel domain size (# of sub-domains) - int iproc,jproc,kproc; int i,j,k,n,Npad; auto filename = argv[1]; auto db = std::make_shared( filename ); @@ -50,7 +49,6 @@ int main(int argc, char **argv) string FILENAME; // Color Model parameters - int timestepMax = color_db->getScalar( "timestepMax" ); double tauA = color_db->getScalar( "tauA" ); double tauB = color_db->getScalar( "tauB" ); double rhoA = color_db->getScalar( "rhoA" ); @@ -62,32 +60,20 @@ int main(int argc, char **argv) double beta = color_db->getScalar( "beta" ); bool Restart = color_db->getScalar( "Restart" ); double din = color_db->getScalar( "din" ); - double dout = color_db->getScalar( "dout" );; - double inletA=1.f; - double inletB=0.f; - double outletA=0.f; - double outletB=1.f; - double flux = 10.f; // Read domain values - auto L = domain_db->getVector( "L" ); auto size = domain_db->getVector( "n" ); auto nproc = domain_db->getVector( "nproc" ); int BoundaryCondition = domain_db->getScalar( "BC" ); int Nx = size[0]; int Ny = size[1]; int Nz = size[2]; - double Lx = L[0]; - double Ly = L[1]; - double Lz = L[2]; int nprocx = nproc[0]; int nprocy = nproc[1]; int nprocz = nproc[2]; int timestep = 6; - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details - // Get the rank info const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); @@ -132,12 +118,9 @@ int main(int argc, char **argv) pBC=false; // Full domain used for averaging (do not use mask for analysis) - Domain Dm(domain_db); - for (int i=0; i Dm(new Domain(domain_db)); + for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; + Dm->CommInit(comm); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; @@ -265,30 +248,30 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.LastInterior(), Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -331,24 +314,24 @@ int main(int argc, char **argv) } // EVEN TIMESTEP // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ @@ -407,31 +390,31 @@ int main(int argc, char **argv) if (rank==0) printf ("Initializing distributions \n"); ScaLBL_D3Q19_Init(fq, Np); if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm.LastInterior(), Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastInterior(), Np); // *************ODD TIMESTEP************* // Compute the Phase indicator field // Read for Aq, Bq happens in this routine (requires communication) // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAodd_DFH(NeighborList, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; @@ -475,24 +458,24 @@ int main(int argc, char **argv) // EVEN TIMESTEP // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm.LastExterior(), Np); + ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE + ScaLBL_D3Q7_AAeven_DFH(Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np); // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm.LastExterior(), Np); - ScaLBL_Comm.RecvGrad(Phi,Gradient); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(NeighborList, Phi, Gradient, SolidPotential, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_Comm->RecvGrad(Phi,Gradient); // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, ScaLBL_Comm.FirstInterior(), ScaLBL_Comm.LastInterior(), Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + alpha, beta, Fx, Fy, Fz, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm.LastExterior(), Np); + alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np); ScaLBL_DeviceBarrier(); MPI_Barrier(comm); timestep++; //************************************************************************ diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index f3ca09d4..17541753 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -23,24 +23,18 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { // parallel domain size (# of sub-domains) int nprocx,nprocy,nprocz; - int iproc,jproc,kproc; - - if (rank == 0){ printf("********************************************************\n"); - printf("Running Color Model: TestColor \n"); + printf("Running Color Model: TestMap \n"); printf("********************************************************\n"); } // BGK Model parameters string FILENAME; - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double Fx,Fy,Fz,tol; // Domain variables double Lx,Ly,Lz; int nspheres; @@ -176,15 +170,15 @@ int main(int argc, char **argv) MPI_Barrier(comm); // Check the neighborlist - printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm.LastExterior(),ScaLBL_Comm.FirstInterior(),ScaLBL_Comm.LastInterior()); - for (int idx=0; idxLastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior()); + for (int idx=0; idxLastExterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); } } - for (int idx=ScaLBL_Comm.FirstInterior(); idxFirstInterior(); idxLastInterior(); idx++){ for (int q=0; q<18; q++){ int nn = neighborList[q*Np+idx]%Np; if (nn>Np) printf("neighborlist error (exterior) at q=%i, idx=%i \n",q,idx); @@ -193,12 +187,8 @@ int main(int argc, char **argv) } //......................device distributions................................. - int dist_mem_size = Np*sizeof(double); - if (rank==0) printf ("Allocating distributions \n"); - int *NeighborList; int *dvcMap; - //........................................................................... ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Npad); diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index d7288d7f..d7cbd15f 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -44,7 +44,6 @@ double ReadFromBlock( char *ID, int iproc, int jproc, int kproc, int Nx, int Ny, char sx[2]; char sy[2]; char sz[2]; - char tmpstr[10]; // array to store ids read from block char *id; @@ -62,9 +61,8 @@ double ReadFromBlock( char *ID, int iproc, int jproc, int kproc, int Nx, int Ny, //fflush(stdout); // Read the file - size_t readID; FILE *IDFILE = fopen(LocalRankFilename,"rb"); - readID=fread(id,1,1024*1024*1024,IDFILE); + fread(id,1,1024*1024*1024,IDFILE); fclose(IDFILE); //printf("Loading data ... \n"); @@ -128,7 +126,6 @@ int main(int argc, char **argv) //....................................................................... int nprocx, nprocy, nprocz, nx, ny, nz, nspheres; double Lx, Ly, Lz; - int Nx,Ny,Nz; int i,j,k,n; int BC=0; // char fluidValue,solidValue; @@ -254,7 +251,7 @@ int main(int argc, char **argv) MeanFilter(Averages->SDs); if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n"); - CalcDist(Averages.SDs,id,Dm); + CalcDist(Averages->SDs,id,*Dm); sprintf(LocalRankFilename,"SignDist.%05i",rank); FILE *DIST = fopen(LocalRankFilename,"wb"); From 27f4ce389104f50a27ca8cd69759a1bdacd352fe Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 16:08:49 -0400 Subject: [PATCH 093/124] the pool of tears --- analysis/runAnalysis.cpp | 2 +- tests/ComponentLabel.cpp | 61 -------------------------------------- tests/TestColorGradDFH.cpp | 22 +++++++------- 3 files changed, 12 insertions(+), 73 deletions(-) diff --git a/analysis/runAnalysis.cpp b/analysis/runAnalysis.cpp index dccf9c3c..38f9bdce 100644 --- a/analysis/runAnalysis.cpp +++ b/analysis/runAnalysis.cpp @@ -289,7 +289,7 @@ runAnalysis::runAnalysis( std::shared_ptr db, d_rank_info( rank_info ), d_Map( Map ), d_ScaLBL_Comm( ScaLBL_Comm), - d_fillData(Dm->Comm,Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,1,1,1,0,1) + d_fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1) { NULL_USE( pBC ); INSIST( db, "Input database is empty" ); diff --git a/tests/ComponentLabel.cpp b/tests/ComponentLabel.cpp index f350a5be..0a18ba0b 100644 --- a/tests/ComponentLabel.cpp +++ b/tests/ComponentLabel.cpp @@ -12,68 +12,7 @@ using namespace std; -/*inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cDistEven, double *cDistOdd, int N) -{ - int q,n; - double value; - ifstream File(FILENAME,ios::binary); - for (n=0; n Dm(new Domain(db)); Nx += 2; Ny += 2; Nz += 2; @@ -101,20 +101,20 @@ int main(int argc, char **argv) for (j=0;jid[n]=1; // Initialize gradient ColorGrad = (1,2,3) double value=double(3*k+2*j+i); PhaseLabel[n]= value; } } } - Dm.CommInit(comm); + Dm->CommInit(comm); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm(Dm); + std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); // LBM variables if (rank==0) printf ("Set up the neighborlist \n"); @@ -131,7 +131,7 @@ int main(int argc, char **argv) int *neighborList; IntArray Map(Nx,Ny,Nz); neighborList= new int[18*Npad]; - Np = ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm.id,Np); + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); MPI_Barrier(comm); //......................device distributions................................. @@ -188,10 +188,10 @@ int main(int argc, char **argv) //........................................................................... // compute the gradient - ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, ScaLBL_Comm.first_interior, ScaLBL_Comm.last_interior, Np); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, 0, ScaLBL_Comm.first_interior, Np); - ScaLBL_Comm.RecvGrad(Phi,ColorGrad); + ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np); + ScaLBL_Comm->SendHalo(Phi); + ScaLBL_D3Q19_Gradient_DFH(neighborList, Phi, ColorGrad, Potential, 0, ScaLBL_Comm->first_interior, Np); + ScaLBL_Comm->RecvGrad(Phi,ColorGrad); double *COLORGRAD; COLORGRAD= new double [3*Np]; @@ -202,7 +202,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); printf("%i ",idx); } @@ -217,7 +217,7 @@ int main(int argc, char **argv) for (j=1;j 0){ + if (Dm->id[n] > 0){ int idx = Map(i,j,k); CX=COLORGRAD[idx]; CY=COLORGRAD[Np+idx]; From f1529e2153ef3fecd8d4e0a32ced8413f2a1e3da Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 17:24:17 -0400 Subject: [PATCH 094/124] advice from a caterpillar --- models/ColorModel.cpp | 24 ++++++--- models/ColorModel.h | 5 +- tests/TestColorBubble.cpp | 102 ++++++++++++++++++++------------------ 3 files changed, 74 insertions(+), 57 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index f6ef66eb..21e58066 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -3,10 +3,10 @@ color lattice boltzmann model */ #include "models/ColorModel.h" -ScaLBL_ColorModel::ScaLBL_ColorModel(): - Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), +ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM): + rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), - Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0) + Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { } @@ -35,7 +35,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ Restart = color_db->getScalar( "Restart" ); din = color_db->getScalar( "din" ); dout = color_db->getScalar( "dout" ); - flux = color_db->getScalar( "flux" );; + flux = color_db->getScalar( "flux" ); inletA=1.f; inletB=0.f; outletA=0.f; @@ -65,6 +65,8 @@ void ScaLBL_ColorModel::ReadParams(string filename){ N = Nx*Ny*Nz; for (int i=0; iid[i] = 1; // initialize this way Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + + id = new char[N]; MPI_Barrier(comm); Dm->CommInit(comm); @@ -81,7 +83,6 @@ void ScaLBL_ColorModel::ReadInput(){ sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); // .......... READ THE INPUT FILE ....................................... - id = new char[N]; //........................................................................... if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; //....................................................................... @@ -201,7 +202,7 @@ void ScaLBL_ColorModel::Create(){ int Npad=(Np/16 + 2)*16; if (rank==0) printf ("Set up memory efficient layout \n"); - Map.resize(Nx,Ny,Nz); Map.fill(0); + Map.resize(Nx,Ny,Nz); Map.fill(-2); auto neighborList= new int[18*Npad]; Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); MPI_Barrier(comm); @@ -510,3 +511,14 @@ void ScaLBL_ColorModel::Run(){ // ************************************************************************ } + +void ScaLBL_ColorModel::WriteDebug(){ + // Copy back final phase indicator field and convert to regular layout + DoubleArray PhaseField(Nx,Ny,Nz); + ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField); + FILE *OUTFILE; + sprintf(LocalRankFilename,"Phase.%05i.raw",rank); + OUTFILE = fopen(LocalRankFilename,"wb"); + fwrite(PhaseField.data(),8,N,OUTFILE); + fclose(OUTFILE); +} \ No newline at end of file diff --git a/models/ColorModel.h b/models/ColorModel.h index f99b683e..17067185 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -18,7 +18,7 @@ Implementation of color lattice boltzmann model class ScaLBL_ColorModel{ public: - ScaLBL_ColorModel(); + ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM); ~ScaLBL_ColorModel(); // functions in they should be run @@ -28,6 +28,7 @@ public: void Create(); void Initialize(); void Run(); + void WriteDebug(); bool Restart,pBC; int timestep,timestepMax; @@ -37,7 +38,7 @@ public: double din,dout,inletA,inletB,outletA,outletB; int Nx,Ny,Nz,N,Np; - int nprocx,nprocy,nprocz; + int rank,nprocx,nprocy,nprocz,nprocs; double Lx,Ly,Lz; std::shared_ptr Dm; // this domain is for analysis diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 550ba8db..96313ac7 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -12,55 +12,58 @@ using namespace std; inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){ - // initialize a bubble - int i,j,k,n; - int Np=0; - double sum=0.f; - int rank = ColorModel.Mask->rank(); - int nprocx = ColorModel.Mask->nprocx(); - int nprocy = ColorModel.Mask->nprocy(); - int nprocz = ColorModel.Mask->nprocz(); - int Nx = ColorModel.Mask->Nx; - int Ny = ColorModel.Mask->Ny; - int Nz = ColorModel.Mask->Nz; - if (rank == 0) cout << "Setting up bubble..." << endl; - sum=0; Np=0; - for (k=0;kSDs(i,j,k) = 100.f; - // Initialize phase positions field - if (ColorModel.Averages->SDs(i,j,k) < 0.0){ - ColorModel.Mask->id[n] = 0; - } - else { - sum++; - Np++; - } - } - } + // initialize a bubble + int i,j,k,n; + int Np=0; + double sum=0.f; + int rank = ColorModel.Mask->rank(); + int nprocx = ColorModel.Mask->nprocx(); + int nprocy = ColorModel.Mask->nprocy(); + int nprocz = ColorModel.Mask->nprocz(); + int Nx = ColorModel.Mask->Nx; + int Ny = ColorModel.Mask->Ny; + int Nz = ColorModel.Mask->Nz; + if (rank == 0) cout << "Setting up bubble..." << endl; + sum=0; Np=0; + for (k=0;kSDs(i,j,k) = 100.f; + // Initialize phase positions field + if (ColorModel.Averages->SDs(i,j,k) < 0.0){ + ColorModel.id[n] = 0; + ColorModel.id[n] = 0; + } + else { + sum++; + Np++; + } + } } - // Initialize the bubble - for (k=0;kiproc(); - int jglobal= j+(Ny-2)*ColorModel.Dm->jproc(); - int kglobal= k+(Nz-2)*ColorModel.Dm->kproc(); - // Initialize phase position field for parallel bubble test - if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) - +(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy) - +(kglobal-0.5*(Nz-2)*nprocz)*(kglobal-0.5*(Nz-2)*nprocz) < BubbleRadius*BubbleRadius){ - ColorModel.Mask->id[n] = 2; - } - else{ - ColorModel.Mask->id[n]=1; - } - } - } + } + // Initialize the bubble + for (k=0;kiproc(); + int jglobal= j+(Ny-2)*ColorModel.Dm->jproc(); + int kglobal= k+(Nz-2)*ColorModel.Dm->kproc(); + // Initialize phase position field for parallel bubble test + if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) + +(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy) + +(kglobal-0.5*(Nz-2)*nprocz)*(kglobal-0.5*(Nz-2)*nprocz) < BubbleRadius*BubbleRadius){ + ColorModel.id[n] = 2; + ColorModel.id[n] = 2; + } + else{ + ColorModel.id[n]=1; + ColorModel.id[n]=1; + } + } } + } } //*************************************************************************************** @@ -75,7 +78,7 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { if (rank == 0){ printf("********************************************************\n"); @@ -87,7 +90,7 @@ int main(int argc, char **argv) } } auto filename = argv[1]; - ScaLBL_ColorModel ColorModel; + ScaLBL_ColorModel ColorModel(rank,nprocs,comm); ColorModel.ReadParams(filename); //ColorModel.ReadInput(); double radius=15.5; @@ -95,6 +98,7 @@ int main(int argc, char **argv) ColorModel.Create(); ColorModel.Initialize(); ColorModel.Run(); + ColorModel.WriteDebug(); } // **************************************************** MPI_Barrier(comm); From 96fe20e097a70853090ae4be44d704cb6db3dc14 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 20:04:03 -0400 Subject: [PATCH 095/124] pig and pepper --- common/Domain.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/Domain.h b/common/Domain.h index af4e5485..b468b76c 100755 --- a/common/Domain.h +++ b/common/Domain.h @@ -175,8 +175,6 @@ public: // Public variables (need to create accessors instead) void CommInit(MPI_Comm comm); void TestCommInit(MPI_Comm comm); - //void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np); - private: int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; From 35d3dfbcc58a406142e12150b2cbdf78b0365324 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 18 May 2018 20:53:49 -0400 Subject: [PATCH 096/124] mad tea party --- common/Domain.cpp | 358 ++++++++++++++++++++---------------------- common/Domain.h | 11 +- models/ColorModel.cpp | 279 +++++++++++++++++--------------- models/ColorModel.h | 1 + 4 files changed, 337 insertions(+), 312 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 0adbec81..0db7f88e 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -29,7 +29,7 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, double lx, double ly, double lz, int BC): Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), - Comm(MPI_COMM_NULL), + Comm(MPI_COMM_WORLD), sendCount_x(0), sendCount_y(0), sendCount_z(0), sendCount_X(0), sendCount_Y(0), sendCount_Z(0), sendCount_xy(0), sendCount_yz(0), sendCount_xz(0), sendCount_Xy(0), sendCount_Yz(0), sendCount_xZ(0), sendCount_xY(0), sendCount_yZ(0), sendCount_Xz(0), sendCount_XY(0), sendCount_YZ(0), sendCount_XZ(0), @@ -55,7 +55,15 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, recvData_xy(NULL), recvData_yz(NULL), recvData_xz(NULL), recvData_Xy(NULL), recvData_Yz(NULL), recvData_xZ(NULL), recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), id(NULL) -{ +{ + // set up the neighbor ranks + int myrank; + MPI_Comm_rank( Comm, &myrank ); + rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); + + initialize( db ); + MPI_Barrier(Comm); + auto db = std::make_shared( ); db->putScalar( "BC", BC ); db->putVector( "nproc", { npx, npx, npx } ); @@ -64,7 +72,7 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, db->putVector( "L", { lx, ly, lz } ); initialize( db ); } -Domain::Domain( std::shared_ptr db ): +Domain::Domain( std::shared_ptr db, MPI_Communicator Communicator): Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), Comm(MPI_COMM_NULL), @@ -94,10 +102,19 @@ Domain::Domain( std::shared_ptr db ): recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL), id(NULL) { + MPI_Comm_dup(Communicator,&Comm); + + // set up the neighbor ranks + int myrank; + MPI_Comm_rank( Comm, &myrank ); + rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); + initialize( db ); + MPI_Barrier(Comm); + } void Domain::initialize( std::shared_ptr db ) -{ +{ d_db = db; auto nproc = d_db->getVector("nproc"); auto n = d_db->getVector("n"); @@ -117,7 +134,7 @@ void Domain::initialize( std::shared_ptr db ) Nz = nz+2; // Initialize ranks int myrank; - MPI_Comm_rank( MPI_COMM_WORLD, &myrank ); + MPI_Comm_rank( Comm, &myrank ); rank_info = RankInfoStruct(myrank,nproc[0],nproc[1],nproc[2]); // Fill remaining variables N = Nx*Ny*Nz; @@ -126,7 +143,7 @@ void Domain::initialize( std::shared_ptr db ) memset(id,0,N); BoundaryCondition = d_db->getScalar("BC"); int nprocs; - MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); + MPI_Comm_size( Comm, &nprocs ); INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!"); } Domain::~Domain() @@ -185,24 +202,11 @@ Domain::~Domain() /******************************************************** * Initialize communication * ********************************************************/ -void Domain::CommInit(MPI_Comm Communicator) +void Domain::CommInit() { int i,j,k,n; int sendtag = 21; int recvtag = 21; - - MPI_Comm_dup(Communicator,&Comm); - - // set up the neighbor ranks - int myrank; - MPI_Comm_rank( Comm, &myrank ); - rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); - - MPI_Barrier(Communicator); - - //...................................................................................... - MPI_Request req1[18], req2[18]; - MPI_Status stat1[18],stat2[18]; //...................................................................................... sendCount_x = sendCount_y = sendCount_z = sendCount_X = sendCount_Y = sendCount_Z = 0; sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0; @@ -316,45 +320,45 @@ void Domain::CommInit(MPI_Comm Communicator) sendBuf_YZ = new int [sendCount_YZ]; sendBuf_XZ = new int [sendCount_XZ]; //...................................................................................... - MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x(),sendtag+0,Communicator,&req1[0]); - MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X(),recvtag+0,Communicator,&req2[0]); - MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X(),sendtag+1,Communicator,&req1[1]); - MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x(),recvtag+1,Communicator,&req2[1]); - MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y(),sendtag+2,Communicator,&req1[2]); - MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y(),recvtag+2,Communicator,&req2[2]); - MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y(),sendtag+3,Communicator,&req1[3]); - MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y(),recvtag+3,Communicator,&req2[3]); - MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z(),sendtag+4,Communicator,&req1[4]); - MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z(),recvtag+4,Communicator,&req2[4]); - MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z(),sendtag+5,Communicator,&req1[5]); - MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z(),recvtag+5,Communicator,&req2[5]); - MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy(),sendtag+6,Communicator,&req1[6]); - MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY(),recvtag+6,Communicator,&req2[6]); - MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY(),sendtag+7,Communicator,&req1[7]); - MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy(),recvtag+7,Communicator,&req2[7]); - MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy(),sendtag+8,Communicator,&req1[8]); - MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY(),recvtag+8,Communicator,&req2[8]); - MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY(),sendtag+9,Communicator,&req1[9]); - MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy(),recvtag+9,Communicator,&req2[9]); - MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz(),sendtag+10,Communicator,&req1[10]); - MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ(),recvtag+10,Communicator,&req2[10]); - MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ(),sendtag+11,Communicator,&req1[11]); - MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz(),recvtag+11,Communicator,&req2[11]); - MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz(),sendtag+12,Communicator,&req1[12]); - MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ(),recvtag+12,Communicator,&req2[12]); - MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ(),sendtag+13,Communicator,&req1[13]); - MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz(),recvtag+13,Communicator,&req2[13]); - MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz(),sendtag+14,Communicator,&req1[14]); - MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ(),recvtag+14,Communicator,&req2[14]); - MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ(),sendtag+15,Communicator,&req1[15]); - MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz(),recvtag+15,Communicator,&req2[15]); - MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz(),sendtag+16,Communicator,&req1[16]); - MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ(),recvtag+16,Communicator,&req2[16]); - MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ(),sendtag+17,Communicator,&req1[17]); - MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz(),recvtag+17,Communicator,&req2[17]); + MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x(),sendtag+0,Comm,&req1[0]); + MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X(),recvtag+0,Comm,&req2[0]); + MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X(),sendtag+1,Comm,&req1[1]); + MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x(),recvtag+1,Comm,&req2[1]); + MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y(),sendtag+2,Comm,&req1[2]); + MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y(),recvtag+2,Comm,&req2[2]); + MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y(),sendtag+3,Comm,&req1[3]); + MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y(),recvtag+3,Comm,&req2[3]); + MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z(),sendtag+4,Comm,&req1[4]); + MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z(),recvtag+4,Comm,&req2[4]); + MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z(),sendtag+5,Comm,&req1[5]); + MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z(),recvtag+5,Comm,&req2[5]); + MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy(),sendtag+6,Comm,&req1[6]); + MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY(),recvtag+6,Comm,&req2[6]); + MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY(),sendtag+7,Comm,&req1[7]); + MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy(),recvtag+7,Comm,&req2[7]); + MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy(),sendtag+8,Comm,&req1[8]); + MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY(),recvtag+8,Comm,&req2[8]); + MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY(),sendtag+9,Comm,&req1[9]); + MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy(),recvtag+9,Comm,&req2[9]); + MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz(),sendtag+10,Comm,&req1[10]); + MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ(),recvtag+10,Comm,&req2[10]); + MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ(),sendtag+11,Comm,&req1[11]); + MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz(),recvtag+11,Comm,&req2[11]); + MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz(),sendtag+12,Comm,&req1[12]); + MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ(),recvtag+12,Comm,&req2[12]); + MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ(),sendtag+13,Comm,&req1[13]); + MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz(),recvtag+13,Comm,&req2[13]); + MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz(),sendtag+14,Comm,&req1[14]); + MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ(),recvtag+14,Comm,&req2[14]); + MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ(),sendtag+15,Comm,&req1[15]); + MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz(),recvtag+15,Comm,&req2[15]); + MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz(),sendtag+16,Comm,&req1[16]); + MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ(),recvtag+16,Comm,&req2[16]); + MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ(),sendtag+17,Comm,&req1[17]); + MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz(),recvtag+17,Comm,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); - MPI_Barrier(Communicator); + MPI_Barrier(Comm); //...................................................................................... // recv buffers recvList_x = new int [recvCount_x]; @@ -376,42 +380,42 @@ void Domain::CommInit(MPI_Comm Communicator) recvList_YZ = new int [recvCount_YZ]; recvList_XZ = new int [recvCount_XZ]; //...................................................................................... - MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x(),sendtag,Communicator,&req1[0]); - MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X(),recvtag,Communicator,&req2[0]); - MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X(),sendtag,Communicator,&req1[1]); - MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x(),recvtag,Communicator,&req2[1]); - MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y(),sendtag,Communicator,&req1[2]); - MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y(),recvtag,Communicator,&req2[2]); - MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y(),sendtag,Communicator,&req1[3]); - MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y(),recvtag,Communicator,&req2[3]); - MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z(),sendtag,Communicator,&req1[4]); - MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z(),recvtag,Communicator,&req2[4]); - MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z(),sendtag,Communicator,&req1[5]); - MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z(),recvtag,Communicator,&req2[5]); - MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy(),sendtag,Communicator,&req1[6]); - MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY(),recvtag,Communicator,&req2[6]); - MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY(),sendtag,Communicator,&req1[7]); - MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy(),recvtag,Communicator,&req2[7]); - MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy(),sendtag,Communicator,&req1[8]); - MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY(),recvtag,Communicator,&req2[8]); - MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY(),sendtag,Communicator,&req1[9]); - MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy(),recvtag,Communicator,&req2[9]); - MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz(),sendtag,Communicator,&req1[10]); - MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ(),recvtag,Communicator,&req2[10]); - MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ(),sendtag,Communicator,&req1[11]); - MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz(),recvtag,Communicator,&req2[11]); - MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz(),sendtag,Communicator,&req1[12]); - MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ(),recvtag,Communicator,&req2[12]); - MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ(),sendtag,Communicator,&req1[13]); - MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz(),recvtag,Communicator,&req2[13]); - MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz(),sendtag,Communicator,&req1[14]); - MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ(),recvtag,Communicator,&req2[14]); - MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ(),sendtag,Communicator,&req1[15]); - MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz(),recvtag,Communicator,&req2[15]); - MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz(),sendtag,Communicator,&req1[16]); - MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ(),recvtag,Communicator,&req2[16]); - MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ(),sendtag,Communicator,&req1[17]); - MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz(),recvtag,Communicator,&req2[17]); + MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x(),sendtag,Comm,&req1[0]); + MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X(),recvtag,Comm,&req2[0]); + MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X(),sendtag,Comm,&req1[1]); + MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x(),recvtag,Comm,&req2[1]); + MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y(),sendtag,Comm,&req1[2]); + MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y(),recvtag,Comm,&req2[2]); + MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y(),sendtag,Comm,&req1[3]); + MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y(),recvtag,Comm,&req2[3]); + MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z(),sendtag,Comm,&req1[4]); + MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z(),recvtag,Comm,&req2[4]); + MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z(),sendtag,Comm,&req1[5]); + MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z(),recvtag,Comm,&req2[5]); + MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy(),sendtag,Comm,&req1[6]); + MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY(),recvtag,Comm,&req2[6]); + MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY(),sendtag,Comm,&req1[7]); + MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy(),recvtag,Comm,&req2[7]); + MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy(),sendtag,Comm,&req1[8]); + MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY(),recvtag,Comm,&req2[8]); + MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY(),sendtag,Comm,&req1[9]); + MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy(),recvtag,Comm,&req2[9]); + MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz(),sendtag,Comm,&req1[10]); + MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ(),recvtag,Comm,&req2[10]); + MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ(),sendtag,Comm,&req1[11]); + MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz(),recvtag,Comm,&req2[11]); + MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz(),sendtag,Comm,&req1[12]); + MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ(),recvtag,Comm,&req2[12]); + MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ(),sendtag,Comm,&req1[13]); + MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz(),recvtag,Comm,&req2[13]); + MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz(),sendtag,Comm,&req1[14]); + MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ(),recvtag,Comm,&req2[14]); + MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ(),sendtag,Comm,&req1[15]); + MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz(),recvtag,Comm,&req2[15]); + MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz(),sendtag,Comm,&req1[16]); + MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ(),recvtag,Comm,&req2[16]); + MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ(),sendtag,Comm,&req1[17]); + MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz(),recvtag,Comm,&req2[17]); MPI_Waitall(18,req1,stat1); MPI_Waitall(18,req2,stat2); //...................................................................................... @@ -497,98 +501,84 @@ void Domain::CommInit(MPI_Comm Communicator) } - -/******************************************************** - * AssignComponentLabels * - ********************************************************/ -void Domain::AssignComponentLabels(double *phase) -{ - int NLABELS=0; - char VALUE=0; - double AFFINITY=0.f; - - vector Label; - vector Affinity; - // Read the labels - if (rank()==0){ - printf("Component labels:\n"); - ifstream iFILE("ComponentLabels.csv"); - if (iFILE.good()){ - int value; - while (!iFILE.eof()){ - iFILE>>value; - iFILE>>AFFINITY; - VALUE=char(value); - Label.push_back(value); - Affinity.push_back(AFFINITY); - NLABELS++; - printf("%i %f\n",VALUE,AFFINITY); - } - } - else{ - printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n"); - // Set default values - VALUE=0; AFFINITY=-1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - VALUE=1; AFFINITY=1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - VALUE=2; AFFINITY=-1.0; - Label.push_back(VALUE); - Affinity.push_back(AFFINITY); - NLABELS++; - } - } - MPI_Barrier(Comm); - - // Broadcast the list - MPI_Bcast(&NLABELS,1,MPI_INT,0,Comm); - //printf("rank=%i, NLABELS=%i \n ",rank(),NLABELS); - - // Copy into contiguous buffers - char *LabelList; - double * AffinityList; - LabelList=new char[NLABELS]; - AffinityList=new double[NLABELS]; - - if (rank()==0){ - for (int idx=0; idx < NLABELS; idx++){ - VALUE=Label[idx]; - AFFINITY=Affinity[idx]; - printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank(),idx,VALUE,AFFINITY); - LabelList[idx]=VALUE; - AffinityList[idx]=AFFINITY; - } - } - MPI_Barrier(Comm); - - MPI_Bcast(LabelList,NLABELS,MPI_CHAR,0,Comm); - MPI_Bcast(AffinityList,NLABELS,MPI_DOUBLE,0,Comm); - - // Assign the labels - for (int k=0;krank()); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + // .......... READ THE INPUT FILE ....................................... + if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); + sprintf(LocalRankFilename,"ID.%05i",rank); + FILE *IDFILE = fopen(LocalRankFilename,"rb"); + if (IDFILE==NULL) ERROR("Domain::ReadIDs -- Error opening file: ID.xxxxx"); + readID=fread(id,1,N,IDFILE); + if (readID != size_t(N)) printf("Domain::ReadIDs -- Error reading ID (rank=%i) \n",Dm->rank()); + fclose(IDFILE); + // Compute the porosity + double sum; + double porosity; + double sum_local=0.0; + double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + //......................................................... + // If external boundary conditions are applied remove solid + if (BoundaryCondition > 0 && Dm->kproc() == 0){ + for (int k=0; k<3; k++){ + for (int j=0;j 0 && Dm->kproc() == nprocz-1){ + for (int k=Nz-3; k 0){ + sum_local+=1.0; + } + } + } + } + MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm); + porosity = sum*iVol_global; + if (rank==0) printf("Media porosity = %f \n",porosity); + //......................................................... +} +int Domain::PoreCount(){ + /* + * count the number of nodes occupied by mobile phases + */ + Npore=0; // number of local pore nodes + for (int k=1;k 0){ + Npore++; + } + } + } + } + return Npore; } - - void Domain::CommunicateMeshHalo(DoubleArray &Mesh) { diff --git a/common/Domain.h b/common/Domain.h index b468b76c..c6d182a7 100755 --- a/common/Domain.h +++ b/common/Domain.h @@ -64,7 +64,7 @@ private: class Domain{ public: //! Default constructor - Domain( std::shared_ptr db ); + Domain( std::shared_ptr db, MPI_Communicator Communicator); //! Obsolete constructor Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, @@ -170,12 +170,15 @@ public: // Public variables (need to create accessors instead) // Solid indicator function char *id; + void ReadIDs(); void CommunicateMeshHalo(DoubleArray &Mesh); - void AssignComponentLabels(double *phase); - void CommInit(MPI_Comm comm); - void TestCommInit(MPI_Comm comm); + void CommInit(); + int PoreCount() private: + //...................................................................................... + MPI_Request req1[18], req2[18]; + MPI_Status stat1[18],stat2[18]; int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z; int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ; diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 21e58066..8f8a198b 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -75,16 +75,16 @@ void ScaLBL_ColorModel::ReadParams(string filename){ void ScaLBL_ColorModel::ReadInput(){ int rank=Dm->rank(); - size_t readID; //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + // .......... READ THE INPUT FILE ....................................... //........................................................................... - if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; + if (rank == 0) cout << "Reading in signed distance function..." << endl; //....................................................................... sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); @@ -92,17 +92,10 @@ void ScaLBL_ColorModel::ReadInput(){ MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; - if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); - sprintf(LocalRankFilename,"ID.%05i",rank); - FILE *IDFILE = fopen(LocalRankFilename,"rb"); - if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: ID.xxxxx"); - readID=fread(id,1,N,IDFILE); - if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading ID (rank=%i) \n",Dm->rank()); - fclose(IDFILE); - // Read restart file if (Restart == true){ if (Dm->rank()==0){ + size_t readID; printf("Reading restart file! \n"); ifstream restart("Restart.txt"); if (restart.is_open()){ @@ -124,132 +117,170 @@ void ScaLBL_ColorModel::ReadInput(){ MPI_Barrier(comm); } } +void Domain::AssignComponentLabels(double *phase) +{ + int NLABELS=0; + char VALUE=0; + double AFFINITY=0.f; + + vector Label; + vector Affinity; + // Read the labels + if (rank()==0){ + printf("Component labels:\n"); + ifstream iFILE("ComponentLabels.csv"); + if (iFILE.good()){ + int value; + while (!iFILE.eof()){ + iFILE>>value; + iFILE>>AFFINITY; + VALUE=char(value); + Label.push_back(value); + Affinity.push_back(AFFINITY); + NLABELS++; + printf("%i %f\n",VALUE,AFFINITY); + } + } + else{ + printf("Using default labels: Solid (0 --> -1.0), NWP (1 --> 1.0), WP (2 --> -1.0)\n"); + // Set default values + VALUE=0; AFFINITY=-1.0; + Label.push_back(VALUE); + Affinity.push_back(AFFINITY); + NLABELS++; + VALUE=1; AFFINITY=1.0; + Label.push_back(VALUE); + Affinity.push_back(AFFINITY); + NLABELS++; + VALUE=2; AFFINITY=-1.0; + Label.push_back(VALUE); + Affinity.push_back(AFFINITY); + NLABELS++; + } + } + MPI_Barrier(Comm); + + // Broadcast the list + MPI_Bcast(&NLABELS,1,MPI_INT,0,Comm); + //printf("rank=%i, NLABELS=%i \n ",rank(),NLABELS); + + // Copy into contiguous buffers + char *LabelList; + double * AffinityList; + LabelList=new char[NLABELS]; + AffinityList=new double[NLABELS]; + + if (rank()==0){ + for (int idx=0; idx < NLABELS; idx++){ + VALUE=Label[idx]; + AFFINITY=Affinity[idx]; + printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank(),idx,VALUE,AFFINITY); + LabelList[idx]=VALUE; + AffinityList[idx]=AFFINITY; + } + } + MPI_Barrier(Comm); + + MPI_Bcast(LabelList,NLABELS,MPI_CHAR,0,Comm); + MPI_Bcast(AffinityList,NLABELS,MPI_DOUBLE,0,Comm); + + // Assign the labels + for (int k=0;krank(); - double sum; - double porosity; - double sum_local=0.0; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); - Np=0; // number of local pore nodes - //....................................................................... - for (int k=1;k 0){ - sum_local+=1.0; - Np++; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = sum*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - //......................................................... - // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm->kproc() == 0){ - for (int k=0; k<3; k++){ - for (int j=0;jSDs(n) = max(Averages->SDs(n),1.0*(2.5-k)); - } - } - } - } - if (BoundaryCondition > 0 && Dm->kproc() == nprocz-1){ - for (int k=Nz-3; kSDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5)); - } - } - } - } - //......................................................... - // don't perform computations at the eight corners - id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - //......................................................... + int nprocs=nprocx*nprocy*nprocz; + int rank=Dm->rank(); + //......................................................... + // don't perform computations at the eight corners + id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; + Mask->CommInit(comm); - // Initialize communication structures in averaging domain - for (int i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; - Mask->CommInit(comm); - double *PhaseLabel; - PhaseLabel = new double[N]; - Mask->AssignComponentLabels(PhaseLabel); - - //........................................................................... - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - // Create a communicator for the device (will use optimized layout) - // ScaLBL_Communicator ScaLBL_Comm(Mask); // original - ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); - //Create a second communicator based on the regular data layout - //ScaLBL_Communicator ScaLBL_Comm_Regular(Mask); - - int Npad=(Np/16 + 2)*16; - if (rank==0) printf ("Set up memory efficient layout \n"); - Map.resize(Nx,Ny,Nz); Map.fill(-2); - auto neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); - MPI_Barrier(comm); + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout \n"); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); - //........................................................................... - // MAIN VARIABLES ALLOCATED HERE - //........................................................................... - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - //......................device distributions................................. - int dist_mem_size = Np*sizeof(double); - int neighborSize=18*(Np*sizeof(int)); + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &SolidPotential, 3*sizeof(double)*Np); - - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Gradient, 3*sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &SolidPotential, 3*sizeof(double)*Np); + + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); // copy the neighbor list ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - int *TmpMap; - TmpMap=new int[Np]; - for (int k=1; k db0); + void AssignComponentLabels(double *phase); void ReadInput(); void Create(); void Initialize(); From cccd902f1ce3f382bab715a0d24be54fff680aa2 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 06:39:44 -0400 Subject: [PATCH 097/124] mad tea party --- common/Domain.cpp | 27 +++++++++++++-------------- common/Domain.h | 4 ++-- models/ColorModel.cpp | 9 +++++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 0db7f88e..666bc992 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -61,7 +61,6 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, MPI_Comm_rank( Comm, &myrank ); rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); - initialize( db ); MPI_Barrier(Comm); auto db = std::make_shared( ); @@ -72,7 +71,7 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, db->putVector( "L", { lx, ly, lz } ); initialize( db ); } -Domain::Domain( std::shared_ptr db, MPI_Communicator Communicator): +Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): Nx(0), Ny(0), Nz(0), Lx(0), Ly(0), Lz(0), Volume(0), BoundaryCondition(0), Comm(MPI_COMM_NULL), @@ -108,9 +107,8 @@ Domain::Domain( std::shared_ptr db, MPI_Communicator Communicator): int myrank; MPI_Comm_rank( Comm, &myrank ); rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); - - initialize( db ); MPI_Barrier(Comm); + initialize( db ); } void Domain::initialize( std::shared_ptr db ) @@ -503,31 +501,32 @@ void Domain::CommInit() void Domain::ReadIDs(){ // Read the IDs from input file + int nprocs=nprocx()*nprocy()*nprocz(); size_t readID; char LocalRankString[8]; char LocalRankFilename[40]; //....................................................................... - if (rank == 0) printf("Read input media... \n"); + if (rank() == 0) printf("Read input media... \n"); //....................................................................... - sprintf(LocalRankString,"%05d",Dm->rank()); + sprintf(LocalRankString,"%05d",rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); // .......... READ THE INPUT FILE ....................................... - if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); - sprintf(LocalRankFilename,"ID.%05i",rank); + if (rank()==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); + sprintf(LocalRankFilename,"ID.%05i",rank()); FILE *IDFILE = fopen(LocalRankFilename,"rb"); if (IDFILE==NULL) ERROR("Domain::ReadIDs -- Error opening file: ID.xxxxx"); readID=fread(id,1,N,IDFILE); - if (readID != size_t(N)) printf("Domain::ReadIDs -- Error reading ID (rank=%i) \n",Dm->rank()); + if (readID != size_t(N)) printf("Domain::ReadIDs -- Error reading ID (rank=%i) \n",rank()); fclose(IDFILE); // Compute the porosity double sum; double porosity; double sum_local=0.0; double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); + if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6)); //......................................................... // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm->kproc() == 0){ + if (BoundaryCondition > 0 && kproc() == 0){ for (int k=0; k<3; k++){ for (int j=0;j 0 && Dm->kproc() == nprocz-1){ + if (BoundaryCondition > 0 && kproc() == nprocz()-1){ for (int k=Nz-3; k db, MPI_Communicator Communicator); + Domain( std::shared_ptr db, MPI_Comm Communicator); //! Obsolete constructor Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, @@ -173,7 +173,7 @@ public: // Public variables (need to create accessors instead) void ReadIDs(); void CommunicateMeshHalo(DoubleArray &Mesh); void CommInit(); - int PoreCount() + int PoreCount(); private: //...................................................................................... diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 8f8a198b..a735328a 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -58,8 +58,8 @@ void ScaLBL_ColorModel::ReadParams(string filename){ if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) - Dm = std::shared_ptr(new Domain(domain_db)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db)); // mask domain removes immobile phases + Dm = std::shared_ptr(new Domain(domain_db),comm); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db),comm); // mask domain removes immobile phases Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; @@ -69,7 +69,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){ id = new char[N]; MPI_Barrier(comm); - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); } @@ -82,6 +82,7 @@ void ScaLBL_ColorModel::ReadInput(){ sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + // .......... READ THE INPUT FILE ....................................... //........................................................................... if (rank == 0) cout << "Reading in signed distance function..." << endl; @@ -552,4 +553,4 @@ void ScaLBL_ColorModel::WriteDebug(){ OUTFILE = fopen(LocalRankFilename,"wb"); fwrite(PhaseField.data(),8,N,OUTFILE); fclose(OUTFILE); -} \ No newline at end of file +} From c8b84faa9589eda5128055a1f487d494eacce960 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 07:01:09 -0400 Subject: [PATCH 098/124] i don't see any wine --- models/ColorModel.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index a735328a..455271dc 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -65,7 +65,8 @@ void ScaLBL_ColorModel::ReadParams(string filename){ N = Nx*Ny*Nz; for (int i=0; iid[i] = 1; // initialize this way Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object - + + // local copy of the ids id = new char[N]; MPI_Barrier(comm); @@ -78,10 +79,12 @@ void ScaLBL_ColorModel::ReadInput(){ //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... + Dm->ReadIds(); + + sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - // .......... READ THE INPUT FILE ....................................... //........................................................................... @@ -118,7 +121,7 @@ void ScaLBL_ColorModel::ReadInput(){ MPI_Barrier(comm); } } -void Domain::AssignComponentLabels(double *phase) +void ScaLBL_ColorModel::AssignComponentLabels(double *phase) { int NLABELS=0; char VALUE=0; @@ -216,11 +219,12 @@ void ScaLBL_ColorModel::Create(){ // don't perform computations at the eight corners id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + //......................................................... // Initialize communication structures in averaging domain for (int i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; - Mask->CommInit(comm); - + Mask->CommInit(); + Np->Mask.PoreCount(); //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) From bb4a03da47f11038ad44d79ec98c8abb0b244cbc Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 07:14:37 -0400 Subject: [PATCH 099/124] there isn't any --- models/ColorModel.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 455271dc..75c742b8 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -58,8 +58,8 @@ void ScaLBL_ColorModel::ReadParams(string filename){ if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) - Dm = std::shared_ptr(new Domain(domain_db),comm); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db),comm); // mask domain removes immobile phases + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; @@ -76,11 +76,11 @@ void ScaLBL_ColorModel::ReadParams(string filename){ void ScaLBL_ColorModel::ReadInput(){ int rank=Dm->rank(); + size_t readID; //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... - Dm->ReadIds(); - + Dm->ReadIDs(); sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); @@ -113,7 +113,7 @@ void ScaLBL_ColorModel::ReadInput(){ } MPI_Bcast(×tep,1,MPI_INT,0,comm); FILE *RESTART = fopen(LocalRestartFile,"rb"); - if (IDFILE==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); + if (RESTART==NULL) ERROR("lbpm_color_simulator: Error opening file: Restart.xxxxx"); readID=fread(id,1,N,RESTART); if (readID != size_t(N)) printf("lbpm_color_simulator: Error reading Restart (rank=%i) \n",rank); fclose(RESTART); @@ -123,6 +123,7 @@ void ScaLBL_ColorModel::ReadInput(){ } void ScaLBL_ColorModel::AssignComponentLabels(double *phase) { + int rank=Dm->rank(); int NLABELS=0; char VALUE=0; double AFFINITY=0.f; @@ -130,7 +131,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) vector Label; vector Affinity; // Read the labels - if (rank()==0){ + if (rank==0){ printf("Component labels:\n"); ifstream iFILE("ComponentLabels.csv"); if (iFILE.good()){ @@ -162,10 +163,10 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) NLABELS++; } } - MPI_Barrier(Comm); + MPI_Barrier(comm); // Broadcast the list - MPI_Bcast(&NLABELS,1,MPI_INT,0,Comm); + MPI_Bcast(&NLABELS,1,MPI_INT,0,comm); //printf("rank=%i, NLABELS=%i \n ",rank(),NLABELS); // Copy into contiguous buffers @@ -174,19 +175,19 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) LabelList=new char[NLABELS]; AffinityList=new double[NLABELS]; - if (rank()==0){ + if (rank==0){ for (int idx=0; idx < NLABELS; idx++){ VALUE=Label[idx]; AFFINITY=Affinity[idx]; - printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank(),idx,VALUE,AFFINITY); + printf("rank=%i, idx=%i, value=%d, affinity=%f \n",rank,idx,VALUE,AFFINITY); LabelList[idx]=VALUE; AffinityList[idx]=AFFINITY; } } - MPI_Barrier(Comm); + MPI_Barrier(comm); - MPI_Bcast(LabelList,NLABELS,MPI_CHAR,0,Comm); - MPI_Bcast(AffinityList,NLABELS,MPI_DOUBLE,0,Comm); + MPI_Bcast(LabelList,NLABELS,MPI_CHAR,0,comm); + MPI_Bcast(AffinityList,NLABELS,MPI_DOUBLE,0,comm); // Assign the labels for (int k=0;krank(); //......................................................... // don't perform computations at the eight corners @@ -224,7 +224,7 @@ void ScaLBL_ColorModel::Create(){ // Initialize communication structures in averaging domain for (int i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; Mask->CommInit(); - Np->Mask.PoreCount(); + Np=Mask->PoreCount(); //........................................................................... if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device (will use optimized layout) @@ -295,7 +295,7 @@ void ScaLBL_ColorModel::Initialize(){ if (rank==0) printf("Computing solid interaction potential \n"); double *PhaseLabel; PhaseLabel=new double [Nx*Ny*Nz]; - Mask->AssignComponentLabels(PhaseLabel); + AssignComponentLabels(PhaseLabel); double *Tmp; Tmp=new double[3*Np]; //Averages->UpdateMeshValues(); // this computes the gradient of distance field (among other things) From 9ade92ba36d2529dab32b441a086f6c9164d90c7 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 07:49:32 -0400 Subject: [PATCH 100/124] the queen's croquet field --- tests/BlobAnalyzeParallel.cpp | 2 +- tests/ComponentLabel.cpp | 2 +- tests/GenerateSphereTest.cpp | 5 +-- tests/TestBlobAnalyze.cpp | 2 +- tests/TestBubble.cpp | 2 +- tests/TestBubbleDFH.cpp | 22 ++++-------- tests/TestColorGrad.cpp | 2 +- tests/TestColorGradDFH.cpp | 4 +-- tests/TestColorMassBounceback.cpp | 6 ++-- tests/TestColorSquareTube.cpp | 8 ++--- tests/TestCommD3Q19.cpp | 2 +- tests/TestFluxBC.cpp | 2 +- tests/TestForceMoments.cpp | 48 +++------------------------ tests/TestInterfaceSpeed.cpp | 7 ++-- tests/TestMRT.cpp | 2 +- tests/TestMap.cpp | 2 +- tests/TestPoiseuille.cpp | 19 ++--------- tests/TestPressVel.cpp | 16 ++------- tests/TestSegDist.cpp | 4 +-- tests/TestTorus.cpp | 4 +-- tests/TestTwoPhase.cpp | 2 +- tests/lbpm_BGK_simulator.cpp | 2 +- tests/lbpm_captube_pp.cpp | 2 +- tests/lbpm_color_macro_simulator.cpp | 2 +- tests/lbpm_color_simulator.cpp | 2 +- tests/lbpm_dfh_simulator.cpp | 2 +- tests/lbpm_inkbottle_pp.cpp | 2 +- tests/lbpm_morphdrain_pp.cpp | 4 +-- tests/lbpm_morphopen_pp.cpp | 4 +-- tests/lbpm_nondarcy_simulator.cpp | 2 +- tests/lbpm_nonnewtonian_simulator.cpp | 4 +-- tests/lbpm_permeability_simulator.cpp | 2 +- tests/lbpm_plates_pp.cpp | 2 +- tests/lbpm_porenetwork_pp.cpp | 4 +-- tests/lbpm_random_pp.cpp | 4 +-- tests/lbpm_refine_pp.cpp | 2 +- tests/lbpm_segmented_decomp.cpp | 2 +- tests/lbpm_segmented_pp.cpp | 2 +- tests/lbpm_squaretube_pp.cpp | 2 +- 39 files changed, 66 insertions(+), 143 deletions(-) diff --git a/tests/BlobAnalyzeParallel.cpp b/tests/BlobAnalyzeParallel.cpp index 3fa10352..c9e3f8fc 100644 --- a/tests/BlobAnalyzeParallel.cpp +++ b/tests/BlobAnalyzeParallel.cpp @@ -290,7 +290,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); // Initialize communications for domains + Dm.CommInit(); // Initialize communications for domains MPI_Allreduce(&sum,&sum_global,1,MPI_DOUBLE,MPI_SUM,comm); porosity = sum_global/Dm.Volume; diff --git a/tests/ComponentLabel.cpp b/tests/ComponentLabel.cpp index 0a18ba0b..07ef6555 100644 --- a/tests/ComponentLabel.cpp +++ b/tests/ComponentLabel.cpp @@ -331,7 +331,7 @@ int main(int argc, char **argv) } porosity /= (Nx*Ny*Nz*1.0); printf("Media porosity is %f \n",porosity); - Dm.CommInit(comm); + Dm.CommInit(); /* **************************************************************** IDENTIFY ALL COMPONENTS FOR BOTH PHASES diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 3293205c..48b4840f 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -420,9 +420,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); - - int dist_mem_size = N*sizeof(double); + Dm.CommInit(); if (rank==0) printf("Number of nodes per side = %i \n", Nx); if (rank==0) printf("Total Number of nodes = %i \n", N); @@ -437,7 +435,6 @@ int main(int argc, char **argv) char LocalRankString[8]; char LocalRankFilename[40]; char LocalRestartFile[40]; - char tmpstr[10]; sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); diff --git a/tests/TestBlobAnalyze.cpp b/tests/TestBlobAnalyze.cpp index 3adbcafe..d2df9f86 100644 --- a/tests/TestBlobAnalyze.cpp +++ b/tests/TestBlobAnalyze.cpp @@ -234,7 +234,7 @@ int main(int argc, char **argv) } } //....................................................................... - Dm->CommInit(comm); // Initialize communications for domains + Dm->CommInit(); // Initialize communications for domains //....................................................................... // Read in sphere pack (initialize the non-wetting phase as inside of spheres) // diff --git a/tests/TestBubble.cpp b/tests/TestBubble.cpp index 84aaddcd..c03e5dea 100644 --- a/tests/TestBubble.cpp +++ b/tests/TestBubble.cpp @@ -157,7 +157,7 @@ int main(int argc, char **argv) // Full domain used for averaging (do not use mask for analysis) Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC); - Dm.CommInit(comm); + Dm.CommInit(); // Mask that excludes the solid phase Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,pBC); diff --git a/tests/TestBubbleDFH.cpp b/tests/TestBubbleDFH.cpp index 39bd6b68..10a990c5 100644 --- a/tests/TestBubbleDFH.cpp +++ b/tests/TestBubbleDFH.cpp @@ -112,9 +112,6 @@ int main(int argc, char **argv) int Nx = size[0]; int Ny = size[1]; int Nz = size[2]; - double Lx = L[0]; - double Ly = L[1]; - double Lz = L[2]; int nprocx = nproc[0]; int nprocy = nproc[1]; int nprocz = nproc[2]; @@ -167,14 +164,14 @@ int main(int argc, char **argv) pBC=false; // Full domain used for averaging (do not use mask for analysis) - std::shared_ptr Dm(new Domain(domain_db)); + std::shared_ptr Dm(new Domain(domain_db,comm)); for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; std::shared_ptr Averages( new TwoPhase(Dm) ); // TwoPhase Averages(Dm); - Dm->CommInit(comm); + Dm->CommInit(); // Mask that excludes the solid phase - std::shared_ptr Mask(new Domain(domain_db)); + std::shared_ptr Mask(new Domain(domain_db,comm)); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; @@ -187,7 +184,6 @@ int main(int argc, char **argv) char LocalRankString[8]; char LocalRankFilename[40]; char LocalRestartFile[40]; - char tmpstr[10]; sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); @@ -198,9 +194,6 @@ int main(int argc, char **argv) char *id; id = new char[N]; double sum, sum_local; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); - double porosity, pore_vol; //........................................................................... if (rank == 0) cout << "Setting up bubble..." << endl; double BubbleRadius = 15.5; // Radius of the capillary tube @@ -250,7 +243,7 @@ int main(int argc, char **argv) // Initialize communication structures in averaging domain for (i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; - Mask->CommInit(comm); + Mask->CommInit(); double *PhaseLabel; PhaseLabel = new double[N]; @@ -407,7 +400,6 @@ int main(int argc, char **argv) //......................................... err = 1.0; - double sat_w_previous = 1.01; // slightly impossible value! if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol); //************ MAIN ITERATION LOOP ***************************************/ @@ -544,7 +536,7 @@ int main(int argc, char **argv) } } FILE *GFILE; - sprintf(LocalRankFilename,"Gradient.raw",rank); + sprintf(LocalRankFilename,"Gradient.raw"); GFILE = fopen(LocalRankFilename,"wb"); fwrite(GradNorm.data(),8,N,GFILE); fclose(GFILE); @@ -554,12 +546,12 @@ int main(int argc, char **argv) ScaLBL_Comm->RegularLayout(Map,&Den[0],Rho1); ScaLBL_Comm->RegularLayout(Map,&Den[Np],Rho2); FILE *RFILE1; - sprintf(LocalRankFilename,"Rho1.raw",rank); + sprintf(LocalRankFilename,"Rho1.raw"); RFILE1 = fopen(LocalRankFilename,"wb"); fwrite(Rho1.data(),8,N,RFILE1); fclose(RFILE1); FILE *RFILE2; - sprintf(LocalRankFilename,"Rho2.raw",rank); + sprintf(LocalRankFilename,"Rho2.raw"); RFILE2 = fopen(LocalRankFilename,"wb"); fwrite(Rho2.data(),8,N,RFILE2); fclose(RFILE2); diff --git a/tests/TestColorGrad.cpp b/tests/TestColorGrad.cpp index 9a00f897..5cd6d924 100644 --- a/tests/TestColorGrad.cpp +++ b/tests/TestColorGrad.cpp @@ -174,7 +174,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); + Dm.CommInit(); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); diff --git a/tests/TestColorGradDFH.cpp b/tests/TestColorGradDFH.cpp index 17837fb9..a4c8ce2c 100644 --- a/tests/TestColorGradDFH.cpp +++ b/tests/TestColorGradDFH.cpp @@ -88,7 +88,7 @@ int main(int argc, char **argv) } // Get the rank info - std::shared_ptr Dm(new Domain(db)); + std::shared_ptr Dm(new Domain(db,comm)); Nx += 2; Ny += 2; Nz += 2; @@ -108,7 +108,7 @@ int main(int argc, char **argv) } } } - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index e8df5ae8..dc6a9525 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -118,9 +118,9 @@ int main(int argc, char **argv) pBC=false; // Full domain used for averaging (do not use mask for analysis) - std::shared_ptr Dm(new Domain(domain_db)); + std::shared_ptr Dm(new Domain(domain_db,comm)); for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); Nx+=2; Ny+=2; Nz += 2; @@ -152,7 +152,7 @@ int main(int argc, char **argv) } } } - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; if (rank==0) printf ("Create ScaLBL_Communicator \n"); diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index dc63f9d6..3a763b22 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -115,8 +115,8 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - std::shared_ptr Dm (new Domain(db)); - Dm->CommInit(comm); + std::shared_ptr Dm (new Domain(db,comm)); + Dm->CommInit(); Nx += 2; Ny += 2; @@ -164,7 +164,7 @@ int main(int argc, char **argv) } } } - Dm->CommInit(comm); + Dm->CommInit(); //....................................................................... // Compute the media porosity, assign phase labels and solid composition @@ -174,7 +174,7 @@ int main(int argc, char **argv) int Np=0; // number of local pore nodes double *PhaseLabel; PhaseLabel = new double[N]; - Dm->AssignComponentLabels(PhaseLabel); + //Dm->AssignComponentLabels(PhaseLabel); //....................................................................... for (k=1;kNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = id[i]; - Dm->CommInit(comm); + Dm->CommInit(); Np=0; // number of local pore nodes //....................................................................... diff --git a/tests/TestForceMoments.cpp b/tests/TestForceMoments.cpp index 94f4c101..1fb1e0a4 100644 --- a/tests/TestForceMoments.cpp +++ b/tests/TestForceMoments.cpp @@ -67,9 +67,7 @@ int main(int argc, char **argv) } // BGK Model parameters - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double tau,Fx,Fy,Fz,tol; + double tau,Fx,Fy,Fz; // Domain variables int i,j,k,n; int dim = 3; if (rank == 0) printf("dim=%d\n",dim); @@ -77,7 +75,6 @@ int main(int argc, char **argv) int timesteps = 2; tau =1.0; - double mu=(tau-0.5)/3.0; double rlx_setA=1.0/tau; double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); Fx = Fy = 1.0; @@ -115,9 +112,7 @@ int main(int argc, char **argv) printf("\n\n"); } - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - - std::shared_ptr Dm(new Domain(domain_db)); + std::shared_ptr Dm(new Domain(domain_db,comm)); Nx += 2; Ny += 2; @@ -147,7 +142,7 @@ int main(int argc, char **argv) } } } - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; @@ -264,43 +259,8 @@ int main(int argc, char **argv) MLUPS *= nprocs; if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS); if (rank==0) printf("********************************************************\n"); - - // Number of memory references from the swap algorithm (per timestep) - // 18 reads and 18 writes for each lattice site - double MemoryRefs = Np*38; - - int SIZE=Np*sizeof(double); - /* - double *Vz; - Vz= new double [Np]; - double *Vx; - Vx= new double [Np]; - double *Vy; - Vy= new double [Np]; - ScaLBL_D3Q19_AA_Velocity(dist, &dist[10*Np],Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&Vx[0],&Velocity[0],SIZE); - ScaLBL_CopyToHost(&Vy[0],&Velocity[Np],SIZE); - ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE); - - printf("Force: %f,%f,%f \n",Fx,Fy,Fz); - double vz; - double W = 1.f*Nx-4; - j=Ny/2; k=Nz/2; - for (j=1;jid[n]); - n = Map(i,j,k); - //printf("%i,%i,%i; %i :",i,j,k,n); - if (n<0) vz =0.f; - else vz=Vz[n]; - printf("%f ",vz); - } - printf("\n"); - } - */ + int SIZE=Np*sizeof(double); double *DIST; DIST= new double [19*Np]; ScaLBL_CopyToHost(&DIST[0],&dist[0],19*SIZE); diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index 84942b85..aa7ca139 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -35,15 +35,12 @@ int main (int argc, char *argv[]) int Nx = domain_db->getVector( "n" )[0]; int Ny = domain_db->getVector( "n" )[1]; int Nz = domain_db->getVector( "n" )[2]; - int nprocx = domain_db->getVector( "nproc" )[0]; - int nprocy = domain_db->getVector( "nproc" )[1]; - int nprocz = domain_db->getVector( "nproc" )[2]; - std::shared_ptr Dm(new Domain(domain_db)); + std::shared_ptr Dm(new Domain(domain_db,comm)); for (i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages(new TwoPhase(Dm)); int timestep=0; diff --git a/tests/TestMRT.cpp b/tests/TestMRT.cpp index 682735f4..30f46689 100644 --- a/tests/TestMRT.cpp +++ b/tests/TestMRT.cpp @@ -651,7 +651,7 @@ int main(int argc, char **argv) fclose(IDFILE); MPI_Barrier(comm); - Dm.CommInit(comm); + Dm.CommInit(); //....................................................................... // Compute the media porosity diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index 17541753..f7a013da 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -146,7 +146,7 @@ int main(int argc, char **argv) } } } - Dm->CommInit(comm); + Dm->CommInit(); // Create a communicator for the device (will use optimized layout) std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 580486c5..73909c93 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -30,8 +30,7 @@ int main(int argc, char **argv) } // BGK Model parameters - int timestepMax, interval; - double tau,Fx,Fy,Fz,tol; + double tau,Fx,Fy,Fz; // Domain variables int i,j,k,n; int timestep = 0; @@ -75,7 +74,7 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - std::shared_ptr Dm( new Domain(domain_db)); + std::shared_ptr Dm( new Domain(domain_db,comm)); Nx += 2; Ny += 2; @@ -108,7 +107,7 @@ int main(int argc, char **argv) } } - Dm->CommInit(comm); + Dm->CommInit(); MPI_Barrier(comm); //....................................................................... @@ -226,18 +225,6 @@ int main(int argc, char **argv) // if (rank==0) printf("CPU time = %f \n", cputime); // if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS); MLUPS *= nprocs; - // if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS); - // if (rank==0) printf("********************************************************\n"); - - // Number of memory references from the swap algorithm (per timestep) - // 18 reads and 18 writes for each lattice site - double MemoryRefs = Np*38; - // number of memory references for the swap algorithm - GigaBytes / second - // if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9/cputime); - // Report bandwidth in Gigabits per second - // communication bandwidth includes both send and recieve - //if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm->CommunicationCount*64*timestep/1e9/cputime); - // if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm->CommunicationCount*64*timestep/1e9/cputime); double *Vz; Vz= new double [Np]; diff --git a/tests/TestPressVel.cpp b/tests/TestPressVel.cpp index 1bbdd3c6..e655ced9 100644 --- a/tests/TestPressVel.cpp +++ b/tests/TestPressVel.cpp @@ -30,18 +30,8 @@ int main(int argc, char **argv) printf("********************************************************\n"); } - // BGK Model parameters - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double tau,Fx,Fy,Fz,tol; // Domain variables int i,j,k,n; - double rlx_setA=1.0; - double rlx_setB=1.0; - - Fx = Fy = 0.f; - Fz = 1.0e-4; - // Load inputs string FILENAME = argv[1]; // Load inputs @@ -76,8 +66,8 @@ int main(int argc, char **argv) double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - std::shared_ptr Dm (new Domain(domain_db)); - Dm->CommInit(comm); + std::shared_ptr Dm (new Domain(domain_db,comm)); + Dm->CommInit(); Nx += 2; Ny += 2; @@ -92,7 +82,7 @@ int main(int argc, char **argv) char LocalRankFilename[40]; sprintf(LocalRankFilename,"ID.%05i",rank); - Dm->CommInit(comm); + Dm->CommInit(); //....................................................................... // Compute the media porosity diff --git a/tests/TestSegDist.cpp b/tests/TestSegDist.cpp index 5866101d..f431042a 100644 --- a/tests/TestSegDist.cpp +++ b/tests/TestSegDist.cpp @@ -55,7 +55,7 @@ int main(int argc, char **argv) // Get the rank info - Domain Dm(db); + Domain Dm(db,comm); for (int k=0;k Dm(new Domain(db)); + std::shared_ptr Dm(new Domain(db,comm)); // const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); std::shared_ptr Averages(new TwoPhase(Dm)); Nx += 2; @@ -77,7 +77,7 @@ int main(int argc, char **argv) } } //....................................................................... - Dm->CommInit(comm); // Initialize communications for domains + Dm->CommInit(); // Initialize communications for domains //....................................................................... //....................................................................... diff --git a/tests/TestTwoPhase.cpp b/tests/TestTwoPhase.cpp index 3ee6d7ef..a979314a 100644 --- a/tests/TestTwoPhase.cpp +++ b/tests/TestTwoPhase.cpp @@ -49,7 +49,7 @@ int main(int argc, char **argv) for (i=0; i Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index d99842ec..1c619c5a 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -258,7 +258,7 @@ int main(int argc, char **argv) for (i=0; i Averages( new TwoPhase(Dm) ); // TwoPhase Averages(Dm); - Dm.CommInit(comm); + Dm.CommInit(); // Mask that excludes the solid phase Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 6268b74f..896391ef 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -326,7 +326,7 @@ int main(int argc, char **argv) Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); for (i=0; i Averages( new TwoPhase(Dm) ); - Dm.CommInit(comm); + Dm.CommInit(); // Mask that excludes the solid phase Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); diff --git a/tests/lbpm_dfh_simulator.cpp b/tests/lbpm_dfh_simulator.cpp index 3b7ff14c..5e1c08b2 100644 --- a/tests/lbpm_dfh_simulator.cpp +++ b/tests/lbpm_dfh_simulator.cpp @@ -148,7 +148,7 @@ int main(int argc, char **argv) std::shared_ptr Dm (new Domain(domain_db)); for (int i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); // Mask that excludes the solid phase diff --git a/tests/lbpm_inkbottle_pp.cpp b/tests/lbpm_inkbottle_pp.cpp index 012dac55..3c39219d 100644 --- a/tests/lbpm_inkbottle_pp.cpp +++ b/tests/lbpm_inkbottle_pp.cpp @@ -115,7 +115,7 @@ int main(int argc, char **argv) // Initialized domain and averaging framework for Two-Phase Flow std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, diff --git a/tests/lbpm_morphdrain_pp.cpp b/tests/lbpm_morphdrain_pp.cpp index 42250dcf..9b5cc7fa 100644 --- a/tests/lbpm_morphdrain_pp.cpp +++ b/tests/lbpm_morphdrain_pp.cpp @@ -148,7 +148,7 @@ int main(int argc, char **argv) } } */ - Dm.CommInit(comm); + Dm.CommInit(); int xdim,ydim,zdim; @@ -177,7 +177,7 @@ int main(int argc, char **argv) fclose(IDFILE); - Dm.CommInit(comm); + Dm.CommInit(); int iproc = Dm.iproc(); int jproc = Dm.jproc(); int kproc = Dm.kproc(); diff --git a/tests/lbpm_morphopen_pp.cpp b/tests/lbpm_morphopen_pp.cpp index 05793748..6f054d9f 100644 --- a/tests/lbpm_morphopen_pp.cpp +++ b/tests/lbpm_morphopen_pp.cpp @@ -131,7 +131,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); + Dm.CommInit(); DoubleArray SignDist(nx,ny,nz); // Read the signed distance from file @@ -270,7 +270,7 @@ int main(int argc, char **argv) MPI_Barrier(comm); */ - Dm.CommInit(comm); + Dm.CommInit(); int iproc = Dm.iproc(); int jproc = Dm.jproc(); int kproc = Dm.kproc(); diff --git a/tests/lbpm_nondarcy_simulator.cpp b/tests/lbpm_nondarcy_simulator.cpp index 2e8da060..40672375 100644 --- a/tests/lbpm_nondarcy_simulator.cpp +++ b/tests/lbpm_nondarcy_simulator.cpp @@ -347,7 +347,7 @@ int main(int argc, char **argv) //......................................................... // Initialize communication structures in averaging domain for (i=0; i Averages( new TwoPhase(Dm) ); - Dm.CommInit(comm); /* 2 */ + Dm.CommInit(); /* 2 */ Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); @@ -482,7 +482,7 @@ int main(int argc, char **argv) // Initialize communication structures in averaging domain // for (i=0; i Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); diff --git a/tests/lbpm_porenetwork_pp.cpp b/tests/lbpm_porenetwork_pp.cpp index 616a3a8d..496f9d86 100644 --- a/tests/lbpm_porenetwork_pp.cpp +++ b/tests/lbpm_porenetwork_pp.cpp @@ -101,11 +101,11 @@ int main(int argc, char **argv) // Initialized domain and averaging framework for Two-Phase Flow // Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC); - //Dm.CommInit(comm); + //Dm.CommInit(); //TwoPhase Averages(Dm); std::shared_ptr Dm (new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); MPI_Barrier(comm); diff --git a/tests/lbpm_random_pp.cpp b/tests/lbpm_random_pp.cpp index b6b75e0a..07c56e6f 100644 --- a/tests/lbpm_random_pp.cpp +++ b/tests/lbpm_random_pp.cpp @@ -141,7 +141,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); + Dm.CommInit(); DoubleArray SignDist(nx,ny,nz); // Read the signed distance from file @@ -171,7 +171,7 @@ int main(int argc, char **argv) float porosity=float(totalGlobal)/(nprocx*nprocy*nprocz*(nx-2)*(ny-2)*(nz-2)); if (rank==0) printf("Media Porosity: %f \n",porosity); - Dm.CommInit(comm); + Dm.CommInit(); int iproc = Dm.iproc(); int jproc = Dm.jproc(); int kproc = Dm.kproc(); diff --git a/tests/lbpm_refine_pp.cpp b/tests/lbpm_refine_pp.cpp index 01c12464..08ca59ab 100644 --- a/tests/lbpm_refine_pp.cpp +++ b/tests/lbpm_refine_pp.cpp @@ -102,7 +102,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); + Dm.CommInit(); DoubleArray SignDist(nx,ny,nz); // Read the signed distance from file diff --git a/tests/lbpm_segmented_decomp.cpp b/tests/lbpm_segmented_decomp.cpp index 75808ee5..3384e454 100644 --- a/tests/lbpm_segmented_decomp.cpp +++ b/tests/lbpm_segmented_decomp.cpp @@ -142,7 +142,7 @@ int main(int argc, char **argv) } } } - Dm.CommInit(comm); + Dm.CommInit(); // number of sites to use for periodic boundary condition transition zone int z_transition_size = (nprocz*nz - (Nz - zStart))/2; diff --git a/tests/lbpm_segmented_pp.cpp b/tests/lbpm_segmented_pp.cpp index d7cbd15f..aea481b4 100644 --- a/tests/lbpm_segmented_pp.cpp +++ b/tests/lbpm_segmented_pp.cpp @@ -189,7 +189,7 @@ int main(int argc, char **argv) std::shared_ptr Dm (new Domain(nx,ny,nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); for (n=0; nid[n]=1; - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages( new TwoPhase(Dm) ); nx+=2; ny+=2; nz+=2; diff --git a/tests/lbpm_squaretube_pp.cpp b/tests/lbpm_squaretube_pp.cpp index 691ada62..42715773 100644 --- a/tests/lbpm_squaretube_pp.cpp +++ b/tests/lbpm_squaretube_pp.cpp @@ -117,7 +117,7 @@ int main(int argc, char **argv) // Initialized domain and averaging framework for Two-Phase Flow std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - Dm->CommInit(comm); + Dm->CommInit(); std::shared_ptr Averages(new TwoPhase(Dm)); InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, From 99f7e2edcd1cf25e93d9821b5c74ce053bc5dd3f Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 13:08:05 -0400 Subject: [PATCH 101/124] fixed bug in Domain.cpp --- common/Domain.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/Domain.cpp b/common/Domain.cpp index 666bc992..ee338d92 100644 --- a/common/Domain.cpp +++ b/common/Domain.cpp @@ -106,10 +106,9 @@ Domain::Domain( std::shared_ptr db, MPI_Comm Communicator): // set up the neighbor ranks int myrank; MPI_Comm_rank( Comm, &myrank ); + initialize( db ); rank_info = RankInfoStruct( myrank, rank_info.nx, rank_info.ny, rank_info.nz ); MPI_Barrier(Comm); - initialize( db ); - } void Domain::initialize( std::shared_ptr db ) { From 7344276c5cce09dde491f3ade4580cdd866d6f7e Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 13:14:58 -0400 Subject: [PATCH 102/124] map --- tests/TestMap.cpp | 117 ++++++++++------------------------------------ 1 file changed, 24 insertions(+), 93 deletions(-) diff --git a/tests/TestMap.cpp b/tests/TestMap.cpp index f7a013da..3e56cdf9 100644 --- a/tests/TestMap.cpp +++ b/tests/TestMap.cpp @@ -11,6 +11,18 @@ using namespace std; +std::shared_ptr loadInputs( int nprocs ) +{ + //auto db = std::make_shared( "Domain.in" ); + auto db = std::make_shared(); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 5, 5, 5 } ); + db->putScalar( "nspheres", 1 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} + //*************************************************************************************** int main(int argc, char **argv) { @@ -25,116 +37,35 @@ int main(int argc, char **argv) MPI_Comm_size(comm,&nprocs); int check=0; { - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - if (rank == 0){ - printf("********************************************************\n"); - printf("Running Color Model: TestMap \n"); - printf("********************************************************\n"); - } - // BGK Model parameters - string FILENAME; - // Domain variables - double Lx,Ly,Lz; - int nspheres; - int Nx,Ny,Nz; int i,j,k,n; - int dim=5; static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}, {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0}, {1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1}, {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}}; - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (domain.good()){ - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - } - else if (nprocs==1){ - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=5; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==2){ - nprocx=2; nprocy=1; - nprocz=1; - Nx=Ny=Nz=dim; - Nx = dim; Ny = dim; Nz = dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==4){ - nprocx=nprocy=2; - nprocz=1; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - else if (nprocs==8){ - nprocx=nprocy=nprocz=2; - Nx=Ny=Nz=dim; - nspheres=0; - Lx=Ly=Lz=1; - } - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - // ************************************************************** - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - if (rank==0){ + if (rank == 0){ printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); + printf("Running unit test: TestMap \n"); printf("********************************************************\n"); } + + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; - MPI_Barrier(comm); - int BoundaryCondition=0; - - std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition)); + std::shared_ptr Dm(new Domain(db,comm)); Nx += 2; Ny += 2; Nz += 2; int N = Nx*Ny*Nz; - //....................................................................... int Np = 0; for (k=1;k Date: Sat, 19 May 2018 13:21:33 -0400 Subject: [PATCH 103/124] I dare say you never even spoke to time --- tests/CMakeLists.txt | 2 +- tests/TestFluxBC.cpp | 42 ++++++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77212b80..e3b02bf8 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,7 +46,7 @@ ADD_LBPM_TEST( TestMap ) #ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db) -ADD_LBPM_TEST( TestColorMassBounceback ) +ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db) diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 8ba375bf..2593fba2 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -3,6 +3,17 @@ #include "common/Utilities.h" #include "common/ScaLBL.h" +std::shared_ptr loadInputs( int nprocs ) +{ + //auto db = std::make_shared( "Domain.in" ); + auto db = std::make_shared(); + db->putScalar( "BC", 0 ); + db->putVector( "nproc", { 1, 1, 1 } ); + db->putVector( "n", { 16, 16, 16 } ); + db->putScalar( "nspheres", 1 ); + db->putVector( "L", { 1, 1, 1 } ); + return db; +} int main (int argc, char **argv) { @@ -31,11 +42,18 @@ int main (int argc, char **argv) double din,dout; int BC=1; - std::shared_ptr Dm(new Domain(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BC)); - - Nz += 2; + // Load inputs + auto db = loadInputs( nprocs ); + int Nx = db->getVector( "n" )[0]; + int Ny = db->getVector( "n" )[1]; + int Nz = db->getVector( "n" )[2]; + int nprocx = db->getVector( "nproc" )[0]; + int nprocy = db->getVector( "nproc" )[1]; + int nprocz = db->getVector( "nproc" )[2]; + std::shared_ptr Dm(new Domain(db,comm)); + + Nx += 2; Ny+=2; Nz += 2; Nx = Ny = Nz; // Cubic domain - int N = Nx*Ny*Nz; //....................................................................... @@ -69,20 +87,8 @@ int main (int argc, char **argv) // Initialize communication structures in averaging domain for (i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = id[i]; Dm->CommInit(); - - Np=0; // number of local pore nodes - //....................................................................... - for (k=1;kid[n] > 0){ - Np++; - } - } - } - } - //........................................................................... + Np=Dm->PoreCount(); + //................................................ if (rank==0) printf ("Create ScaLBL_Communicator \n"); // Create a communicator for the device std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); From 888a9d6176b8729d0f0b2f4189ff62d12db2655e Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 13:35:21 -0400 Subject: [PATCH 104/124] if you knew time as well as i do --- tests/TestFluxBC.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 2593fba2..587f1ea5 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -32,13 +32,9 @@ int main (int argc, char **argv) } { int i,j,k,n; - int Nx,Ny,Nz,Np; bool pBC=true; - int nprocx,nprocy,nprocz; double Lx,Ly,Lz; - Nx = Ny = Nz = 16; Lx = Ly = Lz = 1.f; - nprocx=nprocy=nprocz=1; double din,dout; int BC=1; From cd4a11a6a7cb0808cecd20681b503832cc7fce3b Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 13:36:27 -0400 Subject: [PATCH 105/124] you wouldn't even talk about wasting it --- tests/TestFluxBC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/TestFluxBC.cpp b/tests/TestFluxBC.cpp index 587f1ea5..7548a10b 100644 --- a/tests/TestFluxBC.cpp +++ b/tests/TestFluxBC.cpp @@ -31,7 +31,7 @@ int main (int argc, char **argv) ASSERT(nprocs==1); } { - int i,j,k,n; + int i,j,k,n,Np; bool pBC=true; double Lx,Ly,Lz; Lx = Ly = Lz = 1.f; From a84370999e4cde37f8db930a9a216337b059dfee Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 14:02:45 -0400 Subject: [PATCH 106/124] is that how you manage --- models/ColorModel.cpp | 118 +++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 75c742b8..9fb0c689 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -4,74 +4,74 @@ color lattice boltzmann model #include "models/ColorModel.h" ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, MPI_Comm COMM): - rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), - Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), - Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) { } ScaLBL_ColorModel::~ScaLBL_ColorModel(){ - + } void ScaLBL_ColorModel::ReadParams(string filename){ // read the input database - db = std::make_shared( filename ); - domain_db = db->getDatabase( "Domain" ); - color_db = db->getDatabase( "Color" ); - analysis_db = db->getDatabase( "Analysis" ); - - // Color Model parameters - timestepMax = color_db->getScalar( "timestepMax" ); - tauA = color_db->getScalar( "tauA" ); - tauB = color_db->getScalar( "tauB" ); - rhoA = color_db->getScalar( "rhoA" ); - rhoB = color_db->getScalar( "rhoB" ); - Fx = color_db->getVector( "F" )[0]; - Fy = color_db->getVector( "F" )[1]; - Fz = color_db->getVector( "F" )[2]; - alpha = color_db->getScalar( "alpha" ); - beta = color_db->getScalar( "beta" ); - Restart = color_db->getScalar( "Restart" ); - din = color_db->getScalar( "din" ); - dout = color_db->getScalar( "dout" ); - flux = color_db->getScalar( "flux" ); - inletA=1.f; - inletB=0.f; - outletA=0.f; - outletB=1.f; - - // Read domain parameters - auto L = domain_db->getVector( "L" ); - auto size = domain_db->getVector( "n" ); - auto nproc = domain_db->getVector( "nproc" ); - BoundaryCondition = domain_db->getScalar( "BC" ); - Nx = size[0]; - Ny = size[1]; - Nz = size[2]; - Lx = L[0]; - Ly = L[1]; - Lz = L[2]; - nprocx = nproc[0]; - nprocy = nproc[1]; - nprocz = nproc[2]; - - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + color_db = db->getDatabase( "Color" ); + analysis_db = db->getDatabase( "Analysis" ); - Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis - Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - - Nx+=2; Ny+=2; Nz += 2; - N = Nx*Ny*Nz; - for (int i=0; iid[i] = 1; // initialize this way - Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + // Color Model parameters + timestepMax = color_db->getScalar( "timestepMax" ); + tauA = color_db->getScalar( "tauA" ); + tauB = color_db->getScalar( "tauB" ); + rhoA = color_db->getScalar( "rhoA" ); + rhoB = color_db->getScalar( "rhoB" ); + Fx = color_db->getVector( "F" )[0]; + Fy = color_db->getVector( "F" )[1]; + Fz = color_db->getVector( "F" )[2]; + alpha = color_db->getScalar( "alpha" ); + beta = color_db->getScalar( "beta" ); + Restart = color_db->getScalar( "Restart" ); + din = color_db->getScalar( "din" ); + dout = color_db->getScalar( "dout" ); + flux = color_db->getScalar( "flux" ); + inletA=1.f; + inletB=0.f; + outletA=0.f; + outletB=1.f; - // local copy of the ids - id = new char[N]; + // Read domain parameters + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + BoundaryCondition = domain_db->getScalar( "BC" ); + Nx = size[0]; + Ny = size[1]; + Nz = size[2]; + Lx = L[0]; + Ly = L[1]; + Lz = L[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; - MPI_Barrier(comm); - Dm->CommInit(); - MPI_Barrier(comm); + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) + + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + + Nx+=2; Ny+=2; Nz += 2; + N = Nx*Ny*Nz; + for (int i=0; iid[i] = 1; // initialize this way + Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + + // local copy of the ids + id = new char[N]; + + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); } void ScaLBL_ColorModel::ReadInput(){ @@ -194,7 +194,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) for (int j=0;jid[n]; // Assign the affinity from the paired list for (int idx=0; idx < NLABELS; idx++){ //printf("rank=%i, idx=%i, value=%i, %i, \n",rank(),idx, VALUE,LabelList[idx]); @@ -222,7 +222,7 @@ void ScaLBL_ColorModel::Create(){ //......................................................... // Initialize communication structures in averaging domain - for (int i=0; iNx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i]; + for (int i=0; iid[i] = Dm->id[i]; Mask->CommInit(); Np=Mask->PoreCount(); //........................................................................... From 74d22eebfcfa264f41959b2d7ce25ecde6b15a7a Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 14:05:18 -0400 Subject: [PATCH 107/124] twinkle twinkle little bat --- models/ColorModel.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 9fb0c689..38d1c8bf 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -434,11 +434,6 @@ void ScaLBL_ColorModel::Run(){ starttime = MPI_Wtime(); //......................................... //************ MAIN ITERATION LOOP ***************************************/ - // read the input database - auto db = std::make_shared( "input.db" ); - auto domain_db = db->getDatabase( "Domain" ); - auto color_db = db->getDatabase( "Color" ); - auto analysis_db = db->getDatabase( "Analysis" ); PROFILE_START("Loop"); // runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); From b56cbcdf61f2479e2aa6aae16a28fb42b4ecfe0b Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 14:07:52 -0400 Subject: [PATCH 108/124] how i wonder where you're at --- models/ColorModel.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 38d1c8bf..bdbf5b0e 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -436,7 +436,7 @@ void ScaLBL_ColorModel::Run(){ //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); - // runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); + runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, pBC, beta, Map ); while (timestep < timestepMax ) { //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } PROFILE_START("Update"); @@ -517,10 +517,9 @@ void ScaLBL_ColorModel::Run(){ PROFILE_STOP("Update"); // Run the analysis - // analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); - + analysis.run( timestep, *Averages, Phi, Pressure, Velocity, fq, Den ); } - // analysis.finish(); + analysis.finish(); PROFILE_STOP("Loop"); PROFILE_SAVE("lbpm_color_simulator",1); //************************************************************************ From c18bf1792ff7e3b8396c64da4a0e39a715e03697 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 17:28:39 -0400 Subject: [PATCH 109/124] how doth the little crocodile --- models/ColorModel.cpp | 34 ++++++++++++++++------------------ models/ColorModel.h | 4 +++- tests/TestColorBubble.cpp | 35 ++++++++++++----------------------- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index bdbf5b0e..6d914092 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -40,6 +40,8 @@ void ScaLBL_ColorModel::ReadParams(string filename){ inletB=0.f; outletA=0.f; outletB=1.f; + + if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) // Read domain parameters auto L = domain_db->getVector( "L" ); @@ -56,19 +58,14 @@ void ScaLBL_ColorModel::ReadParams(string filename){ nprocy = nproc[1]; nprocz = nproc[2]; - if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details) - +} +void ScaLBL_ColorModel::SetDomain(){ Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases - Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; for (int i=0; iid[i] = 1; // initialize this way Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object - - // local copy of the ids - id = new char[N]; - MPI_Barrier(comm); Dm->CommInit(); MPI_Barrier(comm); @@ -80,7 +77,7 @@ void ScaLBL_ColorModel::ReadInput(){ //....................................................................... if (rank == 0) printf("Read input media... \n"); //....................................................................... - Dm->ReadIDs(); + Mask->ReadIDs(); sprintf(LocalRankString,"%05d",Dm->rank()); sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); @@ -194,7 +191,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase) for (int j=0;jid[n]; + VALUE=Mask->id[n]; // Assign the affinity from the paired list for (int idx=0; idx < NLABELS; idx++){ //printf("rank=%i, idx=%i, value=%i, %i, \n",rank(),idx, VALUE,LabelList[idx]); @@ -222,7 +219,7 @@ void ScaLBL_ColorModel::Create(){ //......................................................... // Initialize communication structures in averaging domain - for (int i=0; iid[i] = Dm->id[i]; + for (int i=0; iid[i] = Mask->id[i]; Mask->CommInit(); Np=Mask->PoreCount(); //........................................................................... @@ -285,13 +282,7 @@ void ScaLBL_ColorModel::Create(){ /******************************************************** * AssignComponentLabels * ********************************************************/ - -void ScaLBL_ColorModel::Initialize(){ - /* - * This function initializes model (includes both mobile and immobile components) - */ - int rank=Dm->rank(); - // Compute the solid interaction potential and copy result to device +void ScaLBL_ColorModel::SolidPotential(){ if (rank==0) printf("Computing solid interaction potential \n"); double *PhaseLabel; PhaseLabel=new double [Nx*Ny*Nz]; @@ -390,8 +381,15 @@ void ScaLBL_ColorModel::Initialize(){ PFILE = fopen(LocalRankFilename,"wb"); fwrite(Psnorm.data(),8,N,PFILE); fclose(PFILE); - +} +void ScaLBL_ColorModel::Initialize(){ + /* + * This function initializes model + */ + int rank=Dm->rank(); double count_wet=0.f; + double *PhaseLabel; + PhaseLabel=new double [Nx*Ny*Nz]; for (int k=1; k db0); - void AssignComponentLabels(double *phase); + void SetDomain(); void ReadInput(); void Create(); void Initialize(); + void SolidPotential(); void Run(); void WriteDebug(); @@ -74,6 +75,7 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); + void AssignComponentLabels(double *phase); }; diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index 96313ac7..ffbd38d3 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -14,8 +14,6 @@ using namespace std; inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius){ // initialize a bubble int i,j,k,n; - int Np=0; - double sum=0.f; int rank = ColorModel.Mask->rank(); int nprocx = ColorModel.Mask->nprocx(); int nprocy = ColorModel.Mask->nprocy(); @@ -24,21 +22,11 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) int Ny = ColorModel.Mask->Ny; int Nz = ColorModel.Mask->Nz; if (rank == 0) cout << "Setting up bubble..." << endl; - sum=0; Np=0; for (k=0;kSDs(i,j,k) = 100.f; - // Initialize phase positions field - if (ColorModel.Averages->SDs(i,j,k) < 0.0){ - ColorModel.id[n] = 0; - ColorModel.id[n] = 0; - } - else { - sum++; - Np++; - } } } } @@ -47,19 +35,19 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) for (j=0;jiproc(); - int jglobal= j+(Ny-2)*ColorModel.Dm->jproc(); - int kglobal= k+(Nz-2)*ColorModel.Dm->kproc(); + int iglobal= i+(Nx-2)*ColorModel.Mask->iproc(); + int jglobal= j+(Ny-2)*ColorModel.Mask->jproc(); + int kglobal= k+(Nz-2)*ColorModel.Mask->kproc(); // Initialize phase position field for parallel bubble test if ((iglobal-0.5*(Nx-2)*nprocx)*(iglobal-0.5*(Nx-2)*nprocx) +(jglobal-0.5*(Ny-2)*nprocy)*(jglobal-0.5*(Ny-2)*nprocy) +(kglobal-0.5*(Nz-2)*nprocz)*(kglobal-0.5*(Nz-2)*nprocz) < BubbleRadius*BubbleRadius){ - ColorModel.id[n] = 2; - ColorModel.id[n] = 2; + ColorModel.Mask->id[n] = 2; + ColorModel.Mask->id[n] = 2; } else{ - ColorModel.id[n]=1; - ColorModel.id[n]=1; + ColorModel.Mask->id[n]=1; + ColorModel.Mask->id[n]=1; } } } @@ -92,12 +80,13 @@ int main(int argc, char **argv) auto filename = argv[1]; ScaLBL_ColorModel ColorModel(rank,nprocs,comm); ColorModel.ReadParams(filename); - //ColorModel.ReadInput(); + ColorModel.SetDomain(); + //ColorModel.ReadInput(); double radius=15.5; InitializeBubble(ColorModel,radius); - ColorModel.Create(); - ColorModel.Initialize(); - ColorModel.Run(); + ColorModel.Create(); // creating the model will create data structure to match the pore structure and allocate variables + ColorModel.Initialize(); // initializing the model will set initial conditions for variables + ColorModel.Run(); ColorModel.WriteDebug(); } // **************************************************** From c5994631ee992447953ff09a08c12c41a34c3f4e Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 17:32:24 -0400 Subject: [PATCH 110/124] improve his shining tail --- models/ColorModel.cpp | 2 +- models/ColorModel.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 6d914092..1d2fdd33 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -282,7 +282,7 @@ void ScaLBL_ColorModel::Create(){ /******************************************************** * AssignComponentLabels * ********************************************************/ -void ScaLBL_ColorModel::SolidPotential(){ +void ScaLBL_ColorModel::AssignSolidPotential(){ if (rank==0) printf("Computing solid interaction potential \n"); double *PhaseLabel; PhaseLabel=new double [Nx*Ny*Nz]; diff --git a/models/ColorModel.h b/models/ColorModel.h index 67f7a21d..d5549f31 100644 --- a/models/ColorModel.h +++ b/models/ColorModel.h @@ -28,7 +28,7 @@ public: void ReadInput(); void Create(); void Initialize(); - void SolidPotential(); + void AssignSolidPotential(); void Run(); void WriteDebug(); From baf501a14a341be9956bd1b436e136352f7ee08a Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 19:42:58 -0400 Subject: [PATCH 111/124] and pour the waters of the nile --- models/MRTModel.cpp | 181 +++++++++++++++++++++++++++++ models/MRTModel.h | 69 ++++++++++++ tests/TestPoiseuille.cpp | 238 ++++----------------------------------- 3 files changed, 271 insertions(+), 217 deletions(-) create mode 100644 models/MRTModel.cpp create mode 100644 models/MRTModel.h diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp new file mode 100644 index 00000000..74eae2ff --- /dev/null +++ b/models/MRTModel.cpp @@ -0,0 +1,181 @@ +/* + * Multi-relaxation time LBM Model + */ +#include "models/MRTModel.h" + +ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM): +rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0), +Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0), +Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM) +{ + +} +ScaLBL_MRTModel::~ScaLBL_MRTModel(){ + +} + +void ScaLBL_MRTModel::ReadParams(string filename){ + // read the input database + db = std::make_shared( filename ); + domain_db = db->getDatabase( "Domain" ); + mrt_db = db->getDatabase( "MRT" ); + + // Color Model parameters + timestepMax = mrt_db->getScalar( "timestepMax" ); + tauA = mrt_db->getScalar( "tau" ); + Fx = mrt_db->getVector( "F" )[0]; + Fy = mrt_db->getVector( "F" )[1]; + Fz = mrt_db->getVector( "F" )[2]; + Restart = mrt_db->getScalar( "Restart" ); + din = mrt_db->getScalar( "din" ); + dout = mrt_db->getScalar( "dout" ); + flux = mrt_db->getScalar( "flux" ); + + // Read domain parameters + auto L = domain_db->getVector( "L" ); + auto size = domain_db->getVector( "n" ); + auto nproc = domain_db->getVector( "nproc" ); + BoundaryCondition = domain_db->getScalar( "BC" ); + Nx = size[0]; + Ny = size[1]; + Nz = size[2]; + Lx = L[0]; + Ly = L[1]; + Lz = L[2]; + nprocx = nproc[0]; + nprocy = nproc[1]; + nprocz = nproc[2]; + mu=(tau-0.5)/3.0; +} +void ScaLBL_MRTModel::SetDomain(){ + Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases + Nx+=2; Ny+=2; Nz += 2; + N = Nx*Ny*Nz; + for (int i=0; iid[i] = 1; // initialize this way + //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object + MPI_Barrier(comm); + Dm->CommInit(); + MPI_Barrier(comm); +} + +void ScaLBL_MRTModel::ReadInput(){ + int rank=Dm->rank(); + size_t readID; + //....................................................................... + if (rank == 0) printf("Read input media... \n"); + //....................................................................... + Mask->ReadIDs(); + + sprintf(LocalRankString,"%05d",Dm->rank()); + sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); + sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); + + // .......... READ THE INPUT FILE ....................................... + //........................................................................... + if (rank == 0) cout << "Reading in signed distance function..." << endl; + //....................................................................... + sprintf(LocalRankString,"%05d",rank); + sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); + ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); + MPI_Barrier(comm); + if (rank == 0) cout << "Domain set." << endl; +} + +void ScaLBL_MRTModel::Create(){ + /* + * This function creates the variables needed to run a LBM + */ + int rank=Mask->rank(); + //......................................................... + // Initialize communication structures in averaging domain + for (int i=0; iid[i] = Mask->id[i]; + Mask->CommInit(); + Np=Mask->PoreCount(); + //........................................................................... + if (rank==0) printf ("Create ScaLBL_Communicator \n"); + // Create a communicator for the device (will use optimized layout) + // ScaLBL_Communicator ScaLBL_Comm(Mask); // original + ScaLBL_Comm = std::shared_ptr(new ScaLBL_Communicator(Mask)); + + int Npad=(Np/16 + 2)*16; + if (rank==0) printf ("Set up memory efficient layout \n"); + Map.resize(Nx,Ny,Nz); Map.fill(-2); + auto neighborList= new int[18*Npad]; + Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np); + MPI_Barrier(comm); + //........................................................................... + // MAIN VARIABLES ALLOCATED HERE + //........................................................................... + // LBM variables + if (rank==0) printf ("Allocating distributions \n"); + //......................device distributions................................. + int dist_mem_size = Np*sizeof(double); + int neighborSize=18*(Np*sizeof(int)); + //........................................................................... + ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); + ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); + ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); + ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); + //........................................................................... + // Update GPU data structures + if (rank==0) printf ("Setting up device map and neighbor list \n"); + // copy the neighbor list + ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); + MPI_Barrier(comm); +} + +void ScaLBL_MRTModel::Initialize(){ + /* + * This function initializes model + */ + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); +} + +void ScaLBL_MRTModel::Run(){ + double rlx_setA=1.0/tau; + double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); + //.......create and start timer............ + double starttime,stoptime,cputime; + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + starttime = MPI_Wtime(); + if (rank==0) printf("Beginning AA timesteps...\n"); + if (rank==0) printf("********************************************************\n"); + timestep=0; + while (timestep < timestepMax) { + //************************************************************************/ + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + timestep++; + ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL + ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE + ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + //************************************************************************/ + } + //************************************************************************/ + stoptime = MPI_Wtime(); + // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; + cputime = stoptime - starttime; + // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; + double MLUPS = double(Np*timestep)/cputime/1000000; +// if (rank==0) printf("********************************************************\n"); +// if (rank==0) printf("CPU time = %f \n", cputime); +// if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS); + MLUPS *= nprocs; +} + +void ScaLBL_MRTModel::VelocityField(double *Vz){ + + int SIZE=Np*sizeof(double); + ScaLBL_D3Q19_Momentum(fq,Velocity, Np); + ScaLBL_DeviceBarrier(); MPI_Barrier(comm); + ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE); + +} diff --git a/models/MRTModel.h b/models/MRTModel.h new file mode 100644 index 00000000..17e0894c --- /dev/null +++ b/models/MRTModel.h @@ -0,0 +1,69 @@ +/* + * Multi-relaxation time LBM Model + */ +#include +#include +#include +#include +#include +#include +#include + +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "ProfilerApp.h" + +class ScaLBL_MRTModel{ +public: + ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM); + ~ScaLBL_MRTModel(); + + // functions in they should be run + void ReadParams(string filename); + void ReadParams(std::shared_ptr db0); + void SetDomain(); + void ReadInput(); + void Create(); + void Initialize(); + void Run(); + void VelocityField(double *Vz); + + bool Restart,pBC; + int timestep,timestepMax; + int BoundaryCondition; + double tau,mu; + double Fx,Fy,Fz,flux; + double din,dout; + + int Nx,Ny,Nz,N,Np; + int rank,nprocx,nprocy,nprocz,nprocs; + double Lx,Ly,Lz; + + std::shared_ptr Dm; // this domain is for analysis + std::shared_ptr Mask; // this domain is for lbm + std::shared_ptr ScaLBL_Comm; + std::shared_ptr Averages; + + // input database + std::shared_ptr db; + std::shared_ptr domain_db; + std::shared_ptr color_db; + std::shared_ptr analysis_db; + + IntArray Map; + int *NeighborList; + double *fq; + double *Velocity; + double *Pressure; + +private: + MPI_Comm comm; + + // filenames + char LocalRankString[8]; + char LocalRankFilename[40]; + char LocalRestartFile[40]; + + //int rank,nprocs; + void LoadParams(std::shared_ptr db0); +}; \ No newline at end of file diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 73909c93..ba23420d 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -8,6 +8,7 @@ #include #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" +#include "models/MRTModel.h" //*************************************************************************************** int main(int argc, char **argv) @@ -28,226 +29,35 @@ int main(int argc, char **argv) printf("Running Unit Test: TestPoiseuille \n"); printf("********************************************************\n"); } + + ScaLBL_MRTModel MRT(rank,nprocs,comm); - // BGK Model parameters - double tau,Fx,Fy,Fz; - // Domain variables - int i,j,k,n; - int timestep = 0; - - tau = 1.0; - double mu=(tau-0.5)/3.0; - double rlx_setA=1.0/tau; - double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - Fx = 0; Fy = 0; - Fz = 1e-3; //1.f; // 1e-3; - // Load inputs - if (rank==0) printf("Loading input database \n"); - auto FILENAME = argv[1]; - auto db = std::make_shared( FILENAME ); - auto domain_db = db->getDatabase( "Domain" ); - int Nx = domain_db->getVector( "n" )[0]; - int Ny = domain_db->getVector( "n" )[1]; - int Nz = domain_db->getVector( "n" )[2]; - int nprocx = domain_db->getVector( "nproc" )[0]; - int nprocy = domain_db->getVector( "nproc" )[1]; - int nprocz = domain_db->getVector( "nproc" )[2]; - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } - - MPI_Barrier(comm); - int kproc = rank/(nprocx*nprocy); - int jproc = (rank-nprocx*nprocy*kproc)/nprocx; - int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc; - - if (rank == 0) { - printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); - } - MPI_Barrier(comm); - if (rank == 1){ - printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc); - printf("\n\n"); - } - - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - - std::shared_ptr Dm( new Domain(domain_db,comm)); - - Nx += 2; - Ny += 2; - Nz += 2; - int N = Nx*Ny*Nz; - - //....................................................................... - // Assign the phase ID field - //....................................................................... - char LocalRankString[8]; - sprintf(LocalRankString,"%05d",rank); - char LocalRankFilename[40]; - sprintf(LocalRankFilename,"ID.%05i",rank); - /* - FILE *IDFILE = fopen(LocalRankFilename,"rb"); - if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - fread(Dm->id,1,N,IDFILE); - fclose(IDFILE); - */ - - // initialize empty domain - for (k=0;kid[n] = 0; - else if (i>Nx-3) Dm->id[n] = 0; - else Dm->id[n]=1; - } - } - } - - Dm->CommInit(); - MPI_Barrier(comm); - - //....................................................................... - // Compute the media porosity - //....................................................................... - double sum; - double sum_local=0.0, porosity; - int Np=0; // number of local pore nodes - for (k=1;kid[n] > 0){ - sum_local+=1.0; - Np++; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = sum*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - - MPI_Barrier(comm); - if (rank == 0) cout << "Domain set." << endl; - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - - // Create a communicator for the device - std::shared_ptr ScaLBL_Comm(new ScaLBL_Communicator(Dm)); - - //...........device phase ID................................................. - if (rank==0) printf ("Copying phase ID to device \n"); - char *ID; - ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory - // Copy to the device - ScaLBL_CopyToDevice(ID, Dm->id, N); - //........................................................................... - - if (rank==0){ - printf("Total domain size = %i \n",N); - printf("Reduced domain size = %i \n",Np); - } - - // LBM variables - if (rank==0) printf ("Allocating distributions \n"); - if (rank==0) printf ("Set up the neighborlist \n"); - int Npad=Np+32; - int neighborSize=18*Npad*sizeof(int); - int *neighborList; - IntArray Map(Nx,Ny,Nz); - neighborList= new int[18*Npad]; - Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - MPI_Barrier(comm); - - //......................device distributions................................. - int dist_mem_size = Np*sizeof(double); - - int *NeighborList; - // double *f_even,*f_odd; - double * dist; - double * Velocity; - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &dist, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np); - ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize); - //........................................................................... - - /* - * AA Algorithm begins here - * - */ - ScaLBL_D3Q19_Init(dist, Np); - - //.......create and start timer............ - double starttime,stoptime,cputime; - - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - starttime = MPI_Wtime(); - - /************ MAIN ITERATION LOOP (timing communications)***************************************/ - // ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]); - // ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]); - // ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - // - - if (rank==0) printf("Beginning AA timesteps...\n"); - if (rank==0) printf("********************************************************\n"); - - while (timestep < 2000) { - - ScaLBL_Comm->SendD3Q19AA(dist); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - - ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE - ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - //************************************************************************/ - - } - //************************************************************************/ - stoptime = MPI_Wtime(); - // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; - cputime = stoptime - starttime; - // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; - double MLUPS = double(Np*timestep)/cputime/1000000; - // if (rank==0) printf("********************************************************\n"); - // if (rank==0) printf("CPU time = %f \n", cputime); - // if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - - double *Vz; - Vz= new double [Np]; - int SIZE=Np*sizeof(double); - ScaLBL_D3Q19_Momentum(dist,Velocity, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&Vz[0],&Velocity[2*Np],SIZE); - - if (rank == 0) printf("Force: %f,%f,%f \n",Fx,Fy,Fz); + MRT.ReadParams(filename); + MRT.SetDomain(); + MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables + MRT.Initialize(); // initializing the model will set initial conditions for variables + MRT.Run(); + double *Vz; Vz= new double [MRT.Np]; + MRT.VelocityField(Vz); + if (rank == 0) printf("Force: %f,%f,%f \n",MRT.Fx,MRT.Fy,MRT.Fz); + double mu = MRT.mu; + int Nx = MRT.Nx; + int Ny = MRT.Ny; + int Nz = MRT.Nz; + double Fz = MRT.Fz; double vz; double W = 1.f*Nx-4.f; j=Ny/2; k=Nz/2; if (rank == 0) printf("Channel width=%f \n",W); if (rank == 0) printf("ID flag vz analytical\n"); - MPI_Barrier(comm); + if (rank == 0) { for (i=0;iid[n]); - - n = Map(i,j,k); + printf("%i ",MRT.Mask->id[n]); + n = MRT.Map(i,j,k); //printf("%i,%i,%i; %i :",i,j,k,n); if (n<0) {vz =0.f; printf(" b "); } else { vz=Vz[n]; printf(" a "); } @@ -260,14 +70,11 @@ int main(int argc, char **argv) } printf("\n"); } - - if (rank == 1) { for (i=0;iid[n]); - - n = Map(i,j,k); + printf("%i ",MRT.Mask->id[n]); + n = MRT.Map(i,j,k); //printf("%i,%i,%i; %i :",i,j,k,n); if (n<0) {vz =0.f; printf(" b "); } else { vz=Vz[n]; printf(" a "); } @@ -280,9 +87,6 @@ int main(int argc, char **argv) } printf("\n"); } - - - } // **************************************************** From dff7889131d62363712912d113cb5d9244e7de46 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 19:54:46 -0400 Subject: [PATCH 112/124] on every golden scale --- example/Piston/poiseuille.db | 20 ++++++++++++++++++++ models/MRTModel.cpp | 5 +++-- models/MRTModel.h | 9 ++++----- tests/TestPoiseuille.cpp | 5 +++-- 4 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 example/Piston/poiseuille.db diff --git a/example/Piston/poiseuille.db b/example/Piston/poiseuille.db new file mode 100644 index 00000000..af4cb7b5 --- /dev/null +++ b/example/Piston/poiseuille.db @@ -0,0 +1,20 @@ +MRT { + tau = 1.0 + F = 0, 0, 1.0e-5 + Restart = false + pBC = 0 + din = 1.0 + dout = 1.0 + timestepMax = 2000 + tol = 1e-5 + flux = 0.0 +} + +Domain { + nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) + n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz) + n_spheres = 1 // Number of spheres + L = 1, 1, 1 // Length of domain (x,y,z) + BC = 0 // Boundary condition type +} + diff --git a/models/MRTModel.cpp b/models/MRTModel.cpp index 74eae2ff..58982f44 100644 --- a/models/MRTModel.cpp +++ b/models/MRTModel.cpp @@ -22,7 +22,7 @@ void ScaLBL_MRTModel::ReadParams(string filename){ // Color Model parameters timestepMax = mrt_db->getScalar( "timestepMax" ); - tauA = mrt_db->getScalar( "tau" ); + tau = mrt_db->getScalar( "tau" ); Fx = mrt_db->getVector( "F" )[0]; Fy = mrt_db->getVector( "F" )[1]; Fz = mrt_db->getVector( "F" )[2]; @@ -52,6 +52,7 @@ void ScaLBL_MRTModel::SetDomain(){ Mask = std::shared_ptr(new Domain(domain_db,comm)); // mask domain removes immobile phases Nx+=2; Ny+=2; Nz += 2; N = Nx*Ny*Nz; + Distance.resize(Nx,Ny,Nz); for (int i=0; iid[i] = 1; // initialize this way //Averages = std::shared_ptr ( new TwoPhase(Dm) ); // TwoPhase analysis object MPI_Barrier(comm); @@ -77,7 +78,7 @@ void ScaLBL_MRTModel::ReadInput(){ //....................................................................... sprintf(LocalRankString,"%05d",rank); sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); - ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); + ReadBinaryFile(LocalRankFilename, Distance.data(), N); MPI_Barrier(comm); if (rank == 0) cout << "Domain set." << endl; } diff --git a/models/MRTModel.h b/models/MRTModel.h index 17e0894c..ff9db633 100644 --- a/models/MRTModel.h +++ b/models/MRTModel.h @@ -9,6 +9,7 @@ #include #include +#include "common/ScaLBL.h" #include "common/Communication.h" #include "common/MPI_Helpers.h" #include "ProfilerApp.h" @@ -42,15 +43,13 @@ public: std::shared_ptr Dm; // this domain is for analysis std::shared_ptr Mask; // this domain is for lbm std::shared_ptr ScaLBL_Comm; - std::shared_ptr Averages; - // input database std::shared_ptr db; std::shared_ptr domain_db; - std::shared_ptr color_db; - std::shared_ptr analysis_db; + std::shared_ptr mrt_db; IntArray Map; + DoubleArray Distance; int *NeighborList; double *fq; double *Velocity; @@ -66,4 +65,4 @@ private: //int rank,nprocs; void LoadParams(std::shared_ptr db0); -}; \ No newline at end of file +}; diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index ba23420d..bceff106 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -30,10 +30,11 @@ int main(int argc, char **argv) printf("********************************************************\n"); } + int i,j,k,n; ScaLBL_MRTModel MRT(rank,nprocs,comm); - + auto filename = argv[1]; MRT.ReadParams(filename); - MRT.SetDomain(); + MRT.SetDomain(); // this reads in the domain MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables MRT.Initialize(); // initializing the model will set initial conditions for variables MRT.Run(); From b0cf3abfa2c6cd0fe234d61c176b69b671db36e3 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 20:22:16 -0400 Subject: [PATCH 113/124] how cheerfully he seems to grin --- tests/TestPoiseuille.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index bceff106..464396f0 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -10,6 +10,23 @@ #include "common/MPI_Helpers.h" #include "models/MRTModel.h" +void ParallelPlates(ScaLBL_MRTModel &MRT){ + // initialize empty domain + int Nx = MRT.Nx; + int Ny = MRT.Ny; + int Nz = MRT.Nz; + for (k=0;kid[n] = 0; + else if (i>Nx-3) MRT.Mask->id[n] = 0; + else MRT.Mask->id[n]=1; + } + } + } +} + //*************************************************************************************** int main(int argc, char **argv) { @@ -34,7 +51,9 @@ int main(int argc, char **argv) ScaLBL_MRTModel MRT(rank,nprocs,comm); auto filename = argv[1]; MRT.ReadParams(filename); + Set domain is the problem!! MRT.SetDomain(); // this reads in the domain + ParallelPlates(MRT); MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables MRT.Initialize(); // initializing the model will set initial conditions for variables MRT.Run(); From eb223856422f1f84c7e06eb651687f6372d36773 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Sat, 19 May 2018 20:25:16 -0400 Subject: [PATCH 114/124] how neatly he spreads his claws --- example/Piston/poiseuille.db | 2 +- tests/TestPoiseuille.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/Piston/poiseuille.db b/example/Piston/poiseuille.db index af4cb7b5..3f276500 100644 --- a/example/Piston/poiseuille.db +++ b/example/Piston/poiseuille.db @@ -1,6 +1,6 @@ MRT { tau = 1.0 - F = 0, 0, 1.0e-5 + F = 0, 0, 1.0e-4 Restart = false pBC = 0 din = 1.0 diff --git a/tests/TestPoiseuille.cpp b/tests/TestPoiseuille.cpp index 464396f0..a6adcd65 100644 --- a/tests/TestPoiseuille.cpp +++ b/tests/TestPoiseuille.cpp @@ -12,6 +12,7 @@ void ParallelPlates(ScaLBL_MRTModel &MRT){ // initialize empty domain + int i,j,k,n; int Nx = MRT.Nx; int Ny = MRT.Ny; int Nz = MRT.Nz; @@ -51,7 +52,6 @@ int main(int argc, char **argv) ScaLBL_MRTModel MRT(rank,nprocs,comm); auto filename = argv[1]; MRT.ReadParams(filename); - Set domain is the problem!! MRT.SetDomain(); // this reads in the domain ParallelPlates(MRT); MRT.Create(); // creating the model will create data structure to match the pore structure and allocate variables From b0bb1e3680983620856894efdfe9039b6cc7658c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 21 May 2018 06:58:00 -0400 Subject: [PATCH 115/124] how cheerfully he seems to grin --- tests/TestColorMassBounceback.cpp | 2 +- tests/TestColorSquareTube.cpp | 471 +++--------------------------- 2 files changed, 45 insertions(+), 428 deletions(-) diff --git a/tests/TestColorMassBounceback.cpp b/tests/TestColorMassBounceback.cpp index dc6a9525..c05c245e 100644 --- a/tests/TestColorMassBounceback.cpp +++ b/tests/TestColorMassBounceback.cpp @@ -24,7 +24,7 @@ int main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&nprocs); - int check; + int check=0; { // parallel domain size (# of sub-domains) int i,j,k,n,Npad; diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index 3a763b22..2644b710 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -39,6 +39,40 @@ std::shared_ptr loadInputs( int nprocs ) return db; } +void InitSquareTube(ScaLBL_ColorModel &ColorModel){ + for (k=0;kid[n]=0; + } + } + } + + int kproc = rank/(nprocx*nprocy); + int jproc = (rank-nprocx*nprocy*kproc)/nprocx; + int iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; + printf("rank=%i, %i,%i,%i \n",rank,iproc,jproc,kproc); + // Initialize a square tube + for (k=1;kid[n]=0; + else if (iglobal > (Nx-2)*nprocx-2) ColorModel.Mask->id[n]=0; + else if (jglobal < 2) ColorModel.Mask->id[n]=0; + else if (jglobal > (Ny-2)*nprocy-2) ColorModel.Mask->id[n]=0; + else if (kglobal < 20) ColorModel.Mask->id[n]=1; + else ColorModel.Mask->id[n]=2; + } + } + } +} //*************************************************************************************** int main(int argc, char **argv) @@ -59,433 +93,16 @@ int main(int argc, char **argv) printf("Running Color Model: TestColor \n"); printf("********************************************************\n"); } - - // BGK Model parameters - string FILENAME; - unsigned int nBlocks, nthreads; - int timestepMax, interval; - double Fx,Fy,Fz,tol; - // Domain variables - int i,j,k,n; - - //if (rank == 0) printf("dim=%d\n",dim); - int timestep = 1; - int timesteps = 100; - int centralNode = 2; - - double tauA = 1.0; - double tauB = 1.0; - double rhoA = 1.0; - double rhoB = 1.0; - double alpha = 0.001; - double beta = 0.95; - double tau = 1.0; - double mu=(tau-0.5)/3.0; - double rlx_setA=1.0/tau; - double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA); - - Fx = Fy = 0.f; - Fz = 0.f; - - int typeBC; - double din, dout, flux; - double inletA,inletB,outletA,outletB; - inletA=1.f; - inletB=0.f; - outletA=0.f; - outletB=1.f; - typeBC=4; - flux = 10.f; - dout=1.f; - - // Load inputs - auto db = loadInputs( nprocs ); - int Nx = db->getVector( "n" )[0]; - int Ny = db->getVector( "n" )[1]; - int Nz = db->getVector( "n" )[2]; - int nprocx = db->getVector( "nproc" )[0]; - int nprocy = db->getVector( "nproc" )[1]; - int nprocz = db->getVector( "nproc" )[2]; - - if (rank==0){ - printf("********************************************************\n"); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("********************************************************\n"); - } - - double iVol_global = 1.0/Nx/Ny/Nz/nprocx/nprocy/nprocz; - - std::shared_ptr Dm (new Domain(db,comm)); - Dm->CommInit(); - - Nx += 2; - Ny += 2; - Nz += 2; - int N = Nx*Ny*Nz; - - //....................................................................... - // Assign the phase ID field - //....................................................................... - char LocalRankString[8]; - sprintf(LocalRankString,"%05d",rank); - char LocalRankFilename[40]; - sprintf(LocalRankFilename,"ID.%05i",rank); - - - for (k=0;kid[n]=0; - } - } - } - - int kproc = rank/(nprocx*nprocy); - int jproc = (rank-nprocx*nprocy*kproc)/nprocx; - int iproc = rank-nprocx*nprocy*kproc-nprocx*jproc; - printf("rank=%i, %i,%i,%i \n",rank,iproc,jproc,kproc); - // Initialize a square tube - for (k=1;kid[n]=0; - else if (iglobal > (Nx-2)*nprocx-2) Dm->id[n]=0; - else if (jglobal < 2) Dm->id[n]=0; - else if (jglobal > (Ny-2)*nprocy-2) Dm->id[n]=0; - else if (kglobal < 20) Dm->id[n]=1; - else Dm->id[n]=2; - } - } - } - Dm->CommInit(); - - //....................................................................... - // Compute the media porosity, assign phase labels and solid composition - //....................................................................... - double sum; - double sum_local=0.0, porosity; - int Np=0; // number of local pore nodes - double *PhaseLabel; - PhaseLabel = new double[N]; - //Dm->AssignComponentLabels(PhaseLabel); - //....................................................................... - for (k=1;kid[n] > 0){ - sum_local+=1.0; - Np++; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = sum*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - if (rank==0) printf ("Create ScaLBL_Communicator \n"); - MPI_Barrier(comm); - - // Create a communicator for the device (will use optimized layout) - ScaLBL_Communicator ScaLBL_Comm(Dm); - //Create a second communicator based on the regular data layout - ScaLBL_Communicator ScaLBL_Comm_Regular(Dm); - - //...........device phase ID................................................. - if (rank==0) printf ("Copying phase ID to device \n"); - char *ID; - ScaLBL_AllocateDeviceMemory((void **) &ID, N); // Allocate device memory - // Copy to the device - ScaLBL_CopyToDevice(ID, Dm->id, N); - //........................................................................... - - if (rank==0){ - printf("Total domain size = %i \n",N); - printf("Reduced domain size = %i \n",Np); - } - - // LBM variables - if (rank==0) printf ("Set up the neighborlist \n"); - - int neighborSize=18*Np*sizeof(int); - int *neighborList; - IntArray Map(Nx,Ny,Nz); - neighborList= new int[18*Np]; - - ScaLBL_Comm.MemoryOptimizedLayoutAA(Map,neighborList,Dm->id,Np); - MPI_Barrier(comm); - - //......................device distributions................................. - int dist_mem_size = Np*sizeof(double); - if (rank==0) printf ("Allocating distributions \n"); - - int *NeighborList; - int *dvcMap; - // double *f_even,*f_odd; - double *fq, *Aq, *Bq; - double *Den, *Phi; - double *ColorGrad; - double *Vel; - double *Pressure; - - //........................................................................... - ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize); - ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np); - - ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Aq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Bq, 7*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Den, 2*dist_mem_size); - ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nx*Ny*Nz); - ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &Vel, 3*sizeof(double)*Np); - ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np); - - //........................................................................... - // Update GPU data structures - if (rank==0) printf ("Setting up device map and neighbor list \n"); - int *TmpMap; - TmpMap=new int[Np]; - for (k=1; kkproc()==0){ - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2); - } - if (Dm->kproc() == nprocz-1){ - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3); - } - - //************ MAIN ITERATION LOOP (timing communications)*************************************** - - if (rank==0) printf("Beginning AA timesteps...\n"); - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("No. of timesteps for timing: %i \n", timesteps); - - //.......create and start timer............ - double starttime,stoptime,cputime; - - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - starttime = MPI_Wtime(); - //timesteps=20; - //timestep=1; - while (timestep < timesteps) { - - // *************ODD TIMESTEP************* - // Compute the Phase indicator field - // Read for Aq, Bq happens in this routine (requires communication) - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FROM NORMAL - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Set BCs - if (typeBC > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - if (typeBC == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - if (typeBC == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - - // *************EVEN TIMESTEP************* - // Compute the Phase indicator field - ScaLBL_Comm.BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE - ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm.next, Np); - - // Halo exchange for phase field - ScaLBL_Comm_Regular.SendHalo(Phi); - ScaLBL_Comm_Regular.RecvHalo(Phi); - - // Perform the collision operation - ScaLBL_Comm.SendD3Q19AA(fq); //READ FORM NORMAL - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm.next, Np, Np); - ScaLBL_Comm.RecvD3Q19AA(fq); //WRITE INTO OPPOSITE - // Set boundary conditions - if (typeBC > 0){ - ScaLBL_Comm.Color_BC_z(dvcMap, Phi, Den, inletA, inletB); - ScaLBL_Comm.Color_BC_Z(dvcMap, Phi, Den, outletA, outletB); - } - if (typeBC == 3){ - ScaLBL_Comm.D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - else if (typeBC == 4){ - din = ScaLBL_Comm.D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep); - ScaLBL_Comm.D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep); - } - ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Vel, rhoA, rhoB, tauA, tauB, - alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm.next, Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - timestep++; - //************************************************************************ - - } - //************************************************************************ - stoptime = MPI_Wtime(); - // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; - cputime = stoptime - starttime; - // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; - double MLUPS = double(Np*timestep)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (process)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // Number of memory references for color model - double MemoryRefs = double(Np)*(77*8+(9+7+7)*4); // extra memory refs to read from neighborlist (every other timestep) - // number of memory references for the swap algorithm - GigaBytes / second - if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*timestep/1e9/cputime); - // Report bandwidth in Gigabits per second - // communication bandwidth includes both send and recieve - if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime); - if (rank==0) printf("Aggregated communication bandwidth = %f Gbit/sec \n",nprocs*ScaLBL_Comm.CommunicationCount*64*timestep/1e9/cputime); - - double *VEL; - VEL= new double [3*Np]; - int SIZE=3*Np*sizeof(double); - ScaLBL_D3Q19_Momentum(fq,Vel,Np); - ScaLBL_DeviceBarrier(); MPI_Barrier(comm); - ScaLBL_CopyToHost(&VEL[0],&Vel[0],SIZE); - - sum_local=0.f; - sum = 0.f; - for (k=1;kid[n] > 0){ - int idx = Map(i,j,k); - sum_local+=VEL[2*Np+idx]; - } - } - } - } - MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm); - double PoreVel = sum*iVol_global; - if (rank==0) printf("Average velocity = %f \n",PoreVel); - - if (rank==0){ - printf("Printing inlet velocity for rank=0 \n"); - k=1; - for (j=1;jid[n] > 0){ - int idx = Map(i,j,k); - double vz = VEL[2*Np+idx]; - printf("%f ",vz); - } - } - printf("\n"); - } - } - - /* - double *PHASE; - PHASE= new double [Nx*Ny*Nz]; - SIZE=Nx*Ny*Nz*sizeof(double); - ScaLBL_CopyToHost(&PHASE[0],&Phi[0],SIZE); - - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PHASE,8,N,OUTFILE); - fclose(OUTFILE); - - double *DENA, *DENB, *TMPDAT; - SIZE=Np*sizeof(double); - TMPDAT = new double [Np]; - DENA= new double [Nx*Ny*Nz]; - DENB= new double [Nx*Ny*Nz]; - ScaLBL_CopyToHost(&TMPDAT[0],&Den[0],SIZE); - ScaLBL_Comm.RegularLayout(Map,TMPDAT,DENA); - ScaLBL_CopyToHost(&TMPDAT[0],&Den[Np],SIZE); - ScaLBL_Comm.RegularLayout(Map,TMPDAT,DENB); - - FILE *AFILE; - sprintf(LocalRankFilename,"na.%05i.raw",rank); - AFILE = fopen(LocalRankFilename,"wb"); - fwrite(DENA,8,N,AFILE); - fclose(AFILE); - - FILE *BFILE; - sprintf(LocalRankFilename,"nb.%05i.raw",rank); - BFILE = fopen(LocalRankFilename,"wb"); - fwrite(DENB,8,N,BFILE); - fclose(BFILE); - - double *CG; - CG= new double [3*Np]; - ScaLBL_D3Q19_Gradient(dvcMap, Phi, ColorGrad, 0, Np, Np, Nx, Ny, Nz); - - ScaLBL_CopyToHost(&CG[0],&ColorGrad[0],3*SIZE); - for (int idx=0; idx Date: Mon, 21 May 2018 07:13:18 -0400 Subject: [PATCH 116/124] how neatly he spreads his claws --- tests/CMakeLists.txt | 2 +- tests/TestColorSquareTube.cpp | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e3b02bf8..2147265a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -48,7 +48,7 @@ ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db) ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) -ADD_LBPM_TEST( TestPoiseuille ../example/Piston/input.db) +ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db) ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db) ADD_LBPM_TEST( TestForceD3Q19 ) ADD_LBPM_TEST( TestMomentsD3Q19 ) diff --git a/tests/TestColorSquareTube.cpp b/tests/TestColorSquareTube.cpp index 2644b710..9807f0e8 100644 --- a/tests/TestColorSquareTube.cpp +++ b/tests/TestColorSquareTube.cpp @@ -8,7 +8,7 @@ #include #include "common/ScaLBL.h" #include "common/MPI_Helpers.h" - +#include "models/ColorModel.h" std::shared_ptr loadInputs( int nprocs ) { @@ -39,7 +39,18 @@ std::shared_ptr loadInputs( int nprocs ) return db; } -void InitSquareTube(ScaLBL_ColorModel &ColorModel){ +void InitializeSquareTube(ScaLBL_ColorModel &ColorModel){ + int i,j,k,n; + int rank = ColorModel.Mask->rank(); + int Nx = ColorModel.Mask->Nx; + int Ny = ColorModel.Mask->Ny; + int Nz = ColorModel.Mask->Nz; + int nprocx = ColorModel.Mask->rank_info.nx; + int nprocy = ColorModel.Mask->rank_info.ny; + int iproc = ColorModel.Mask->rank_info.ix; + int jproc = ColorModel.Mask->rank_info.jy; + int kproc = ColorModel.Mask->rank_info.kz; + for (k=0;k Date: Mon, 21 May 2018 10:13:23 -0400 Subject: [PATCH 117/124] and welcomes little fishes in --- models/ColorModel.cpp | 18 +- tests/TestColorBubble.cpp | 2 +- tests/lbpm_color_simulator.cpp | 930 +-------------------------------- 3 files changed, 19 insertions(+), 931 deletions(-) diff --git a/models/ColorModel.cpp b/models/ColorModel.cpp index 1d2fdd33..51543410 100644 --- a/models/ColorModel.cpp +++ b/models/ColorModel.cpp @@ -214,8 +214,8 @@ void ScaLBL_ColorModel::Create(){ int rank=Dm->rank(); //......................................................... // don't perform computations at the eight corners - id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; + //id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; + //id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; //......................................................... // Initialize communication structures in averaging domain @@ -386,7 +386,7 @@ void ScaLBL_ColorModel::Initialize(){ /* * This function initializes model */ - int rank=Dm->rank(); + int rank=Dm->rank(); double count_wet=0.f; double *PhaseLabel; PhaseLabel=new double [Nx*Ny*Nz]; @@ -411,11 +411,12 @@ void ScaLBL_ColorModel::Initialize(){ ScaLBL_CopyToDevice(Phi, PhaseLabel, Np*sizeof(double)); //........................................................................... - if (rank==0) printf ("Initializing distributions \n"); - ScaLBL_D3Q19_Init(fq, Np); - if (rank==0) printf ("Initializing phase field \n"); - ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->last_interior, Np); - + if (rank==0) printf ("Initializing distributions \n"); + ScaLBL_D3Q19_Init(fq, Np); + if (rank==0) printf ("Initializing phase field \n"); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np); + ScaLBL_DFH_Init(Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np); + } void ScaLBL_ColorModel::Run(){ @@ -529,7 +530,6 @@ void ScaLBL_ColorModel::Run(){ cputime = (stoptime - starttime)/timestep; // Performance obtained from each node double MLUPS = double(Np)/cputime/1000000; - if (rank==0) printf("********************************************************\n"); if (rank==0) printf("CPU time = %f \n", cputime); if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); diff --git a/tests/TestColorBubble.cpp b/tests/TestColorBubble.cpp index ffbd38d3..8025176b 100644 --- a/tests/TestColorBubble.cpp +++ b/tests/TestColorBubble.cpp @@ -52,7 +52,7 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius) } } } - + // initialize the phase indicator field } //*************************************************************************************** int main(int argc, char **argv) diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 896391ef..8d2a6e97 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -28,75 +28,6 @@ using namespace std; //************************************************************************* // Implementation of Two-Phase Immiscible LBM using CUDA //************************************************************************* -inline void PackID(int *list, int count, char *sendbuf, char *ID){ - // Fill in the phase ID values from neighboring processors - // This packs up the values that need to be sent from one processor to another - int idx,n; - - for (idx=0; idx> tau; // Viscosity parameter - input >> alpha; // Surface Tension parameter - input >> beta; // Width of the interface - input >> phi_s; // value of phi at the solid surface - // Line 2: wetting phase saturation to initialize - input >> wp_saturation; - // Line 3: External force components (Fx,Fy, Fz) - input >> Fx; - input >> Fy; - input >> Fz; - // Line 4: Pressure Boundary conditions - input >> InitialCondition; - input >> BoundaryCondition; - input >> din; - input >> dout; - // Line 5: time-stepping criteria - input >> timestepMax; // max no. of timesteps - input >> RESTART_INTERVAL; // restart interval - input >> tol; // error tolerance - // Line 6: Analysis options - input >> BLOB_ANALYSIS_INTERVAL; // interval to analyze blob states - //............................................................. - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Color.in is missing)! Default parameters will be used. \n"); - tau = 1.0; - alpha=0.005; - beta= 0.9; - Fx = Fy = Fz = 0.0; - InitialCondition=0; - BoundaryCondition=0; - din=dout=1.0; - timestepMax=0; - } - - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - if (input.is_open()){ - - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> nspheres; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } - else{ - // Set default values - // Print warning - printf("WARNING: No input file provided (Domain.in is missing)! Default parameters will be used. \n"); - nprocx=nprocy=nprocz=1; - Nx=Ny=Nz=10; - nspheres=0; - Lx=Ly=Lz=1.0; - } - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - MPI_Bcast(&tau,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&alpha,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&beta,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&das,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dbs,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&phi_s,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&wp_saturation,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&BoundaryCondition,1,MPI_INT,0,comm); - MPI_Bcast(&InitialCondition,1,MPI_INT,0,comm); - MPI_Bcast(&din,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&dout,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fy,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Fz,1,MPI_DOUBLE,0,comm); - MPI_Bcast(×tepMax,1,MPI_INT,0,comm); - MPI_Bcast(&RESTART_INTERVAL,1,MPI_INT,0,comm); - MPI_Bcast(&tol,1,MPI_DOUBLE,0,comm); - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&nspheres,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - - // Get the rank info - const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz); - MPI_Barrier(comm); - - // ************************************************************** - // ************************************************************** - double Ps = -(das-dbs)/(das+dbs); - double rlxA = 1.f/tau; - double rlxB = 8.f*(2.f-rlxA)/(8.f-rlxA); - //double xIntPos = log((1.0+phi_s)/(1.0-phi_s))/(2.0*beta); - - // Set the density values inside the solid based on the input value phi_s - das = (phi_s+1.0)*0.5; - dbs = 1.0 - das; - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - if (rank==0){ - printf("********************************************************\n"); - printf("tau = %f \n", tau); - printf("alpha = %f \n", alpha); - printf("beta = %f \n", beta); - printf("das = %f \n", das); - printf("dbs = %f \n", dbs); - printf("gamma_{wn} = %f \n", 5.796*alpha); - printf("Force(x) = %f \n", Fx); - printf("Force(y) = %f \n", Fy); - printf("Force(z) = %f \n", Fz); - printf("Sub-domain size = %i x %i x %i\n",Nx,Ny,Nz); - printf("Parallel domain size = %i x %i x %i\n",nprocx,nprocy,nprocz); - if (BoundaryCondition==0) printf("Periodic boundary conditions will applied \n"); - if (BoundaryCondition==1) printf("Pressure boundary conditions will be applied \n"); - if (BoundaryCondition==2) printf("Velocity boundary conditions will be applied \n"); - if (BoundaryCondition==3) printf("Dynamic pressure boundary conditions will be applied \n"); - if (InitialCondition==0) printf("Initial conditions assigned from phase ID file \n"); - if (InitialCondition==1) printf("Initial conditions assigned from restart file \n"); - printf("********************************************************\n"); - } - - // Initialized domain and averaging framework for Two-Phase Flow - bool pBC,velBC; - if (BoundaryCondition==1 || BoundaryCondition==3) - pBC=true; - else pBC=false; - if (BoundaryCondition==2) velBC=true; - else velBC=false; - - bool Restart; - if (InitialCondition==1) Restart=true; - else Restart=false; - NULL_USE(pBC); NULL_USE(velBC); - - // Full domain used for averaging (do not use mask for analysis) - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - for (i=0; i Averages( new TwoPhase(Dm) ); - Dm.CommInit(); - - // Mask that excludes the solid phase - Domain Mask(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - - MPI_Barrier(comm); - - Nx+=2; Ny+=2; Nz += 2; - //Nx = Ny = Nz; // Cubic domain - - int N = Nx*Ny*Nz; - int dist_mem_size = N*sizeof(double); - - //....................................................................... - if (rank == 0) printf("Read input media... \n"); - //....................................................................... - - //....................................................................... - // Filenames used - char LocalRankString[8]; - char LocalRankFilename[40]; - char LocalRestartFile[40]; - char tmpstr[10]; - sprintf(LocalRankString,"%05d",rank); - sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); - sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString); - -// printf("Local File Name = %s \n",LocalRankFilename); - // .......... READ THE INPUT FILE ....................................... -// char value; - char *id; - id = new char[N]; - int sum = 0; - double sum_local; - double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs); - if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6)); - double porosity, pore_vol; - //........................................................................... - if (rank == 0) cout << "Reading in domain from signed distance function..." << endl; - - //....................................................................... - sprintf(LocalRankString,"%05d",rank); -// sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString); -// WriteLocalSolidID(LocalRankFilename, id, N); - sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString); - ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N); - MPI_Barrier(comm); - if (rank == 0) cout << "Domain set." << endl; - - //....................................................................... - // Assign the phase ID field based on the signed distance - //....................................................................... - for (k=0;kSDs(n) > 0.0){ - id[n] = 2; - } - // compute the porosity (actual interface location used) - if (Averages->SDs(n) > 0.0){ - sum++; - } - } - } - } - - if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); - sprintf(LocalRankFilename,"ID.%05i",rank); - size_t readID; - FILE *IDFILE = fopen(LocalRankFilename,"rb"); - if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - readID=fread(id,1,N,IDFILE); - if (readID != size_t(N)) printf("lbpm_segmented_pp: Error reading ID (rank=%i) \n",rank); - - fclose(IDFILE); - - /* for ( k=0;k 0){ - if (kproc==0 && k==0) id[n]=0; - if (kproc==0 && k==1) id[n]=0; - if (kproc==nprocz-1 && k==Nz-2) id[n]=0; - if (kproc==nprocz-1 && k==Nz-1) id[n]=0; - } - } - } - } - */ - // Set up kstart, kfinish so that the reservoirs are excluded from averaging - int kstart,kfinish; - kstart = 1; - kfinish = Nz-1; - if (BoundaryCondition > 0 && Dm.kproc==0) kstart = 4; - if (BoundaryCondition > 0 && Dm.kproc==nprocz-1) kfinish = Nz-4; - - // Compute the pore volume - sum_local = 0.0; - for ( k=kstart;k 0){ - sum_local += 1.0; - } - } - } - } - MPI_Allreduce(&sum_local,&pore_vol,1,MPI_DOUBLE,MPI_SUM,comm); -// MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm); - porosity = pore_vol*iVol_global; - if (rank==0) printf("Media porosity = %f \n",porosity); - //......................................................... - // If external boundary conditions are applied remove solid - if (BoundaryCondition > 0 && Dm.kproc == 0){ - for (k=0; k<3; k++){ - for (j=0;jSDs(n) = max(Averages->SDs(n),1.0*(2.5-k)); - } - } - } - } - if (BoundaryCondition > 0 && Dm.kproc == nprocz-1){ - for (k=Nz-3; kSDs(n) = max(Averages->SDs(n),1.0*(k-Nz+2.5)); - } - } - } - } - //......................................................... - // don't perform computations at the eight corners - id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0; - id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0; - //......................................................... - - // Initialize communication structures in averaging domain - for (i=0; i 0){ - for ( k=0;kSDs.data(), dist_mem_size); - //........................................................................... - - int logcount = 0; // number of surface write-outs - - //........................................................................... - // MAIN VARIABLES INITIALIZED HERE - //........................................................................... - //........................................................................... - //........................................................................... - if (rank==0) printf("Setting the distributions, size = %i\n", N); - //........................................................................... - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_Init(ID, f_even, f_odd, Nx, Ny, Nz); - ScaLBL_Color_Init(ID, Den, Phi, das, dbs, Nx, Ny, Nz); - ScaLBL_DeviceBarrier(); - //...................................................................... - - if (Restart == true){ - - - if (rank==0){ - printf("Reading restart file! \n"); - ifstream restart("Restart.txt"); - if (restart.is_open()){ - restart >> timestep; - printf("Restarting from timestep =%i \n",timestep); - } - else{ - printf("WARNING:No Restart.txt file, setting timestep=0 \n"); - timestep=5; - } - } - MPI_Bcast(×tep,1,MPI_INT,0,comm); - - // Read in the restart file to CPU buffers - double *cDen = new double[2*N]; - double *cDistEven = new double[10*N]; - double *cDistOdd = new double[9*N]; - ReadCheckpoint(LocalRestartFile, cDen, cDistEven, cDistOdd, N); - // Copy the restart data to the GPU - ScaLBL_CopyToDevice(f_even,cDistEven,10*N*sizeof(double)); - ScaLBL_CopyToDevice(f_odd,cDistOdd,9*N*sizeof(double)); - ScaLBL_CopyToDevice(Den,cDen,2*N*sizeof(double)); - ScaLBL_DeviceBarrier(); - delete [] cDen; - delete [] cDistEven; - delete [] cDistOdd; - MPI_Barrier(comm); - } - - //...................................................................... - ScaLBL_D3Q7_Init(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ScaLBL_D3Q7_Init(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - //....................................................................... - // Once phase has been initialized, map solid to account for 'smeared' interface - //for (i=0; iSDs(i) -= (1.0); - // Make sure the id match for the two domains - for (i=0; iSetupCubes(Mask); - Averages->UpdateSolid(); - //....................................................................... - - //************************************************************************* - // Compute the phase indicator field and reset Copy, Den - //************************************************************************* - ScaLBL_ComputePhaseField(ID, Phi, Den, N); - //************************************************************************* - ScaLBL_DeviceBarrier(); - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_Comm.RecvHalo(Phi); - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - //************************************************************************* - - if (rank==0 && BoundaryCondition==1){ - printf("Setting inlet pressure = %f \n", din); - printf("Setting outlet pressure = %f \n", dout); - } - if (BoundaryCondition==1 && Mask.kproc == 0) { - ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - } - - if (BoundaryCondition==1 && Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - } - - if (rank==0 && BoundaryCondition==2){ - printf("Setting inlet velocity = %f \n", din); - printf("Setting outlet velocity = %f \n", dout); - } - if (BoundaryCondition==2 && Mask.kproc == 0) { - ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - } - - if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - } - // Set dynamic pressure boundary conditions - double dp, slope; - dp = slope = 0.0; - if (BoundaryCondition==3){ - slope = abs(dout-din)/timestepMax; - dp = abs(timestep)*slope; - if (rank==0) printf("Change in pressure / time =%.3e \n",slope); - // set the initial value - din = 1.0+0.5*dp; - dout = 1.0-0.5*dp; - // set the initial boundary conditions - if (Mask.kproc == 0) { - ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - } - if (Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - } - } - - ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - - //........................................................................... - // Copy the phase indicator field for the earlier timestep - ScaLBL_DeviceBarrier(); - ScaLBL_CopyToHost(Averages->Phase_tplus.data(),Phi,N*sizeof(double)); - //........................................................................... - //........................................................................... - // Copy the data for for the analysis timestep - //........................................................................... - // Copy the phase from the GPU -> CPU - //........................................................................... - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q19_Pressure(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); - ScaLBL_CopyToHost(Averages->Press.data(),Pressure,N*sizeof(double)); - ScaLBL_CopyToHost(Averages->Vel_x.data(),&Velocity[0],N*sizeof(double)); - ScaLBL_CopyToHost(Averages->Vel_y.data(),&Velocity[N],N*sizeof(double)); - ScaLBL_CopyToHost(Averages->Vel_z.data(),&Velocity[2*N],N*sizeof(double)); - //........................................................................... - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("No. of timesteps: %i \n", timestepMax); - - //.......create and start timer............ - double starttime,stoptime,cputime; - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - starttime = MPI_Wtime(); - //......................................... - - err = 1.0; - double sat_w_previous = 1.01; // slightly impossible value! - if (rank==0) printf("Begin timesteps: error tolerance is %f \n", tol); - // Create the thread pool - int N_threads = 4; - if ( provided_thread_support < MPI_THREAD_MULTIPLE ) - N_threads = 0; - if ( N_threads > 0 ) { - // Set the affinity - int N_procs = ThreadPool::getNumberOfProcessors(); - std::vector procs(N_procs); - for (int i=0; i fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1); - std::vector meshData(1); - meshData[0].meshName = "domain"; - meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) ); - std::shared_ptr PhaseVar( new IO::Variable() ); - std::shared_ptr PressVar( new IO::Variable() ); - std::shared_ptr SignDistVar( new IO::Variable() ); - std::shared_ptr BlobIDVar( new IO::Variable() ); - PhaseVar->name = "phase"; - PhaseVar->type = IO::VariableType::VolumeVariable; - PhaseVar->dim = 1; - PhaseVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(PhaseVar); - PressVar->name = "Pressure"; - PressVar->type = IO::VariableType::VolumeVariable; - PressVar->dim = 1; - PressVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(PressVar); - SignDistVar->name = "SignDist"; - SignDistVar->type = IO::VariableType::VolumeVariable; - SignDistVar->dim = 1; - SignDistVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(SignDistVar); - BlobIDVar->name = "BlobID"; - BlobIDVar->type = IO::VariableType::VolumeVariable; - BlobIDVar->dim = 1; - BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(BlobIDVar); - - //************ MAIN ITERATION LOOP ***************************************/ - PROFILE_START("Loop"); - BlobIDstruct last_ids, last_index; - BlobIDList last_id_map; - writeIDMap(ID_map_struct(),0,id_map_filename); - AnalysisWaitIdStruct work_ids; - while (timestep < timestepMax && err > tol ) { - //if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } - PROFILE_START("Update"); - - //************************************************************************* - // Fused Color Gradient and Collision - //************************************************************************* - ScaLBL_D3Q19_ColorCollide( ID,f_even,f_odd,Phi,ColorGrad, - Velocity,Nx,Ny,Nz,rlxA,rlxB,alpha,beta,Fx,Fy,Fz); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - //************************************************************************* - // Pack and send the D3Q19 distributions - ScaLBL_Comm.SendD3Q19(f_even, f_odd); - //************************************************************************* - - //************************************************************************* - // Carry out the density streaming step for mass transport - //************************************************************************* - ScaLBL_D3Q7_ColorCollideMass(ID, A_even, A_odd, B_even, B_odd, Den, Phi, - ColorGrad, Velocity, beta, N, pBC); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - //************************************************************************* - // Swap the distributions for momentum transport - //************************************************************************* - ScaLBL_D3Q19_Swap(ID, f_even, f_odd, Nx, Ny, Nz); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - //************************************************************************* - // Wait for communications to complete and unpack the distributions - ScaLBL_Comm.RecvD3Q19(f_even, f_odd); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - //************************************************************************* - // Pack and send the D3Q7 distributions - ScaLBL_Comm.BiSendD3Q7(A_even, A_odd, B_even, B_odd); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - ScaLBL_D3Q7_Swap(ID, A_even, A_odd, Nx, Ny, Nz); - ScaLBL_D3Q7_Swap(ID, B_even, B_odd, Nx, Ny, Nz); - - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - //************************************************************************* - // Wait for communication and unpack the D3Q7 distributions - ScaLBL_Comm.BiRecvD3Q7(A_even, A_odd, B_even, B_odd); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - //.................................................................................. - ScaLBL_D3Q7_Density(ID, A_even, A_odd, &Den[0], Nx, Ny, Nz); - ScaLBL_D3Q7_Density(ID, B_even, B_odd, &Den[N], Nx, Ny, Nz); - //************************************************************************* - // Compute the phase indicator field - //************************************************************************* - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - - ScaLBL_ComputePhaseField(ID, Phi, Den, N); - //************************************************************************* - ScaLBL_Comm.SendHalo(Phi); - ScaLBL_DeviceBarrier(); - ScaLBL_Comm.RecvHalo(Phi); - //************************************************************************* - - ScaLBL_DeviceBarrier(); - - // Pressure boundary conditions - if (BoundaryCondition==1 && Mask.kproc == 0) { - ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - } - if (BoundaryCondition==1 && Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - } - - // Velocity boundary conditions - if (BoundaryCondition==2 && Mask.kproc == 0) { - ScaLBL_D3Q19_Velocity_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,0); - } - if (BoundaryCondition==2 && Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Velocity_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,Nz-1); - } - - if (BoundaryCondition==3){ - // Increase the pressure difference - dp += slope; - din = 1.0+0.5*dp; - dout = 1.0-0.5*dp; - // set the initial boundary conditions - if (Mask.kproc == 0) { - ScaLBL_D3Q19_Pressure_BC_z(f_even,f_odd,din,Nx,Ny,Nz); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0); - } - if (Mask.kproc == nprocz-1){ - ScaLBL_D3Q19_Pressure_BC_Z(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); - ScaLBL_D3Q19_Velocity(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); - ScaLBL_Color_BC_Z(Phi,Den,Velocity,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); - ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); - } - } - - //................................................................................... - - MPI_Barrier(comm); - PROFILE_STOP("Update"); - - // Timestep completed! - timestep++; - - // Run the analysis, blob identification, and write restart files - // if (BLOB_ANALYSIS_INTERVAL > 0){ - //if (timestep > 5) - run_analysis(timestep,RESTART_INTERVAL,rank_info,*Averages,last_ids,last_index,last_id_map, - Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den, - LocalRestartFile,meshData,fillData,tpool,work_ids); - /* } - else{ - ComputeMacroscaleAverages(timestep,ANALYSIS_INTERVAL,RESTART_INTERVAL,rank_info,*Averages, - Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den, - LocalRestartFile,meshData,fillData,tpool,work_ids); - } - */ - // Save the timers - if ( timestep%50==0 ) - PROFILE_SAVE("lbpm_color_simulator",1); - } - tpool.wait_pool_finished(); - PROFILE_STOP("Loop"); - //************************************************************************ - ScaLBL_DeviceBarrier(); - MPI_Barrier(comm); - stoptime = MPI_Wtime(); - if (rank==0) printf("-------------------------------------------------------------------\n"); - // Compute the walltime per timestep - cputime = (stoptime - starttime)/timestep; - // Performance obtained from each node - double MLUPS = double(Nx*Ny*Nz)/cputime/1000000; - - if (rank==0) printf("********************************************************\n"); - if (rank==0) printf("CPU time = %f \n", cputime); - if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS); - MLUPS *= nprocs; - if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS); - if (rank==0) printf("********************************************************\n"); - - // ************************************************************************ -/* // Perform component averaging and write tcat averages - Averages->Initialize(); - Averages->ComponentAverages(); - Averages->SortBlobs(); - Averages->PrintComponents(timestep); - // ************************************************************************ - - int NumberComponents_NWP = ComputeGlobalPhaseComponent(Mask.Nx-2,Mask.Ny-2,Mask.Nz-2,Mask.rank_info,Averages->PhaseID,1,Averages->Label_NWP); - printf("Number of non-wetting phase components: %i \n ",NumberComponents_NWP); - ScaLBL_DeviceBarrier(); - ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); -*/ - -/* Averages->WriteSurfaces(0); - - sprintf(LocalRankFilename,"%s%s","Phase.",LocalRankString); - FILE *PHASE; - PHASE = fopen(LocalRankFilename,"wb"); - fwrite(Averages->SDn.data(),8,N,PHASE); - fclose(PHASE); - */ - - /* sprintf(LocalRankFilename,"%s%s","Pressure.",LocalRankString); - FILE *PRESS; - PRESS = fopen(LocalRankFilename,"wb"); - fwrite(Averages->Press.data(),8,N,PRESS); - fclose(PRESS); - - ScaLBL_CopyToHost(Averages->Phase.data(),Phi,N*sizeof(double)); - double * Grad; - Grad = new double [3*N]; - ScaLBL_CopyToHost(Grad,ColorGrad,3*N*sizeof(double)); - sprintf(LocalRankFilename,"%s%s","ColorGrad.",LocalRankString); - FILE *GRAD; - GRAD = fopen(LocalRankFilename,"wb"); - fwrite(Grad,8,3*N,GRAD); - fclose(GRAD); - */ PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); // **************************************************** From 4b693147a2936f1c0204817e5304f210b921747b Mon Sep 17 00:00:00 2001 From: James E McClure Date: Mon, 21 May 2018 10:15:07 -0400 Subject: [PATCH 118/124] with gently smiling jaws --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2147265a..d7510853 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ # Copy files for the tests #ADD_LBPM_EXECUTABLE( lbpm_nonnewtonian_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator ) -#ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) +ADD_LBPM_EXECUTABLE( lbpm_color_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator ) #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator ) From 5d6d8667af9a4b80f276e148972c34bf96e3127c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 22 May 2018 09:53:29 -0400 Subject: [PATCH 119/124] and welcomes little fishes in --- example/Bubble/input.db | 4 ++-- tests/lbpm_color_simulator.cpp | 17 +---------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/example/Bubble/input.db b/example/Bubble/input.db index 72a9aecb..b1a80cd2 100644 --- a/example/Bubble/input.db +++ b/example/Bubble/input.db @@ -1,6 +1,6 @@ Color { - tauA = 1.0; - tauB = 1.0; + tauA = 1.0; // relaxation time for fluid A + tauB = 1.0; // relaxation time for fluid B rhoA = 1.0; rhoB = 1.0; alpha = 1e-2; diff --git a/tests/lbpm_color_simulator.cpp b/tests/lbpm_color_simulator.cpp index 8d2a6e97..eb9d8f62 100644 --- a/tests/lbpm_color_simulator.cpp +++ b/tests/lbpm_color_simulator.cpp @@ -6,15 +6,7 @@ #include #include -#include "common/ScaLBL.h" -#include "common/Communication.h" -#include "analysis/TwoPhase.h" -#include "common/MPI_Helpers.h" - -#include "ProfilerApp.h" -#include "threadpool/thread_pool.h" - -#include "lbpm_color_simulator.h" +#include "models/ColorModel.h" //#define WRE_SURFACES @@ -42,18 +34,11 @@ int main(int argc, char **argv) std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl; { // Limit scope so variables that contain communicators will free before MPI_Finialize - // parallel domain size (# of sub-domains) - int nprocx,nprocy,nprocz; - - MPI_Request req1[18],req2[18]; - MPI_Status stat1[18],stat2[18]; - if (rank == 0){ printf("********************************************************\n"); printf("Running Color LBM \n"); printf("********************************************************\n"); } - PROFILE_ENABLE(1); //PROFILE_ENABLE_TRACE(); //PROFILE_ENABLE_MEMORY(); From 14e434fead2e214cf66dc8c5b7da644bd0b79cca Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 22 May 2018 10:38:38 -0400 Subject: [PATCH 120/124] with gently smiling jaws --- tests/GenerateSphereTest.cpp | 93 ++++++++++-------------------------- 1 file changed, 24 insertions(+), 69 deletions(-) diff --git a/tests/GenerateSphereTest.cpp b/tests/GenerateSphereTest.cpp index 48b4840f..fb17464d 100644 --- a/tests/GenerateSphereTest.cpp +++ b/tests/GenerateSphereTest.cpp @@ -328,13 +328,6 @@ int main(int argc, char **argv) int nprocx,nprocy,nprocz; int iproc,jproc,kproc; //***************************************** - // MPI ranks for all 18 neighbors - //********************************** - int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z; - int rank_xy,rank_XY,rank_xY,rank_Xy; - int rank_xz,rank_XZ,rank_xZ,rank_Xz; - int rank_yz,rank_YZ,rank_yZ,rank_Yz; - //********************************** if (rank == 0){ printf("********************************************************\n"); @@ -343,72 +336,36 @@ int main(int argc, char **argv) } // Variables that specify the computational domain - string FILENAME; + string filename; int Nx,Ny,Nz; // local sub-domain size int nspheres; // number of spheres in the packing double Lx,Ly,Lz; // Domain length double D = 1.0; // reference length for non-dimensionalization int i,j,k,n; + filename = argv[1]; + auto db = std::make_shared( filename ); + auto domain_db = db->getDatabase( "Domain" ); - if (argc > 1) - nspheres=atoi(argv[1]); - else nspheres=0; - - if (rank==0){ - //....................................................................... - // Reading the domain information file - //....................................................................... - ifstream domain("Domain.in"); - domain >> nprocx; - domain >> nprocy; - domain >> nprocz; - domain >> Nx; - domain >> Ny; - domain >> Nz; - domain >> Lx; - domain >> Ly; - domain >> Lz; - //....................................................................... - } - // ************************************************************** - // Broadcast simulation parameters from rank 0 to all other procs - MPI_Barrier(comm); - //................................................. - // Computational domain - MPI_Bcast(&Nx,1,MPI_INT,0,comm); - MPI_Bcast(&Ny,1,MPI_INT,0,comm); - MPI_Bcast(&Nz,1,MPI_INT,0,comm); - MPI_Bcast(&nprocx,1,MPI_INT,0,comm); - MPI_Bcast(&nprocy,1,MPI_INT,0,comm); - MPI_Bcast(&nprocz,1,MPI_INT,0,comm); - MPI_Bcast(&Lx,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Ly,1,MPI_DOUBLE,0,comm); - MPI_Bcast(&Lz,1,MPI_DOUBLE,0,comm); - //................................................. - MPI_Barrier(comm); - - // ************************************************************** - - if (nprocs != nprocx*nprocy*nprocz){ - printf("nprocx = %i \n",nprocx); - printf("nprocy = %i \n",nprocy); - printf("nprocz = %i \n",nprocz); - INSIST(nprocs == nprocx*nprocy*nprocz,"Fatal error in processor count!"); - } - - InitializeRanks( rank, nprocx, nprocy, nprocz, iproc, jproc, kproc, - rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, - rank_xy, rank_XY, rank_xY, rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, - rank_yz, rank_YZ, rank_yZ, rank_Yz ); - - MPI_Barrier(comm); + auto Dm = std::shared_ptr(new Domain(domain_db,comm)); // full domain for analysis + Nx = Dm->Nx; + Ny = Dm->Ny; + Nz = Dm->Nz; + Lx = Dm->Lx; + Ly = Dm->Ly; + Lz = Dm->Lz; + iproc = Dm->iproc(); + jproc = Dm->jproc(); + kproc = Dm->kproc(); + nprocx = Dm->nprocx(); + nprocy = Dm->nprocy(); + nprocz = Dm->nprocz(); + nspheres = domain_db->getScalar( "nspheres"); + printf("Set domain \n"); int BoundaryCondition=1; - Domain Dm(Nx,Ny,Nz,rank,nprocx,nprocy,nprocz,Lx,Ly,Lz,BoundaryCondition); - - Nz += 2; - Nx = Ny = Nz; // Cubic domain + //Nz += 2; + //Nx = Ny = Nz; // Cubic domain int N = Nx*Ny*Nz; // Define Dm.Communication sub-domain -- everywhere @@ -416,11 +373,11 @@ int main(int argc, char **argv) for (int j=0; jid[n] = 1; } } } - Dm.CommInit(); + Dm->CommInit(); if (rank==0) printf("Number of nodes per side = %i \n", Nx); if (rank==0) printf("Total Number of nodes = %i \n", N); @@ -490,8 +447,6 @@ int main(int argc, char **argv) SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz, iproc,jproc,kproc,nprocx,nprocy,nprocz); //....................................................................... - - // Assign the phase ID field based on the signed distance //....................................................................... for (k=0;k Date: Tue, 22 May 2018 10:43:07 -0400 Subject: [PATCH 121/124] GenerateSphereTest refactor --- example/Sph125/input.db | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/example/Sph125/input.db b/example/Sph125/input.db index 3e8edefa..6c37e1c4 100644 --- a/example/Sph125/input.db +++ b/example/Sph125/input.db @@ -1,9 +1,10 @@ Color { - tau = 1.0; - alpha = 1e-2; - beta = 0.95; - phi_s = 0.8; - wp_saturation = 0.7 + tauA = 1.0 + tauB = 1.0 + rhoA = 1.0 + rhoB = 1.0 + alpha = 1e-2 + beta = 0.95 F = 0, 0, 0 Restart = false pBC = 0 @@ -11,15 +12,16 @@ Color { dout = 1.0 timestepMax = 200 interval = 1000 - tol = 1e-5; + tol = 1e-5 das = 0.1 dbs = 0.9 + flux = 0.0 } Domain { nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) - n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) - n_spheres = 1 // Number of spheres + n = 40, 40, 40 // Size of local domain (Nx,Ny,Nz) + nspheres = 125 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) BC = 0 // Boundary condition type } @@ -28,7 +30,7 @@ Analysis { blobid_interval = 1000 // Frequency to perform blob identification analysis_interval = 1000 // Frequency to perform analysis restart_interval = 20000 // Frequency to write restart data - vis_interval = 20000 // Frequency to write visualization data + visualization_interval = 20000 // Frequency to write visualization data restart_file = "Restart" // Filename to use for restart file (will append rank) N_threads = 4 // Number of threads to use load_balance = "independent" // Load balance method to use: "none", "default", "independent" From fb6bf3c02ef8121c29c721384ab9cf270fe43505 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Tue, 22 May 2018 11:23:40 -0400 Subject: [PATCH 122/124] updated input database for Sph125 --- example/Sph125/input.db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/Sph125/input.db b/example/Sph125/input.db index 6c37e1c4..74f922a6 100644 --- a/example/Sph125/input.db +++ b/example/Sph125/input.db @@ -20,7 +20,7 @@ Color { Domain { nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) - n = 40, 40, 40 // Size of local domain (Nx,Ny,Nz) + n = 100, 100, 100 // Size of local domain (Nx,Ny,Nz) nspheres = 125 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) BC = 0 // Boundary condition type From 3990db0b795b1cfdeea01e2dec23d1222910d49d Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 24 May 2018 20:45:54 -0400 Subject: [PATCH 123/124] fixing unit tests --- example/Bubble/input.db | 2 +- tests/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/example/Bubble/input.db b/example/Bubble/input.db index b1a80cd2..dab2ca5d 100644 --- a/example/Bubble/input.db +++ b/example/Bubble/input.db @@ -19,7 +19,7 @@ Color { Domain { nproc = 1, 1, 1 // Number of processors (Npx,Npy,Npz) - n = 16, 16, 16 // Size of local domain (Nx,Ny,Nz) + n = 80, 80, 80 // Size of local domain (Nx,Ny,Nz) n_spheres = 1 // Number of spheres L = 1, 1, 1 // Length of domain (x,y,z) BC = 0 // Boundary condition type diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d7510853..53e89702 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,7 +46,7 @@ ADD_LBPM_TEST( TestMap ) #ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestColorGradDFH ) ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db) -ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db) +#ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db) ADD_LBPM_TEST( TestPressVel ../example/Piston/input.db) ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db) ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db) From 0d528a465c3979c0797241fd1bbbc1c700e84122 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Thu, 24 May 2018 21:10:08 -0400 Subject: [PATCH 124/124] fixing unit tests --- example/Bubble/input.db | 1 + tests/CMakeLists.txt | 3 +-- tests/TestInterfaceSpeed.cpp | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/example/Bubble/input.db b/example/Bubble/input.db index dab2ca5d..8a88ec54 100644 --- a/example/Bubble/input.db +++ b/example/Bubble/input.db @@ -10,6 +10,7 @@ Color { pBC = 0 din = 1.0 dout = 1.0 + flux = 1.0e-3 timestepMax = 200 interval = 1000 tol = 1e-5; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 53e89702..e56742e2 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -52,8 +52,7 @@ ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db) ADD_LBPM_TEST( TestForceMoments ../example/Piston/input.db) ADD_LBPM_TEST( TestForceD3Q19 ) ADD_LBPM_TEST( TestMomentsD3Q19 ) - -ADD_LBPM_TEST( TestInterfaceSpeed ../example/Piston/input.db) +ADD_LBPM_TEST( TestInterfaceSpeed ../example/Bubble/input.db) ADD_LBPM_TEST( TestSphereCurvature ) #ADD_LBPM_TEST_1_2_4( TestTwoPhase ) ADD_LBPM_TEST_1_2_4( TestBlobIdentify ) diff --git a/tests/TestInterfaceSpeed.cpp b/tests/TestInterfaceSpeed.cpp index aa7ca139..94b3c47a 100644 --- a/tests/TestInterfaceSpeed.cpp +++ b/tests/TestInterfaceSpeed.cpp @@ -38,7 +38,9 @@ int main (int argc, char *argv[]) std::shared_ptr Dm(new Domain(domain_db,comm)); - for (i=0; iNx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1; + Nx+=2; Ny+=2; Nz+=2; + + for (i=0; iid[i] = 1; Dm->CommInit(); @@ -137,11 +139,11 @@ int main (int argc, char *argv[]) toReturn = 5; printf("TestCylinderArea.cpp: error tolerance exceeded for common curve length \n"); } - if ( fabs(Averages->vawn_global(2)+0.2) > 0.01){ + if ( fabs(Averages->vawn_global(2)+0.25) > 0.01){ printf("TestInterfaceSpeed: Error too high for kinematic velocity of wn interface \n"); toReturn = 6; } - if ( fabs(Averages->vawns_global(2)+0.2) > 0.01){ + if ( fabs(Averages->vawns_global(2)+0.25) > 0.01){ printf("TestInterfaceSpeed: Error too high for kinematic velocity of common curve \n"); toReturn = 7; }