diff --git a/common/Array.hpp b/common/Array.hpp index fe85b41c..894382c4 100644 --- a/common/Array.hpp +++ b/common/Array.hpp @@ -141,7 +141,7 @@ void Array::resize( const std::vector& N ) // Store the old data const size_t ndim_max = sizeof(d_N)/sizeof(size_t); std::vector N1(ndim_max,1), N2(ndim_max,1); - for (size_t d=0; d void fillHalo::copy( const Array& src, Array& dst ) { PROFILE_START("fillHalo::copy",1); - ASSERT(src.size(0)==nx||src.size(0)==nx+2*ngx); - ASSERT(dst.size(0)==nx||dst.size(0)==nx+2*ngx); - bool src_halo = src.size(0)==nx+2*ngx; - bool dst_halo = dst.size(0)==nx+2*ngx; + ASSERT( (int)src.size(0)==nx || (int)src.size(0)==nx+2*ngx ); + ASSERT( (int)dst.size(0)==nx || (int)dst.size(0)==nx+2*ngx ); + bool src_halo = (int)src.size(0)==nx+2*ngx; + bool dst_halo = (int)dst.size(0)==nx+2*ngx; if ( src_halo ) { - ASSERT(src.size(0)==nx+2*ngx); - ASSERT(src.size(1)==ny+2*ngy); - ASSERT(src.size(2)==nz+2*ngz); + ASSERT((int)src.size(0)==nx+2*ngx); + ASSERT((int)src.size(1)==ny+2*ngy); + ASSERT((int)src.size(2)==nz+2*ngz); } else { - ASSERT(src.size(0)==nx); - ASSERT(src.size(1)==ny); - ASSERT(src.size(2)==nz); + ASSERT((int)src.size(0)==nx); + ASSERT((int)src.size(1)==ny); + ASSERT((int)src.size(2)==nz); } if ( dst_halo ) { - ASSERT(dst.size(0)==nx+2*ngx); - ASSERT(dst.size(1)==ny+2*ngy); - ASSERT(dst.size(2)==nz+2*ngz); + ASSERT((int)dst.size(0)==nx+2*ngx); + ASSERT((int)dst.size(1)==ny+2*ngy); + ASSERT((int)dst.size(2)==nz+2*ngz); } else { - ASSERT(dst.size(0)==nx); - ASSERT(dst.size(1)==ny); - ASSERT(dst.size(2)==nz); + ASSERT((int)dst.size(0)==nx); + ASSERT((int)dst.size(1)==ny); + ASSERT((int)dst.size(2)==nz); } if ( src_halo == dst_halo ) { // Src and dst halos match @@ -235,18 +235,18 @@ void fillHalo::copy( const Array& src, Array& dst ) dst(i) = src(i); } else if ( src_halo && !dst_halo ) { // Src has halos - for (size_t k=0; k& rhs, char *buffer ) size_t size = rhs.size(); memcpy(buffer,&size,sizeof(size_t)); size_t pos = sizeof(size_t); - for (int i=0; i& data, const char *buffer ) data.clear(); data.resize(size); size_t pos = sizeof(size_t); - for (int i=0; i 0.999 ) DistData[n] = 4.0; - else if (value < -0.999 ) DistData[n] = -4.0; - else DistData[n] = factor*log((1.0+value)/(1.0-value)); - if (DistData[n] > 1.0) DistData[n] = 1.0; - if (DistData[n] < -1.0) DistData[n] = -1.0; - } // Initialize to -1,1 (segmentation) for (int k=0; k 1.0) DistData(i,j,k) = 1.0; - else if (temp < -1.0) DistData(i,j,k) = -1.0; + temp = factor*log((1.0+value)/(1.0-value)); + if (value > 0.8) DistData(i,j,k) = 2.94*factor; + else if (value < -0.8) DistData(i,j,k) = -2.94*factor; else DistData(i,j,k) = temp; + // Basic threshold + //if (value > 0) DistData(i,j,k) = 1.0; + //else DistData(i,j,k) = -1.0; } } } - SSO(DistData,Dm.id,Dm,10); -*/ - for (int k=0; k 0){ @@ -446,6 +454,7 @@ void TwoPhase::ComputeLocal() pmmc_CommonCurveSpeed(CubeValues, dPdt, vawns, SDn_x, SDn_y, SDn_z,SDs_x,SDs_y,SDs_z, local_nws_pts,i,j,k,n_local_nws_pts); + pmmc_CurveCurvature(SDn, SDs, SDn_x, SDn_y, SDn_z, SDs_x, SDs_y, SDs_z, KNwns_values, KGwns_values, KNwns, KGwns, nws_pts, n_nws_pts, i, j, k); @@ -1070,6 +1079,7 @@ void TwoPhase::Reduce() MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); + MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); // Phase averages MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm.Comm); @@ -1115,6 +1125,7 @@ void TwoPhase::Reduce() if (awn_global > 0.0){ Jwn_global /= awn_global; Kwn_global /= awn_global; + wwndnw_global /= awn_global; for (i=0; i<3; i++) vawn_global(i) /= awn_global; for (i=0; i<6; i++) Gwn_global(i) /= awn_global; } @@ -1174,7 +1185,7 @@ void TwoPhase::PrintAll(int timestep) Gns_global(0),Gns_global(1),Gns_global(2),Gns_global(3),Gns_global(4),Gns_global(5)); // orientation of ns interface fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g ", Gws_global(0),Gws_global(1),Gws_global(2),Gws_global(3),Gws_global(4),Gws_global(5)); // orientation of ws interface - fprintf(TIMELOG,"%.5g %.5g %.5g ",trawn_global, trJwn_global, trRwn_global); // Trimmed curvature + fprintf(TIMELOG,"%.5g %.5g %.5g %.5g ",trawn_global, trJwn_global, trRwn_global, wwndnw_global); // Trimmed curvature fprintf(TIMELOG,"%.5g %.5g %.5g %.5g\n",euler_global, Kn_global, Jn_global, An_global); // minkowski measures fflush(TIMELOG); } diff --git a/common/TwoPhase.h b/common/TwoPhase.h index 82a1c264..898c0d21 100644 --- a/common/TwoPhase.h +++ b/common/TwoPhase.h @@ -95,6 +95,7 @@ public: double nwp_volume_global; // volume for the non-wetting phase double wp_volume_global; // volume for the wetting phase double As_global; + double wwndnw, wwndnw_global; double dEs,dAwn,dAns; // Global surface energy (calculated by rank=0) DoubleArray van; DoubleArray vaw; diff --git a/common/Utilities.cpp b/common/Utilities.cpp index 55369141..3e8eca8b 100644 --- a/common/Utilities.cpp +++ b/common/Utilities.cpp @@ -236,9 +236,9 @@ size_t Utilities::getMemoryUsage() size_t N_bytes = 0; #if defined(USE_LINUX) struct mallinfo meminfo = mallinfo(); - size_t size_hblkhd = static_cast( meminfo.hblkhd ); - size_t size_uordblks = static_cast( meminfo.uordblks ); - N_bytes = static_cast( size_hblkhd + size_uordblks ); + size_t size_hblkhd = static_cast( meminfo.hblkhd ); + size_t size_uordblks = static_cast( meminfo.uordblks ); + N_bytes = size_hblkhd + size_uordblks; #elif defined(USE_MAC) struct task_basic_info t_info; mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; diff --git a/common/pmmc.h b/common/pmmc.h index 1b0720c3..73144127 100644 --- a/common/pmmc.h +++ b/common/pmmc.h @@ -4317,12 +4317,12 @@ inline void pmmc_CurveCurvature(DoubleArray &f, DoubleArray &s, //-------------------------------------------------------------------------------------------------------- inline void pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArray &P_y, DoubleArray &P_z, DoubleArray &CubeValues, DTMutableList &Points, IntArray &Triangles, - DoubleArray &SurfaceVector, DoubleArray &SurfaceValues, DoubleArray &AvgVel, + DoubleArray &SurfaceVector, DoubleArray &AvgSpeed, DoubleArray &AvgVel, int i, int j, int k, int npts, int ntris) { Point A,B,C,P; double x,y,z; - double s,s1,s2,s3,temp; + double s,s1,s2,s3,area; double norm, zeta; TriLinPoly Px,Py,Pz,Pt; @@ -4342,23 +4342,27 @@ inline void pmmc_InterfaceSpeed(DoubleArray &dPdt, DoubleArray &P_x, DoubleArray s2 = sqrt((A.x-C.x)*(A.x-C.x)+(A.y-C.y)*(A.y-C.y)+(A.z-C.z)*(A.z-C.z)); s3 = sqrt((B.x-C.x)*(B.x-C.x)+(B.y-C.y)*(B.y-C.y)+(B.z-C.z)*(B.z-C.z)); s = 0.5*(s1+s2+s3); - temp = s*(s-s1)*(s-s2)*(s-s3); + area = sqrt(s*(s-s1)*(s-s2)*(s-s3)); // Compute the centroid P P.x = 0.33333333333333333*(A.x+B.x+C.x); P.y = 0.33333333333333333*(A.y+B.y+C.y); P.z = 0.33333333333333333*(A.z+B.z+C.z); - if (temp > 0.0){ + if (area > 0.0){ x = Px.eval(P); y = Py.eval(P); z = Pz.eval(P); norm = sqrt(x*x+y*y+z*z); if (norm==0.0) norm=1.0; + // Compute the interface speed from time derivative and gradient (Level Set Equation) zeta = -Pt.eval(P) / norm; - temp = sqrt(temp)/norm; - AvgVel(0) += temp*zeta*x; - AvgVel(1) += temp*zeta*y; - AvgVel(2) += temp*zeta*z; + //temp = sqrt(temp)/norm; <--- what was I thinking with this? (James) + + // Compute the average + AvgVel(0) += area*zeta*x; + AvgVel(1) += area*zeta*y; + AvgVel(2) += area*zeta*z; + AvgSpeed(r) = zeta*area; } } //............................................................................. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4a4459a5..1560278a 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -12,6 +12,7 @@ ADD_LBPM_EXECUTABLE( lbpm_BlobAnalysis ) ADD_LBPM_EXECUTABLE( TestBubble ) ADD_LBPM_EXECUTABLE( BasicSimulator ) ADD_LBPM_EXECUTABLE( ComponentLabel ) +ADD_LBPM_EXECUTABLE( ColorToBinary ) ADD_LBPM_EXECUTABLE( BlobAnalysis ) ADD_LBPM_EXECUTABLE( BlobIdentify ) ADD_LBPM_EXECUTABLE( BlobIdentifyParallel ) diff --git a/tests/ColorToBinary.cpp b/tests/ColorToBinary.cpp new file mode 100644 index 00000000..9d21578d --- /dev/null +++ b/tests/ColorToBinary.cpp @@ -0,0 +1,261 @@ +// Sequential component labeling for two phase systems +// Reads parallel simulation data and performs connectivity analysis +// and averaging on a blob-by-blob basis +// James E. McClure 2015 + +#include +#include +#include "analysis/analysis.h" +#include "common/TwoPhase.h" + + +using namespace std; + + +inline void ReadFromRank(char *FILENAME, DoubleArray &Phase, int nx, int ny, int nz, int iproc, int jproc, int kproc) +{ + int i,j,k,q,n,N; + int iglobal,jglobal,kglobal; + double value; + double denA,denB; + double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9; + double f10,f11,f12,f13,f14,f15,f16,f17,f18; + double vx,vy,vz; + + N = nx*ny*nz; + + double *Den, *DistEven, *DistOdd; + + Den = new double[2*N]; + DistEven = new double[10*N]; + DistOdd = new double[9*N]; + + ifstream File(FILENAME,ios::binary); + for (n=0; n> nprocx; + domain >> nprocy; + domain >> nprocz; + domain >> nx; + domain >> ny; + domain >> nz; + domain >> nspheres; + domain >> Lx; + domain >> Ly; + domain >> Lz; + //....................................................................... + + nx+=2; + ny+=2; + nz+=2; + + nprocs = nprocx*nprocy*nprocz; + printf("Number of MPI ranks: %i \n", nprocs); + int BoundaryCondition=0; + Nx = (nx-2)*nprocx; + Ny = (ny-2)*nprocy; + Nz = (nz-2)*nprocz; + Domain Dm(Nx,Ny,Nz,rank,1,1,1,Lx,Ly,Lz,BoundaryCondition); + Nx+=2; Ny+=2; Nz+=2; + printf("Full domain size: %i x %i x %i \n", Nx,Ny,Nz); + + DoubleArray Phase(Nx,Ny,Nz); + DoubleArray SignDist(Nx,Ny,Nz); + + // Filenames used + char LocalRankString[8]; + char LocalRankFilename[40]; + char BaseFilename[20]; + + int proc,iglobal,kglobal,jglobal; + + double * Temp; + Temp = new double[nx*ny*nz]; + + // read the files and populate main arrays + for ( kproc=0; kprocSDs(i) -= (1.0); // + // for (i=0; iSDs(i) -= (1.0); // //....................................................................... // Finalize setup for averaging domain //Averages->SetupCubes(Dm); @@ -624,6 +627,26 @@ int main(int argc, char **argv) SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } + // Set dynamic pressure boundary conditions + double dp, slope; + if (BoundaryCondition==3){ + slope = (dout-din)/timestepMax; + dp = din; + if (rank==0) printf("Change in pressure / time =%f \n",slope); + // set the initial value + din = 1.0+0.5*dp; + dout = 1.0-0.5*dp; + // set the initial boundary conditions + if (Dm.kproc == 0) { + PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + } + if (Dm.kproc == nprocz-1){ + PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + } + } + ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); ComputeVelocityD3Q19(ID,f_even,f_odd,Velocity,Nx,Ny,Nz); @@ -684,6 +707,37 @@ int main(int argc, char **argv) ThreadPool::setProcessAffinity(procs); } ThreadPool tpool(N_threads); + + // Create the MeshDataStruct + fillHalo fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1); + std::vector meshData(1); + meshData[0].meshName = "domain"; + meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) ); + std::shared_ptr PhaseVar( new IO::Variable() ); + std::shared_ptr PressVar( new IO::Variable() ); + std::shared_ptr SignDistVar( new IO::Variable() ); + std::shared_ptr BlobIDVar( new IO::Variable() ); + PhaseVar->name = "phase"; + PhaseVar->type = IO::VolumeVariable; + PhaseVar->dim = 1; + PhaseVar->data.resize(Nx-2,Ny-2,Nz-2); + meshData[0].vars.push_back(PhaseVar); + PressVar->name = "Pressure"; + PressVar->type = IO::VolumeVariable; + PressVar->dim = 1; + PressVar->data.resize(Nx-2,Ny-2,Nz-2); + meshData[0].vars.push_back(PressVar); + SignDistVar->name = "SignDist"; + SignDistVar->type = IO::VolumeVariable; + SignDistVar->dim = 1; + SignDistVar->data.resize(Nx-2,Ny-2,Nz-2); + meshData[0].vars.push_back(SignDistVar); + BlobIDVar->name = "BlobID"; + BlobIDVar->type = IO::VolumeVariable; + BlobIDVar->dim = 1; + BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2); + meshData[0].vars.push_back(BlobIDVar); + //************ MAIN ITERATION LOOP ***************************************/ PROFILE_START("Loop"); int timestep = -1; @@ -692,6 +746,7 @@ int main(int argc, char **argv) writeIDMap(ID_map_struct(),0,id_map_filename); AnalysisWaitIdStruct work_ids; while (timestep < timestepMax && err > tol ) { + if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); } PROFILE_START("Update"); //************************************************************************* @@ -787,6 +842,23 @@ int main(int argc, char **argv) //ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); SetPhiSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1); } + + if (BoundaryCondition==3){ + // Increase the pressure difference + dp += slope; + din = 1.0+0.5*dp; + dout = 1.0-0.5*dp; + // set the initial boundary conditions + if (Dm.kproc == 0) { + PressureBC_inlet(f_even,f_odd,din,Nx,Ny,Nz); + ColorBC_inlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + } + if (Dm.kproc == nprocz-1){ + PressureBC_outlet(f_even,f_odd,dout,Nx,Ny,Nz,Nx*Ny*(Nz-2)); + ColorBC_outlet(Phi,Den,A_even,A_odd,B_even,B_odd,Nx,Ny,Nz); + } + } + //................................................................................... MPI_Barrier(comm); @@ -794,12 +866,15 @@ int main(int argc, char **argv) // Timestep completed! timestep++; - + // Run the analysis, blob identification, and write restart files run_analysis(timestep,RESTART_INTERVAL,rank_info,*Averages,last_ids,last_index,last_id_map, Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den, - LocalRestartFile,tpool,work_ids); + LocalRestartFile,meshData,fillData,tpool,work_ids); + // Save the timers + if ( timestep%50==0 ) + PROFILE_SAVE("lbpm_color_simulator",1); } tpool.wait_pool_finished(); PROFILE_STOP("Loop"); @@ -833,40 +908,6 @@ int main(int argc, char **argv) DeviceBarrier(); CopyToHost(Averages->Phase.get(),Phi,N*sizeof(double)); */ - // Create the MeshDataStruct - fillHalo fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1); - std::vector meshData(1); - meshData[0].meshName = "domain"; - meshData[0].mesh = std::shared_ptr( new IO::DomainMesh(Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz) ); - std::shared_ptr PhaseVar( new IO::Variable() ); - std::shared_ptr PressVar( new IO::Variable() ); - std::shared_ptr SignDistVar( new IO::Variable() ); - std::shared_ptr BlobIDVar( new IO::Variable() ); - PhaseVar->name = "phase"; - PhaseVar->type = IO::VolumeVariable; - PhaseVar->dim = 1; - PhaseVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(PhaseVar); - PressVar->name = "Pressure"; - PressVar->type = IO::VolumeVariable; - PressVar->dim = 1; - PressVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(PressVar); - SignDistVar->name = "SignDist"; - SignDistVar->type = IO::VolumeVariable; - SignDistVar->dim = 1; - SignDistVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(SignDistVar); - BlobIDVar->name = "BlobID"; - BlobIDVar->type = IO::VolumeVariable; - BlobIDVar->dim = 1; - BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2); - meshData[0].vars.push_back(BlobIDVar); - - fillData.copy(Averages->SDn,PhaseVar->data); - fillData.copy(Averages->SDs,SignDistVar->data); - fillData.copy(Averages->Label_NWP,BlobIDVar->data); - IO::writeData( 0, meshData, 2, comm ); /* Averages->WriteSurfaces(0); diff --git a/tests/lbpm_color_simulator.h b/tests/lbpm_color_simulator.h index 91d24ee2..43351c4b 100644 --- a/tests/lbpm_color_simulator.h +++ b/tests/lbpm_color_simulator.h @@ -1,17 +1,22 @@ // Run the analysis, blob identification, and write restart files #include "common/Array.h" +#include "common/Communication.h" +#include "common/MPI_Helpers.h" +#include "IO/MeshDatabase.h" +//#define ANALYSIS_INTERVAL 6 #define ANALYSIS_INTERVAL 1000 -#define BLOBID_INTERVAL 1000 +#define BLOBID_INTERVAL 250 enum AnalysisType{ AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02, - CopyAverages=0x04, CalcDist=0x08, CreateRestart=0x10 }; + CopyAverages=0x04, CalcDist=0x08, CreateRestart=0x10, WriteVis=0x20 }; // Structure used to store ids struct AnalysisWaitIdStruct { ThreadPool::thread_id_t blobID; ThreadPool::thread_id_t analysis; + ThreadPool::thread_id_t vis; ThreadPool::thread_id_t restart; }; @@ -28,7 +33,6 @@ public: PROFILE_START("Save Checkpoint",1); WriteCheckpoint(filename,cDen.get(),cDistEven.get(),cDistOdd.get(),N); PROFILE_STOP("Save Checkpoint",1); - PROFILE_SAVE("lbpm_color_simulator",1); ThreadPool::WorkItem::d_state = 2; // Change state to finished }; private: @@ -123,6 +127,44 @@ private: }; +// Helper class to write the vis file from a thread +class WriteVisWorkItem: public ThreadPool::WorkItem +{ +public: + WriteVisWorkItem( int timestep_, std::vector& visData_, + TwoPhase& Avgerages_, fillHalo& fillData_ ): + timestep(timestep_), visData(visData_), Averages(Avgerages_), fillData(fillData_) {} + virtual void run() { + ThreadPool::WorkItem::d_state = 1; // Change state to in progress + PROFILE_START("Save Vis",1); + ASSERT(visData[0].vars[0]->name=="phase"); + ASSERT(visData[0].vars[1]->name=="Pressure"); + ASSERT(visData[0].vars[2]->name=="SignDist"); + ASSERT(visData[0].vars[3]->name=="BlobID"); + Array& PhaseData = visData[0].vars[0]->data; + Array& PressData = visData[0].vars[1]->data; + Array& SignData = visData[0].vars[2]->data; + Array& BlobData = visData[0].vars[3]->data; + fillData.copy(Averages.SDn,PhaseData); + fillData.copy(Averages.Press,PressData); + fillData.copy(Averages.SDs,SignData); + fillData.copy(Averages.Label_NWP,BlobData); + MPI_Comm newcomm; + MPI_Comm_dup(MPI_COMM_WORLD,&newcomm); + IO::writeData( timestep, visData, 2, newcomm ); + MPI_Comm_free(&newcomm); + PROFILE_STOP("Save Vis",1); + ThreadPool::WorkItem::d_state = 2; // Change state to finished + }; +private: + WriteVisWorkItem(); + int timestep; + std::vector& visData; + TwoPhase& Averages; + fillHalo& fillData; +}; + + // Helper class to run the analysis from within a thread // Note: Averages will be modified after the constructor is called class AnalysisWorkItem: public ThreadPool::WorkItem @@ -170,6 +212,8 @@ private: double beta; }; + + // Function to start the analysis void run_analysis( int timestep, int restart_interval, const RankInfoStruct& rank_info, TwoPhase& Averages, @@ -177,7 +221,8 @@ void run_analysis( int timestep, int restart_interval, int Nx, int Ny, int Nz, bool pBC, double beta, double err, const double *Phi, double *Pressure, const double *Velocity, const char *ID, const double *f_even, const double *f_odd, const double *Den, - const char *LocalRestartFile, ThreadPool& tpool, AnalysisWaitIdStruct& wait ) + const char *LocalRestartFile, std::vector& visData, fillHalo& fillData, + ThreadPool& tpool, AnalysisWaitIdStruct& wait ) { int N = Nx*Ny*Nz; @@ -191,7 +236,7 @@ void run_analysis( int timestep, int restart_interval, // Identify blobs and update global ids in time type = static_cast( type | IdentifyBlobs ); } -/* #ifdef USE_CUDA + #ifdef USE_CUDA if ( tpool.getQueueSize()<=3 && tpool.getNumThreads()>0 && timestep%50==0 ) { // Keep a few blob identifications queued up to keep the processors busy, // allowing us to track the blobs as fast as possible @@ -199,7 +244,7 @@ void run_analysis( int timestep, int restart_interval, type = static_cast( type | IdentifyBlobs ); } #endif - */ + if ( timestep%ANALYSIS_INTERVAL == 0 ) { // Copy the averages to the CPU (and identify blobs) type = static_cast( type | CopyAverages ); @@ -213,6 +258,12 @@ void run_analysis( int timestep, int restart_interval, // Write the restart file type = static_cast( type | CreateRestart ); } + if (timestep%restart_interval == 0) { + // Write the visualization data + type = static_cast( type | WriteVis ); + type = static_cast( type | CopyAverages ); + type = static_cast( type | IdentifyBlobs ); + } // Return if we are not doing anything if ( type == AnalyzeNone ) @@ -238,14 +289,22 @@ void run_analysis( int timestep, int restart_interval, } if ( (type&CopyAverages) != 0 ) { // Copy the members of Averages to the cpu (phase was copied above) + // Wait + PROFILE_START("Copy-Pressure",1); ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz); - memcpy(Averages.Phase.get(),phase->get(),N*sizeof(double)); DeviceBarrier(); + PROFILE_STOP("Copy-Pressure",1); + PROFILE_START("Copy-Wait",1); + tpool.wait(wait.analysis); + tpool.wait(wait.vis); // Make sure we are done using analysis before modifying + PROFILE_STOP("Copy-Wait",1); + PROFILE_START("Copy-Averages",1); + memcpy(Averages.Phase.get(),phase->get(),N*sizeof(double)); CopyToHost(Averages.Press.get(),Pressure,N*sizeof(double)); CopyToHost(Averages.Vel_x.get(),&Velocity[0],N*sizeof(double)); CopyToHost(Averages.Vel_y.get(),&Velocity[N],N*sizeof(double)); CopyToHost(Averages.Vel_z.get(),&Velocity[2*N],N*sizeof(double)); - + PROFILE_STOP("Copy-Averages",1); } std::shared_ptr cDen, cDistEven, cDistOdd; if ( (type&CreateRestart) != 0 ) { @@ -282,6 +341,7 @@ void run_analysis( int timestep, int restart_interval, type,timestep,Averages,last_index,last_id_map,beta); work->add_dependency(wait.blobID); work->add_dependency(wait.analysis); + work->add_dependency(wait.vis); // Make sure we are done using analysis before modifying wait.analysis = tpool.add_work(work); } @@ -295,12 +355,27 @@ void run_analysis( int timestep, int restart_interval, } else { // Not clear yet } + // Wait for previous restart files to finish writing (not necessary, but helps to ensure memory usage is limited) + tpool.wait(wait.restart); // Write the restart file (using a seperate thread) WriteRestartWorkItem *work = new WriteRestartWorkItem(LocalRestartFile,cDen,cDistEven,cDistOdd,N); work->add_dependency(wait.restart); wait.restart = tpool.add_work(work); } + + // Save the results for visualization + if ( (type&CreateRestart) != 0 ) { + // Wait for previous restart files to finish writing (not necessary, but helps to ensure memory usage is limited) + tpool.wait(wait.vis); + // Write the vis files + ThreadPool::WorkItem *work = new WriteVisWorkItem( timestep, visData, Averages, fillData ); + work->add_dependency(wait.blobID); + work->add_dependency(wait.analysis); + work->add_dependency(wait.vis); + wait.vis = tpool.add_work(work); + } PROFILE_STOP("start_analysis"); } +