Reverging to old lbpm_color_Simulator.h

sample_scripts/config_build_titan

@@ -18,7 +18,7 @@ cmake                                    \
     -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=aprun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-    -D CMAKE_BUILD_TYPE:STRING=Debug     \
+    -D CMAKE_BUILD_TYPE:STRING=Release     \
     -D CUDA_FLAGS="-arch sm_35"          \
     -D CUDA_HOST_COMPILER="/usr/bin/gcc" \
     -D USE_CUDA=1                        \

tests/lbpm_color_simulator.cpp

@@ -156,7 +156,7 @@ int main(int argc, char **argv)
 	//solid_isovalue = 0.0;
 	
 	int RESTART_INTERVAL=20000;
-	int ANALYSIS_INTERVAL=1000;	
+	//int ANALYSIS_INTERVAL=1000;	
 	int BLOB_ANALYSIS_INTERVAL=1000;
 
 	if (rank==0){
@@ -889,16 +889,17 @@ int main(int argc, char **argv)
 		timestep++;
   
         // Run the analysis, blob identification, and write restart files
-		if (BLOB_ANALYSIS_INTERVAL > 0){
+	//	if (BLOB_ANALYSIS_INTERVAL > 0){
         run_analysis(timestep,RESTART_INTERVAL,rank_info,*Averages,last_ids,last_index,last_id_map,
             Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den,
             LocalRestartFile,meshData,fillData,tpool,work_ids);
-		}
+	/*		}
 		else{
 		  ComputeMacroscaleAverages(timestep,ANALYSIS_INTERVAL,RESTART_INTERVAL,rank_info,*Averages,
             Nx,Ny,Nz,pBC,beta,err,Phi,Pressure,Velocity,ID,f_even,f_odd,Den,
 				  LocalRestartFile,meshData,fillData,tpool,work_ids);
 		}
+	*/
         // Save the timers
         if ( timestep%50==0 )
             PROFILE_SAVE("lbpm_color_simulator",1);

tests/lbpm_color_simulator.h

@@ -5,12 +5,13 @@
 #include "IO/MeshDatabase.h"
 
 //#define ANALYSIS_INTERVAL 6
-#define ANALYSIS_INTERVAL_DEFAULT 1000
+#define ANALYSIS_INTERVAL 1000
 #define BLOBID_INTERVAL 250
 
 enum AnalysisType{ AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02, 
     CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20 };
 
+
 // Structure used to store ids
 struct AnalysisWaitIdStruct {
     ThreadPool::thread_id_t blobID;
@@ -168,46 +169,7 @@ private:
 class AnalysisWorkItem: public ThreadPool::WorkItem
 {
 public:
-    AnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_, double beta_ ):
+    AnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_, 
-        type(type_), timestep(timestep_), Averages(Averages_), beta(beta_) { }
-    virtual void run() {
-        ThreadPool::WorkItem::d_state = 1;  // Change state to in progress
-
-        if ( (type&CopyPhaseIndicator) != 0 ) {
-            // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
-        }
-        if ( (type&ComputeAverages) != 0 ) {
-            PROFILE_START("Macroscale averages",1);
-            Averages.Initialize();
-            Averages.ComputeDelPhi();
-            Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.SDn);
-            Averages.ColorToSignedDistance(beta,Averages.Phase_tminus,Averages.Phase_tminus);
-            Averages.ColorToSignedDistance(beta,Averages.Phase_tplus,Averages.Phase_tplus);
-            Averages.UpdateMeshValues();
-            Averages.ComputeLocal();
-            Averages.Reduce();
-            Averages.PrintAll(timestep);
-
-            PROFILE_STOP("Macroscale averages",1);
-        }
-        ThreadPool::WorkItem::d_state = 2;  // Change state to finished
-    }
-private:
-    AnalysisWorkItem();
-    AnalysisType type;
-    int timestep;
-    TwoPhase& Averages;
-    BlobIDstruct blob_ids;
-    BlobIDList id_list;
-    double beta;
-};
-
-// Helper class to run the analysis from within a thread
-// Note: Averages will be modified after the constructor is called
-class BlobAnalysisWorkItem: public ThreadPool::WorkItem
-{
-public:
-    BlobAnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_,
         BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
         type(type_), timestep(timestep_), Averages(Averages_), 
         blob_ids(ids), id_list(id_list_), beta(beta_) { }
@@ -222,7 +184,7 @@ public:
             // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
         }
         if ( (type&ComputeAverages) != 0 ) {
-            PROFILE_START("Analyze blobs",1);
+            PROFILE_START("Compute dist",1);
             Averages.Initialize();
             Averages.ComputeDelPhi();
             Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.SDn);
@@ -232,17 +194,16 @@ public:
             Averages.ComputeLocal();
             Averages.Reduce();
             Averages.PrintAll(timestep);
-
             Averages.Initialize();
             Averages.ComponentAverages();
             Averages.SortBlobs();
             Averages.PrintComponents(timestep);
-            PROFILE_STOP("Analyze blobs",1);
+            PROFILE_STOP("Compute dist",1);
         }
         ThreadPool::WorkItem::d_state = 2;  // Change state to finished
     }
 private:
-    BlobAnalysisWorkItem();
+    AnalysisWorkItem();
     AnalysisType type;
     int timestep;
     TwoPhase& Averages;
@@ -252,6 +213,7 @@ private:
 };
 
 
+
 // Function to start the analysis
 void run_analysis( int timestep, int restart_interval, 
     const RankInfoStruct& rank_info, TwoPhase& Averages,
@@ -266,7 +228,7 @@ void run_analysis( int timestep, int restart_interval,
 
     // Determin the analysis we want to perform
     AnalysisType type = AnalyzeNone;
-    if ( timestep%ANALYSIS_INTERVAL_DEFAULT + 5 == ANALYSIS_INTERVAL_DEFAULT ) {
+    if ( timestep%ANALYSIS_INTERVAL + 5 == ANALYSIS_INTERVAL ) {
         // Copy the phase indicator field for the earlier timestep
         type = static_cast<AnalysisType>( type | CopyPhaseIndicator );
     }
@@ -283,12 +245,12 @@ void run_analysis( int timestep, int restart_interval,
         }
     #endif
 
-    if ( timestep%ANALYSIS_INTERVAL_DEFAULT == 0 ) {
+    if ( timestep%ANALYSIS_INTERVAL == 0 ) {
         // Copy the averages to the CPU (and identify blobs)
         type = static_cast<AnalysisType>( type | CopySimState );
         type = static_cast<AnalysisType>( type | IdentifyBlobs );
     }
-    if ( timestep%ANALYSIS_INTERVAL_DEFAULT == 5 ) {
+    if ( timestep%ANALYSIS_INTERVAL == 5 ) {
         // Run the analysis
         type = static_cast<AnalysisType>( type | ComputeAverages );
     }
@@ -377,7 +339,7 @@ void run_analysis( int timestep, int restart_interval,
 
     // Spawn threads to do the analysis work
     if ( (type&ComputeAverages) != 0 ) {
-        ThreadPool::WorkItem *work = new BlobAnalysisWorkItem(
+        ThreadPool::WorkItem *work = new AnalysisWorkItem(
             type,timestep,Averages,last_index,last_id_map,beta);
         work->add_dependency(wait.blobID);
         work->add_dependency(wait.analysis);
@@ -418,134 +380,4 @@ void run_analysis( int timestep, int restart_interval,
 }
 
 
-// Function to start the analysis
-void ComputeMacroscaleAverages( int timestep, int analysis_interval, int restart_interval,
-    const RankInfoStruct& rank_info, TwoPhase& Averages,
-    int Nx, int Ny, int Nz, bool pBC, double beta, double err,
-    const double *Phi, double *Pressure, const double *Velocity,
-    const char *ID, const double *f_even, const double *f_odd, const double *Den,
-    const char *LocalRestartFile, std::vector<IO::MeshDataStruct>& visData, fillHalo<double>& fillData,
-    ThreadPool& tpool, AnalysisWaitIdStruct& wait )
-{
-    int N = Nx*Ny*Nz;
-
-    // Determine the analysis we want to perform
-    AnalysisType type = AnalyzeNone;
-    if ( timestep%analysis_interval + 5 == analysis_interval ) {
-        // Copy the phase indicator field for the earlier timestep
-        type = static_cast<AnalysisType>( type | CopyPhaseIndicator );
-    }
-
-    if ( timestep%analysis_interval == 0 ) {
-        // Copy the averages to the CPU (and identify blobs)
-        type = static_cast<AnalysisType>( type | CopySimState );
-    }
-    if ( timestep%analysis_interval == 5 ) {
-        // Run the analysis
-        type = static_cast<AnalysisType>( type | ComputeAverages );
-    }
-    if (timestep%restart_interval == 0) {
-        // Write the restart file
-        type = static_cast<AnalysisType>( type | CreateRestart );
-    }
-    if (timestep%restart_interval == 0) {
-        // Write the visualization data
-        type = static_cast<AnalysisType>( type | WriteVis );
-        type = static_cast<AnalysisType>( type | CopySimState );
-    }
-
-    // Return if we are not doing anything
-    if ( type == AnalyzeNone )
-        return;
-
-    PROFILE_START("start_analysis");
-
-    // Copy the appropriate variables to the host (so we can spawn new threads)
-    DeviceBarrier();
-    PROFILE_START("Copy data to host",1);
-    std::shared_ptr<DoubleArray> phase;
-    if ( (type&CopyPhaseIndicator)!=0 || (type&ComputeAverages)!=0 ||
-         (type&CopySimState)!=0  )
-    {
-        phase = std::shared_ptr<DoubleArray>(new DoubleArray(Nx,Ny,Nz));
-        CopyToHost(phase->get(),Phi,N*sizeof(double));
-    }
-    if ( (type&CopyPhaseIndicator)!=0 ) {
-        memcpy(Averages.Phase_tplus.get(),phase->get(),N*sizeof(double));
-        //Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
-    }
-    if ( (type&ComputeAverages)!=0 ) {
-        memcpy(Averages.Phase_tminus.get(),phase->get(),N*sizeof(double));
-        //Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tminus);
-    }
-    if ( (type&CopySimState) != 0 ) {
-        // Copy the members of Averages to the cpu (phase was copied above)
-        // Wait
-        PROFILE_START("Copy-Pressure",1);
-        ComputePressureD3Q19(ID,f_even,f_odd,Pressure,Nx,Ny,Nz);
-        DeviceBarrier();
-        PROFILE_STOP("Copy-Pressure",1);
-        PROFILE_START("Copy-Wait",1);
-        tpool.wait(wait.analysis);
-        tpool.wait(wait.vis);   // Make sure we are done using analysis before modifying
-        PROFILE_STOP("Copy-Wait",1);
-        PROFILE_START("Copy-State",1);
-        memcpy(Averages.Phase.get(),phase->get(),N*sizeof(double));
-        CopyToHost(Averages.Press.get(),Pressure,N*sizeof(double));
-        CopyToHost(Averages.Vel_x.get(),&Velocity[0],N*sizeof(double));
-        CopyToHost(Averages.Vel_y.get(),&Velocity[N],N*sizeof(double));
-        CopyToHost(Averages.Vel_z.get(),&Velocity[2*N],N*sizeof(double));
-        PROFILE_STOP("Copy-State",1);
-    }
-    std::shared_ptr<double> cDen, cDistEven, cDistOdd;
-    if ( (type&CreateRestart) != 0 ) {
-        // Copy restart data to the CPU
-        cDen = std::shared_ptr<double>(new double[2*N],DeleteArray<double>);
-        cDistEven = std::shared_ptr<double>(new double[10*N],DeleteArray<double>);
-        cDistOdd = std::shared_ptr<double>(new double[9*N],DeleteArray<double>);
-        CopyToHost(cDistEven.get(),f_even,10*N*sizeof(double));
-        CopyToHost(cDistOdd.get(),f_odd,9*N*sizeof(double));
-        CopyToHost(cDen.get(),Den,2*N*sizeof(double));
-    }
-    PROFILE_STOP("Copy data to host",1);
-
-    // Spawn threads to do the analysis work
-    if ( (type&ComputeAverages) != 0 ) {
-        ThreadPool::WorkItem *work = new AnalysisWorkItem(
-            type,timestep,Averages,beta);
-        work->add_dependency(wait.analysis);
-        work->add_dependency(wait.vis);     // Make sure we are done using analysis before modifying
-        wait.analysis = tpool.add_work(work);
-    }
-
-    // Spawn a thread to write the restart file
-    if ( (type&CreateRestart) != 0 ) {
-        int rank = MPI_WORLD_RANK();
-        if (pBC) {
-            //err = fabs(sat_w - sat_w_previous);
-            //sat_w_previous = sat_w;
-            if (rank==0) printf("Timestep %i: change in saturation since last checkpoint is %f \n",timestep,err);
-        } else {
-            // Not clear yet
-        }
-        // Wait for previous restart files to finish writing (not necessary, but helps to ensure memory usage is limited)
-        tpool.wait(wait.restart);
-        // Write the restart file (using a seperate thread)
-        WriteRestartWorkItem *work = new WriteRestartWorkItem(LocalRestartFile,cDen,cDistEven,cDistOdd,N);
-        work->add_dependency(wait.restart);
-        wait.restart = tpool.add_work(work);
-    }
-
-    // Save the results for visualization
-    if ( (type&CreateRestart) != 0 ) {
-        // Wait for previous restart files to finish writing (not necessary, but helps to ensure memory usage is limited)
-        tpool.wait(wait.vis);
-        // Write the vis files
-        ThreadPool::WorkItem *work = new WriteVisWorkItem( timestep, visData, Averages, fillData );
-        work->add_dependency(wait.analysis);
-        work->add_dependency(wait.vis);
-        wait.vis = tpool.add_work(work);
-    }
-    PROFILE_STOP("start_analysis");
-}