Fixing compile errors without MPI

StackTrace/ErrorHandlers.h

@@ -3,11 +3,10 @@
 
 
 #include "StackTrace/StackTrace.h"
+#include "common/MPI.h"
 
 #include <functional>
 
-#include "mpi.h"
-
 
 namespace StackTrace
 {

common/ScaLBL.cpp

@@ -1,5 +1,8 @@
 #include "common/ScaLBL.h"
 
+#include <chrono>
+
+
 ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
 	//......................................................................................
 	Lock=false; // unlock the communicator
@@ -411,20 +414,19 @@ double ScaLBL_Communicator::GetPerformance(int *NeighborList, double *fq, int Np
 	double FZ = 0.0;
     ScaLBL_D3Q19_Init(fq, Np);
 	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	Barrier();
-	starttime = MPI_Wtime();
+    auto t1 = std::chrono::system_clock::now();
-	//.........................................
 	for (int t=0; t<TIMESTEPS; t++){
 		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq,  FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
 		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
 		ScaLBL_D3Q19_AAeven_MRT(fq, FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
 		ScaLBL_D3Q19_AAeven_MRT(fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
 	}
-	stoptime = MPI_Wtime();
+    auto t2 = std::chrono::system_clock::now();
 	Barrier();
 	// Compute the walltime per timestep
-	cputime = 0.5*(stoptime - starttime)/TIMESTEPS;
+    double diff = std::chrono::duration<double>( t2 - t1 ).count();
+	double cputime = 0.5*diff/TIMESTEPS;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 	return MLUPS;

common/WideHalo.cpp

@@ -298,11 +298,10 @@ ScaLBLWideHalo_Communicator::~ScaLBLWideHalo_Communicator()
 void ScaLBLWideHalo_Communicator::Recv(double *data){
 
 	//...................................................................................
-	MPI_Waitall(26,req1,stat1);
+	Utilities::MPI::waitAll(26,req1);
-	MPI_Waitall(26,req2,stat2);
+	Utilities::MPI::waitAll(26,req2);
 	ScaLBL_DeviceBarrier();
 	//...................................................................................
-	//...................................................................................
 	ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh);
 	ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh);
 	ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh);

common/WideHalo.h

@@ -4,6 +4,7 @@ This class implements support for halo widths larger than 1
 #ifndef WideHalo_H
 #define WideHalo_H
 #include "common/ScaLBL.h"
+#include "common/MPI.h"
 
 class ScaLBLWideHalo_Communicator{
 public:
@@ -52,9 +53,7 @@ private:
 	int sendtag,recvtag;
 	// Give the object it's own MPI communicator
 	RankInfoStruct rank_info;
-	MPI_Group Group;	// Group of processors associated with this domain
 	MPI_Request req1[26],req2[26];
-	MPI_Status stat1[26],stat2[26];
 	//......................................................................................
 	// MPI ranks for all 18 neighbors
 	//......................................................................................

models/ColorModel.cpp

@@ -688,20 +688,15 @@ void ScaLBL_ColorModel::Run(){
 		fflush(stdout);
 	}
 
-	//.......create and start timer............
-	double starttime,stoptime,cputime;
-	ScaLBL_Comm->Barrier();
-	comm.barrier();
-	starttime = MPI_Wtime();
-	//.........................................
-
 	//************ MAIN ITERATION LOOP ***************************************/
+	comm.barrier();
 	PROFILE_START("Loop");
     //std::shared_ptr<Database> analysis_db;
 	bool Regular = false;
 	auto current_db = db->cloneDatabase();
 	runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
 	//analysis.createThreads( analysis_method, 4 );
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax ) {
 		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
 		PROFILE_START("Update");
@@ -1034,10 +1029,10 @@ void ScaLBL_ColorModel::Run(){
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
 	ScaLBL_Comm->Barrier();
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 

models/DFHModel.cpp

@@ -490,14 +490,10 @@ void ScaLBL_DFHModel::Run(){
 
 	if (rank==0) printf("********************************************************\n");
 	if (rank==0)    printf("No. of timesteps: %i \n", timestepMax);
-	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_DeviceBarrier();
 	comm.barrier();
-	starttime = MPI_Wtime();
-	//.........................................
 	//************ MAIN ITERATION LOOP ***************************************/
-
+    auto t1 = std::chrono::system_clock::now();
 	bool Regular = true;
 	PROFILE_START("Loop");
 	runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
@@ -589,10 +585,10 @@ void ScaLBL_DFHModel::Run(){
 	//************************************************************************
 	ScaLBL_DeviceBarrier();
 	comm.barrier();
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 	if (rank==0) printf("********************************************************\n");

models/FreeLeeModel.cpp

@@ -719,14 +719,9 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 		fflush(stdout);
 	}
 
-	//.......create and start timer............
-	double starttime,stoptime,cputime;
-	ScaLBL_Comm->Barrier();
-	comm.barrier();
-	starttime = MPI_Wtime();
-	//.........................................
-
 	//************ MAIN ITERATION LOOP ***************************************/
+	comm.barrier();
+    auto t1 = std::chrono::system_clock::now();
 	PROFILE_START("Loop");
 	while (timestep < timestepMax ) {
 		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
@@ -818,10 +813,10 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 	PROFILE_STOP("Loop");
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 
@@ -846,14 +841,13 @@ void ScaLBL_FreeLeeModel::Run_SingleFluid(){
 	}
 
 	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_Comm->Barrier();
 	comm.barrier();
-	starttime = MPI_Wtime();
 	//.........................................
 
 	//************ MAIN ITERATION LOOP ***************************************/
 	PROFILE_START("Loop");
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax ) {
 		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
 		PROFILE_START("Update");
@@ -916,10 +910,10 @@ void ScaLBL_FreeLeeModel::Run_SingleFluid(){
 	PROFILE_STOP("Loop");
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 

models/GreyscaleColorModel.cpp

@@ -910,10 +910,8 @@ void ScaLBL_GreyscaleColorModel::Run(){
 	}
 
 	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_Comm->Barrier();
 	comm.barrier();
-	starttime = MPI_Wtime();
 	//.........................................
 
 	//************ MAIN ITERATION LOOP ***************************************/
@@ -923,6 +921,7 @@ void ScaLBL_GreyscaleColorModel::Run(){
 	auto current_db = db->cloneDatabase();
 	//runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
 	//analysis.createThreads( analysis_method, 4 );
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax ) {
 		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
 		PROFILE_START("Update");
@@ -1319,10 +1318,10 @@ void ScaLBL_GreyscaleColorModel::Run(){
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
 	ScaLBL_Comm->Barrier();
-		stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 

models/GreyscaleModel.cpp

@@ -485,10 +485,8 @@ void ScaLBL_GreyscaleModel::Run(){
 	}
 
 	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_DeviceBarrier();
 	comm.barrier();
-	starttime = MPI_Wtime();
 	//.........................................
 	
 	Minkowski Morphology(Mask);
@@ -500,6 +498,7 @@ void ScaLBL_GreyscaleModel::Run(){
     double rlx_eff = 1.0/tau_eff;
 	double error = 1.0;
 	double flow_rate_previous = 0.0;
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax && error > tolerance) {
 		//************************************************************************/
 		// *************ODD TIMESTEP*************//
@@ -744,10 +743,10 @@ void ScaLBL_GreyscaleModel::Run(){
 	//************************************************************************
 	ScaLBL_DeviceBarrier();
 	comm.barrier();
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 

models/IonModel.cpp

@@ -784,7 +784,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
 	//.......create and start timer............
 	//double starttime,stoptime,cputime;
 	//ScaLBL_Comm->Barrier(); comm.barrier();
-	//starttime = MPI_Wtime();
+    //auto t1 = std::chrono::system_clock::now();
 
 	for (int ic=0; ic<number_ion_species; ic++){
         timestep=0;
@@ -886,10 +886,10 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
         ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np);
     }
 	//************************************************************************/
-	//stoptime = MPI_Wtime();
 	//if (rank==0) printf("-------------------------------------------------------------------\n");
 	//// Compute the walltime per timestep
-	//cputime = (stoptime - starttime)/timestep;
+    //auto t2 = std::chrono::system_clock::now();
+	//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	//// Performance obtained from each node
 	//double MLUPS = double(Np)/cputime/1000000;
 

models/MRTModel.cpp

@@ -230,14 +230,13 @@ void ScaLBL_MRTModel::Run(){
 	}
 
 	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_DeviceBarrier(); comm.barrier();
-	starttime = MPI_Wtime();
 	if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax);
 	if (rank==0) printf("********************************************************\n");
 	timestep=0;
 	double error = 1.0;
 	double flow_rate_previous = 0.0;
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax && error > tolerance) {
 		//************************************************************************/
 		timestep++;
@@ -354,10 +353,10 @@ void ScaLBL_MRTModel::Run(){
 		}
 	}
 	//************************************************************************/
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 

models/PoissonSolver.cpp

@@ -522,8 +522,8 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
     
 	//.......create and start timer............
 	//double starttime,stoptime,cputime;
-	//ScaLBL_Comm->Barrier(); comm.barrier();
+	//comm.barrier();
-	//starttime = MPI_Wtime();
+    //auto t1 = std::chrono::system_clock::now();
 
 	timestep=0;
 	double error = 1.0;
@@ -579,11 +579,11 @@ void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
     }
 
 	//************************************************************************/
-	//stoptime = MPI_Wtime();
 	////if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n");
 	////if (rank==0) printf("---------------------------------------------------------------------------\n");
 	//// Compute the walltime per timestep
-	//cputime = (stoptime - starttime)/timestep;
+	//auto t2 = std::chrono::system_clock::now();
+	//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	//// Performance obtained from each node
 	//double MLUPS = double(Np)/cputime/1000000;
 

models/StokesModel.cpp

@@ -573,16 +573,14 @@ void ScaLBL_StokesModel::Run(){
 		}
 	}
 
-	//.......create and start timer............
-	double starttime,stoptime,cputime;
 	ScaLBL_Comm->Barrier(); comm.barrier();
-	starttime = MPI_Wtime();
 	if (rank==0) printf("****************************************************************\n");
 	if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: timestepMax = %i\n", timestepMax);
 	if (rank==0) printf("****************************************************************\n");
 	timestep=0;
 	double error = 1.0;
 	double flow_rate_previous = 0.0;
+    auto t1 = std::chrono::system_clock::now();
 	while (timestep < timestepMax && error > tolerance) {
 		//************************************************************************/
 		timestep++;
@@ -700,10 +698,10 @@ void ScaLBL_StokesModel::Run(){
 		}
 	}
 	//************************************************************************/
-	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
-	cputime = (stoptime - starttime)/timestep;
+    auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;