fix the build

models/DFHModel.cpp

@@ -3,7 +3,7 @@ color lattice boltzmann model
  */
 #include "models/DFHModel.h"
 
-ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_DFHModel::ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
@@ -114,9 +114,9 @@ void ScaLBL_DFHModel::SetDomain(){
 	id = new char [N];
 	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
 	Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
-	MPI_Barrier(comm);
+	comm.barrier();
 	Dm->CommInit();
-	MPI_Barrier(comm);
+	comm.barrier();
 	rank = Dm->rank();
 }
 
@@ -138,7 +138,7 @@ void ScaLBL_DFHModel::ReadInput(){
 	sprintf(LocalRankString,"%05d",rank);
 	sprintf(LocalRankFilename,"%s%s","SignDist.",LocalRankString);
 	ReadBinaryFile(LocalRankFilename, Averages->SDs.data(), N);
-	MPI_Barrier(comm);
+	comm.barrier();
 	if (rank == 0) cout << "Domain set." << endl;
 }
 
@@ -430,7 +430,8 @@ void ScaLBL_DFHModel::Initialize(){
 			}
 		}
 	}
-	MPI_Allreduce(&count_wet,&count_wet_global,1,MPI_DOUBLE,MPI_SUM,comm);
+	count_wet_global=Dm->Comm.sumReduce(  count_wet);
+
 	if (rank==0)	printf("Wetting phase volume fraction =%f \n",count_wet_global/double(Nx*Ny*Nz*nprocs));
 	// initialize phi based on PhaseLabel (include solid component labels)
 	ScaLBL_CopyToDevice(Phi, PhaseLabel, Np*sizeof(double));
@@ -452,7 +453,7 @@ void ScaLBL_DFHModel::Initialize(){
 				timestep=0;
 			}
 		}
-		MPI_Bcast(&timestep,1,MPI_INT,0,comm);
+		//MPI_Bcast(&timestep,1,MPI_INT,0,comm);
 		// Read in the restart file to CPU buffers
 		double *cPhi = new double[Np];
 		double *cDist = new double[19*Np];
@@ -474,7 +475,7 @@ void ScaLBL_DFHModel::Initialize(){
 		ScaLBL_DeviceBarrier();
 		delete [] cPhi;
 		delete [] cDist;
-		MPI_Barrier(comm);
+		comm.barrier();
 	}
 
 	if (rank==0)    printf ("Initializing phase field \n");
@@ -492,7 +493,7 @@ void ScaLBL_DFHModel::Run(){
 	//.......create and start timer............
 	double starttime,stoptime,cputime;
 	ScaLBL_DeviceBarrier();
-	MPI_Barrier(comm);
+	comm.barrier();
 	starttime = MPI_Wtime();
 	//.........................................
 	//************ MAIN ITERATION LOOP ***************************************/
@@ -538,7 +539,7 @@ void ScaLBL_DFHModel::Run(){
 		}
 		ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
 				alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->LastExterior(), Np);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); comm.barrier();
 
 		// *************EVEN TIMESTEP*************
 		timestep++;
@@ -574,9 +575,9 @@ void ScaLBL_DFHModel::Run(){
 		}
 		ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
 				alpha, beta, Fx, Fy, Fz,  0, ScaLBL_Comm->LastExterior(), Np);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); comm.barrier();
 		//************************************************************************
-		MPI_Barrier(comm);
+		comm.barrier();
 		PROFILE_STOP("Update");
 
 		// Run the analysis
@@ -587,7 +588,7 @@ void ScaLBL_DFHModel::Run(){
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
 	ScaLBL_DeviceBarrier();
-	MPI_Barrier(comm);
+	comm.barrier();
 	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep

models/DFHModel.h

@@ -18,7 +18,7 @@ Implementation of color lattice boltzmann model
 
 class ScaLBL_DFHModel{
 public:
-	ScaLBL_DFHModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_DFHModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_DFHModel();	
 	
 	// functions in they should be run
@@ -66,7 +66,7 @@ public:
     double *Pressure;
 		
 private:
-	MPI_Comm comm;
+	Utilities::MPI comm;
     
 	int dist_mem_size;
 	int neighborSize;

models/GreyscaleColorModel.cpp

@@ -15,7 +15,7 @@ void DeleteArray( const TYPE *p )
     delete [] p;
 }
 
-ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
@@ -135,9 +135,9 @@ void ScaLBL_GreyscaleColorModel::SetDomain(){
 	id = new signed char [N];
 	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
 	Averages = std::shared_ptr<GreyPhaseAnalysis> ( new GreyPhaseAnalysis(Dm) ); // TwoPhase analysis object
-	MPI_Barrier(comm);
+	comm.barrier();
 	Dm->CommInit();
-	MPI_Barrier(comm);
+	comm.barrier();
 	// Read domain parameters
 	rank = Dm->rank();	
 	nprocx = Dm->nprocx();
@@ -601,7 +601,7 @@ void ScaLBL_GreyscaleColorModel::Create(){
 	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
 	auto neighborList= new int[18*Npad];
        	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
-	MPI_Barrier(comm);
+	comm.barrier();
 
 	//...........................................................................
 	//                MAIN  VARIABLES ALLOCATED HERE
@@ -740,7 +740,7 @@ void ScaLBL_GreyscaleColorModel::Initialize(){
 		ScaLBL_CopyToDevice(Phi,cPhi,N*sizeof(double));
 		ScaLBL_Comm->Barrier();
 
-		MPI_Barrier(comm);
+		comm.barrier();
 
         if (rank==0)	printf ("Initializing phase field from Restart\n");
         ScaLBL_PhaseField_InitFromRestart(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
@@ -912,7 +912,7 @@ void ScaLBL_GreyscaleColorModel::Run(){
 	//.......create and start timer............
 	double starttime,stoptime,cputime;
 	ScaLBL_Comm->Barrier();
-	MPI_Barrier(comm);
+	comm.barrier();
 	starttime = MPI_Wtime();
 	//.........................................
 
@@ -1071,7 +1071,7 @@ void ScaLBL_GreyscaleColorModel::Run(){
                 }
             }
             RESTARTFILE.close();
-		    MPI_Barrier(comm);
+		    comm.barrier();
         }
 		if (timestep%visualization_interval==0){
             WriteVisFiles();

models/GreyscaleColorModel.h

@@ -17,7 +17,7 @@ Implementation of two-fluid greyscale color lattice boltzmann model
 
 class ScaLBL_GreyscaleColorModel{
 public:
-	ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_GreyscaleColorModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_GreyscaleColorModel();	
 	
 	// functions in they should be run
@@ -72,7 +72,7 @@ public:
     double *Permeability_dvc;
 		
 private:
-	MPI_Comm comm;
+	Utilities::MPI comm;
     
 	int dist_mem_size;
 	int neighborSize;

models/GreyscaleModel.cpp

@@ -13,7 +13,7 @@ void DeleteArray( const TYPE *p )
     delete [] p;
 }
 
-ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
 {
@@ -121,9 +121,9 @@ void ScaLBL_GreyscaleModel::SetDomain(){
 
 	id = new signed char [N];
 	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
-	MPI_Barrier(comm);
+	comm.barrier();
 	Dm->CommInit();
-	MPI_Barrier(comm);
+	comm.barrier();
 	// Read domain parameters
 	rank = Dm->rank();	
 	nprocx = Dm->nprocx();
@@ -367,7 +367,7 @@ void ScaLBL_GreyscaleModel::Create(){
 	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
 	auto neighborList= new int[18*Npad];
 	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
-	MPI_Barrier(comm);
+	comm.barrier();
 
 	//...........................................................................
 	//                MAIN  VARIABLES ALLOCATED HERE
@@ -454,7 +454,7 @@ void ScaLBL_GreyscaleModel::Initialize(){
 		ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double));
 		ScaLBL_DeviceBarrier();
 
-		MPI_Barrier(comm);
+		comm.barrier();
 	}
 }
 
@@ -487,7 +487,7 @@ void ScaLBL_GreyscaleModel::Run(){
 	//.......create and start timer............
 	double starttime,stoptime,cputime;
 	ScaLBL_DeviceBarrier();
-	MPI_Barrier(comm);
+	comm.barrier();
 	starttime = MPI_Wtime();
 	//.........................................
 	
@@ -540,7 +540,7 @@ void ScaLBL_GreyscaleModel::Run(){
 		            ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
                     break;
         }
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); comm.barrier();
 
 		// *************EVEN TIMESTEP*************//
 		timestep++;
@@ -580,7 +580,7 @@ void ScaLBL_GreyscaleModel::Run(){
 		            ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
                     break;
         }
-        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+        ScaLBL_DeviceBarrier(); comm.barrier();
 		//************************************************************************/
 		
 		if (timestep%analysis_interval==0){
@@ -735,7 +735,7 @@ void ScaLBL_GreyscaleModel::Run(){
             RESTARTFILE=fopen(LocalRestartFile,"wb");
             fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE);
             fclose(RESTARTFILE);
-		    MPI_Barrier(comm);
+		    comm.barrier();
         }
 	}
 
@@ -743,7 +743,7 @@ void ScaLBL_GreyscaleModel::Run(){
 	PROFILE_SAVE("lbpm_greyscale_simulator",1);
 	//************************************************************************
 	ScaLBL_DeviceBarrier();
-	MPI_Barrier(comm);
+	comm.barrier();
 	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
@@ -766,7 +766,7 @@ void ScaLBL_GreyscaleModel::VelocityField(){
 /*	Minkowski Morphology(Mask);
 	int SIZE=Np*sizeof(double);
 	ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
-	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	ScaLBL_DeviceBarrier(); comm.barrier();
 	ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE);
 
 	memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double));

models/GreyscaleModel.h

@@ -18,7 +18,7 @@ Implementation of color lattice boltzmann model
 
 class ScaLBL_GreyscaleModel{
 public:
-	ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_GreyscaleModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_GreyscaleModel();	
 	
 	// functions in they should be run
@@ -76,7 +76,7 @@ public:
     DoubleArray Pressure;
 		
 private:
-	MPI_Comm comm;
+	Utilities::MPI comm;
     
 	int dist_mem_size;
 	int neighborSize;

models/IonModel.cpp

@@ -6,7 +6,7 @@
 #include "analysis/distance.h"
 #include "common/ReadMicroCT.h"
 
-ScaLBL_IonModel::ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_IonModel::ScaLBL_IonModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK),nprocs(NP),timestep(0),timestepMax(0),time_conv(0),kb(0),electron_charge(0),T(0),Vt(0),k2_inv(0),h(0),
 tolerance(0),number_ion_species(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),
 fluidVelx_dummy(0),fluidVely_dummy(0),fluidVelz_dummy(0),

models/IonModel.h

@@ -22,7 +22,7 @@
 
 class ScaLBL_IonModel{
 public:
-	ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_IonModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_IonModel();	
 	
 	// functions in they should be run

models/MRTModel.cpp

@@ -4,8 +4,7 @@
 #include "models/MRTModel.h"
 #include "analysis/distance.h"
 #include "common/ReadMicroCT.h"
-
-ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)

models/MRTModel.h

@@ -17,7 +17,7 @@
 
 class ScaLBL_MRTModel{
 public:
-	ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_MRTModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_MRTModel();	
 	
 	// functions in they should be run
@@ -63,7 +63,7 @@ public:
     DoubleArray Velocity_y;
     DoubleArray Velocity_z;
 private:
-	Utilities::MPI comm;
+    Utilities::MPI comm;
 	
 	// filenames
     char LocalRankString[8];

models/MultiPhysController.cpp

@@ -1,6 +1,6 @@
 #include "models/MultiPhysController.h"
 
-ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),
 analysis_interval(0),visualization_interval(0),tolerance(0),comm(COMM)
 {

models/MultiPhysController.h

@@ -19,7 +19,7 @@
 
 class ScaLBL_Multiphys_Controller{
 public:
-	ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_Multiphys_Controller(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_Multiphys_Controller();	
 	
 	void ReadParams(string filename);
@@ -44,7 +44,7 @@ public:
     std::shared_ptr<Database> study_db;
 
 private:
-	MPI_Comm comm;
+	Utilities::MPI comm;
 	
 	// filenames
     char LocalRankString[8];

models/PoissonSolver.cpp

@@ -5,7 +5,7 @@
 #include "analysis/distance.h"
 #include "common/ReadMicroCT.h"
 
-ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
 epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
 chargeDen_dummy(0),WriteLog(0),

models/PoissonSolver.h

@@ -21,7 +21,7 @@
 
 class ScaLBL_Poisson{
 public:
-	ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_Poisson();	
 	
 	// functions in they should be run

models/StokesModel.cpp

@@ -5,7 +5,7 @@
 #include "analysis/distance.h"
 #include "common/ReadMicroCT.h"
 
-ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM):
+ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, const Utilities::MPI& COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),h(0),nu_phys(0),rho_phys(0),rho0(0),den_scale(0),time_conv(0),tolerance(0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)

models/StokesModel.h

@@ -20,7 +20,7 @@
 
 class ScaLBL_StokesModel{
 public:
-	ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM);
+	ScaLBL_StokesModel(int RANK, int NP, const Utilities::MPI& COMM);
 	~ScaLBL_StokesModel();	
 	
 	// functions in they should be run
@@ -89,4 +89,4 @@ private:
     void Velocity_LB_to_Phys(DoubleArray &Vel_reg);
     vector<double> computeElectricForceAvg(double *ChargeDensity, double *ElectricField);
 };
-#endif
+#endif

tests/GenerateSphereTest.cpp

@@ -9,7 +9,7 @@
 //#include "common/pmmc.h"
 #include "common/Domain.h"
 #include "common/SpherePack.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Communication.h"
 
 /*
@@ -70,8 +70,8 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
 		}
 	}
 	// total Global is the number of nodes in the pore-space
-	MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
-	MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
+	totalGlobal = Dm.Comm.sumReduce( count );
+	maxdistGlobal = Dm.Comm.sumReduce( maxdist );
 	double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
 	double porosity=totalGlobal/volume;
 	if (rank==0) printf("Media Porosity: %f \n",porosity);
@@ -145,10 +145,9 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
 
 	// Increase the critical radius until the target saturation is met
 	double deltaR=0.05; // amount to change the radius in voxel units
-	double Rcrit_old=0.0;
-	double Rcrit_new=0.0;
+	double Rcrit_old;
+	double Rcrit_new;
 
-	double GlobalNumber = 1.f;
 	int imin,jmin,kmin,imax,jmax,kmax;
     
 	Rcrit_new = maxdistGlobal;
@@ -253,7 +252,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
         UnpackID(Dm.recvList("YZ"), Dm.recvCount("YZ") ,recvID_YZ, id);
         //......................................................................................
 
-        MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
+        //double GlobalNumber = Dm.Comm.sumReduce( LocalNumber );
 
         count = 0.f;
         for (int k=1; k<Nz-1; k++){
@@ -266,7 +265,7 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
                 }
             }
         }
-        MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
+        countGlobal = Dm.Comm.sumReduce( count );
         sw_new = countGlobal/totalGlobal;
         sw_diff_new = abs(sw_new-SW);
         // for test only
@@ -296,364 +295,11 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
 
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-Auto-merging tests/test_dcel_tri_normal.cpp
-CONFLICT (content): Merge conflict in tests/test_dcel_tri_normal.cpp
-Auto-merging tests/test_dcel_minkowski.cpp
-CONFLICT (content): Merge conflict in tests/test_dcel_minkowski.cpp
-Auto-merging tests/testCommunication.cpp
-CONFLICT (content): Merge conflict in tests/testCommunication.cpp
-Auto-merging tests/lbpm_uCT_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_uCT_pp.cpp
-Auto-merging tests/lbpm_uCT_maskfilter.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_uCT_maskfilter.cpp
-Auto-merging tests/lbpm_squaretube_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_squaretube_pp.cpp
-Auto-merging tests/lbpm_sphere_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_sphere_pp.cpp
-Auto-merging tests/lbpm_segmented_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_segmented_pp.cpp
-Auto-merging tests/lbpm_segmented_decomp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_segmented_decomp.cpp
-Auto-merging tests/lbpm_refine_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_refine_pp.cpp
-Auto-merging tests/lbpm_random_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_random_pp.cpp
-Auto-merging tests/lbpm_porenetwork_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_porenetwork_pp.cpp
-Auto-merging tests/lbpm_plates_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_plates_pp.cpp
-Auto-merging tests/lbpm_permeability_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_permeability_simulator.cpp
-Auto-merging tests/lbpm_nonnewtonian_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_nonnewtonian_simulator.cpp
-Auto-merging tests/lbpm_nondarcy_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_nondarcy_simulator.cpp
-Auto-merging tests/lbpm_morphopen_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morphopen_pp.cpp
-Auto-merging tests/lbpm_morphdrain_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morphdrain_pp.cpp
-Auto-merging tests/lbpm_morph_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morph_pp.cpp
-Auto-merging tests/lbpm_minkowski_scalar.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_minkowski_scalar.cpp
-Auto-merging tests/lbpm_juanes_bench_disc_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_juanes_bench_disc_pp.cpp
-Auto-merging tests/lbpm_inkbottle_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_inkbottle_pp.cpp
-Auto-merging tests/lbpm_disc_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_disc_pp.cpp
-Auto-merging tests/lbpm_dfh_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_dfh_simulator.cpp
-Auto-merging tests/lbpm_color_simulator.cpp
-Auto-merging tests/lbpm_color_macro_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_color_macro_simulator.cpp
-Auto-merging tests/lbpm_captube_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_captube_pp.cpp
-Auto-merging tests/lbpm_BGK_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_BGK_simulator.cpp
-Auto-merging tests/lb2_Color_blob_wia_mpi.cpp
-CONFLICT (content): Merge conflict in tests/lb2_Color_blob_wia_mpi.cpp
-Auto-merging tests/lb2_CMT_wia.cpp
-CONFLICT (content): Merge conflict in tests/lb2_CMT_wia.cpp
-Auto-merging tests/hello_world.cpp
-CONFLICT (content): Merge conflict in tests/hello_world.cpp
-Auto-merging tests/convertIO.cpp
-CONFLICT (content): Merge conflict in tests/convertIO.cpp
-Auto-merging tests/TestWriter.cpp
-CONFLICT (content): Merge conflict in tests/TestWriter.cpp
-Auto-merging tests/TestTwoPhase.cpp
-CONFLICT (content): Merge conflict in tests/TestTwoPhase.cpp
-Auto-merging tests/TestTorusEvolve.cpp
-CONFLICT (content): Merge conflict in tests/TestTorusEvolve.cpp
-Auto-merging tests/TestTorus.cpp
-CONFLICT (content): Merge conflict in tests/TestTorus.cpp
-Auto-merging tests/TestTopo3D.cpp
-CONFLICT (content): Merge conflict in tests/TestTopo3D.cpp
-Auto-merging tests/TestSubphase.cpp
-CONFLICT (content): Merge conflict in tests/TestSubphase.cpp
-Auto-merging tests/TestSegDist.cpp
-CONFLICT (content): Merge conflict in tests/TestSegDist.cpp
-Auto-merging tests/TestPressVel.cpp
-CONFLICT (content): Merge conflict in tests/TestPressVel.cpp
-Auto-merging tests/TestPoiseuille.cpp
-CONFLICT (content): Merge conflict in tests/TestPoiseuille.cpp
-Auto-merging tests/TestNetcdf.cpp
-CONFLICT (content): Merge conflict in tests/TestNetcdf.cpp
-Auto-merging tests/TestMomentsD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestMomentsD3Q19.cpp
-Auto-merging tests/TestMicroCTReader.cpp
-CONFLICT (content): Merge conflict in tests/TestMicroCTReader.cpp
-Auto-merging tests/TestMassConservationD3Q7.cpp
-CONFLICT (content): Merge conflict in tests/TestMassConservationD3Q7.cpp
-Auto-merging tests/TestMap.cpp
-CONFLICT (content): Merge conflict in tests/TestMap.cpp
-Auto-merging tests/TestMRT.cpp
-CONFLICT (content): Merge conflict in tests/TestMRT.cpp
-Auto-merging tests/TestInterfaceSpeed.cpp
-CONFLICT (content): Merge conflict in tests/TestInterfaceSpeed.cpp
-Auto-merging tests/TestForceMoments.cpp
-CONFLICT (content): Merge conflict in tests/TestForceMoments.cpp
-Auto-merging tests/TestForceD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestForceD3Q19.cpp
-Auto-merging tests/TestFluxBC.cpp
-CONFLICT (content): Merge conflict in tests/TestFluxBC.cpp
-Auto-merging tests/TestDatabase.cpp
-CONFLICT (content): Merge conflict in tests/TestDatabase.cpp
-Auto-merging tests/TestCommD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestCommD3Q19.cpp
-Auto-merging tests/TestColorSquareTube.cpp
-CONFLICT (content): Merge conflict in tests/TestColorSquareTube.cpp
-Auto-merging tests/TestColorMassBounceback.cpp
-CONFLICT (content): Merge conflict in tests/TestColorMassBounceback.cpp
-Auto-merging tests/TestColorGradDFH.cpp
-CONFLICT (content): Merge conflict in tests/TestColorGradDFH.cpp
-Auto-merging tests/TestColorGrad.cpp
-CONFLICT (content): Merge conflict in tests/TestColorGrad.cpp
-Auto-merging tests/TestColorBubble.cpp
-CONFLICT (content): Merge conflict in tests/TestColorBubble.cpp
-Auto-merging tests/TestBubbleDFH.cpp
-CONFLICT (content): Merge conflict in tests/TestBubbleDFH.cpp
-Auto-merging tests/TestBubble.cpp
-CONFLICT (content): Merge conflict in tests/TestBubble.cpp
-Auto-merging tests/TestBlobIdentifyCorners.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobIdentifyCorners.cpp
-Auto-merging tests/TestBlobIdentify.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobIdentify.cpp
-Auto-merging tests/TestBlobAnalyze.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobAnalyze.cpp
-Auto-merging tests/GenerateSphereTest.cpp
-CONFLICT (content): Merge conflict in tests/GenerateSphereTest.cpp
-Auto-merging tests/ComponentLabel.cpp
-CONFLICT (content): Merge conflict in tests/ComponentLabel.cpp
-Auto-merging tests/ColorToBinary.cpp
-CONFLICT (content): Merge conflict in tests/ColorToBinary.cpp
-Auto-merging tests/CMakeLists.txt
-Auto-merging tests/BlobIdentifyParallel.cpp
-CONFLICT (content): Merge conflict in tests/BlobIdentifyParallel.cpp
-Auto-merging tests/BlobAnalyzeParallel.cpp
-CONFLICT (content): Merge conflict in tests/BlobAnalyzeParallel.cpp
-Auto-merging models/MRTModel.cpp
-CONFLICT (content): Merge conflict in models/MRTModel.cpp
-Auto-merging models/DFHModel.cpp
-CONFLICT (content): Merge conflict in models/DFHModel.cpp
-Auto-merging models/ColorModel.cpp
-CONFLICT (content): Merge conflict in models/ColorModel.cpp
-Auto-merging cuda/exe/lb2_Color_pBC_wia_mpi.cpp
-Auto-merging cuda/exe/lb2_Color_mpi.cpp
-Auto-merging cuda/exe/lb2_Color.cu
-Auto-merging cuda/exe/lb1_MRT_mpi.cu
-Auto-merging cuda/exe/lb1_MRT_mpi.cpp
-Auto-merging cuda/D3Q7.cu
-Auto-merging cuda/D3Q19.cu
-Auto-merging cuda/Color.cu
-Auto-merging common/Utilities.cpp
-Auto-merging common/ScaLBL.h
-Auto-merging common/ScaLBL.cpp
-Auto-merging common/MPI_Helpers.hpp
-CONFLICT (modify/delete): common/MPI.h deleted in electrokinetic and modified in HEAD. Version HEAD of common/MPI.h left in tree.
-CONFLICT (modify/delete): common/MPI.cpp deleted in electrokinetic and modified in HEAD. Version HEAD of common/MPI.cpp left in tree.
-Removing common/MPI.I
-Auto-merging common/Domain.h
-CONFLICT (content): Merge conflict in common/Domain.h
-Auto-merging common/Domain.cpp
-CONFLICT (content): Merge conflict in common/Domain.cpp
-Auto-merging common/Communication.h
-Removing cmake/FindHIP.cmake
-Auto-merging analysis/runAnalysis.h
-Auto-merging analysis/runAnalysis.cpp
-CONFLICT (content): Merge conflict in analysis/runAnalysis.cpp
-Auto-merging analysis/morphology.cpp
-CONFLICT (content): Merge conflict in analysis/morphology.cpp
-Auto-merging IO/silo.h
-Auto-merging IO/netcdf.h
-Removing IO/PackData.h
-Removing IO/PackData.cpp
-Auto-merging CMakeLists.txt
-CONFLICT (content): Merge conflict in CMakeLists.txt
-Automatic merge failed; fix conflicts and then commit the result.
-Auto-merging tests/test_dcel_tri_normal.cpp
-CONFLICT (content): Merge conflict in tests/test_dcel_tri_normal.cpp
-Auto-merging tests/test_dcel_minkowski.cpp
-CONFLICT (content): Merge conflict in tests/test_dcel_minkowski.cpp
-Auto-merging tests/testCommunication.cpp
-CONFLICT (content): Merge conflict in tests/testCommunication.cpp
-Auto-merging tests/lbpm_uCT_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_uCT_pp.cpp
-Auto-merging tests/lbpm_uCT_maskfilter.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_uCT_maskfilter.cpp
-Auto-merging tests/lbpm_squaretube_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_squaretube_pp.cpp
-Auto-merging tests/lbpm_sphere_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_sphere_pp.cpp
-Auto-merging tests/lbpm_segmented_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_segmented_pp.cpp
-Auto-merging tests/lbpm_segmented_decomp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_segmented_decomp.cpp
-Auto-merging tests/lbpm_refine_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_refine_pp.cpp
-Auto-merging tests/lbpm_random_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_random_pp.cpp
-Auto-merging tests/lbpm_porenetwork_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_porenetwork_pp.cpp
-Auto-merging tests/lbpm_plates_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_plates_pp.cpp
-Auto-merging tests/lbpm_permeability_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_permeability_simulator.cpp
-Auto-merging tests/lbpm_nonnewtonian_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_nonnewtonian_simulator.cpp
-Auto-merging tests/lbpm_nondarcy_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_nondarcy_simulator.cpp
-Auto-merging tests/lbpm_morphopen_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morphopen_pp.cpp
-Auto-merging tests/lbpm_morphdrain_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morphdrain_pp.cpp
-Auto-merging tests/lbpm_morph_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_morph_pp.cpp
-Auto-merging tests/lbpm_minkowski_scalar.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_minkowski_scalar.cpp
-Auto-merging tests/lbpm_juanes_bench_disc_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_juanes_bench_disc_pp.cpp
-Auto-merging tests/lbpm_inkbottle_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_inkbottle_pp.cpp
-Auto-merging tests/lbpm_disc_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_disc_pp.cpp
-Auto-merging tests/lbpm_dfh_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_dfh_simulator.cpp
-Auto-merging tests/lbpm_color_simulator.cpp
-Auto-merging tests/lbpm_color_macro_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_color_macro_simulator.cpp
-Auto-merging tests/lbpm_captube_pp.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_captube_pp.cpp
-Auto-merging tests/lbpm_BGK_simulator.cpp
-CONFLICT (content): Merge conflict in tests/lbpm_BGK_simulator.cpp
-Auto-merging tests/lb2_Color_blob_wia_mpi.cpp
-CONFLICT (content): Merge conflict in tests/lb2_Color_blob_wia_mpi.cpp
-Auto-merging tests/lb2_CMT_wia.cpp
-CONFLICT (content): Merge conflict in tests/lb2_CMT_wia.cpp
-Auto-merging tests/hello_world.cpp
-CONFLICT (content): Merge conflict in tests/hello_world.cpp
-Auto-merging tests/convertIO.cpp
-CONFLICT (content): Merge conflict in tests/convertIO.cpp
-Auto-merging tests/TestWriter.cpp
-CONFLICT (content): Merge conflict in tests/TestWriter.cpp
-Auto-merging tests/TestTwoPhase.cpp
-CONFLICT (content): Merge conflict in tests/TestTwoPhase.cpp
-Auto-merging tests/TestTorusEvolve.cpp
-CONFLICT (content): Merge conflict in tests/TestTorusEvolve.cpp
-Auto-merging tests/TestTorus.cpp
-CONFLICT (content): Merge conflict in tests/TestTorus.cpp
-Auto-merging tests/TestTopo3D.cpp
-CONFLICT (content): Merge conflict in tests/TestTopo3D.cpp
-Auto-merging tests/TestSubphase.cpp
-CONFLICT (content): Merge conflict in tests/TestSubphase.cpp
-Auto-merging tests/TestSegDist.cpp
-CONFLICT (content): Merge conflict in tests/TestSegDist.cpp
-Auto-merging tests/TestPressVel.cpp
-CONFLICT (content): Merge conflict in tests/TestPressVel.cpp
-Auto-merging tests/TestPoiseuille.cpp
-CONFLICT (content): Merge conflict in tests/TestPoiseuille.cpp
-Auto-merging tests/TestNetcdf.cpp
-CONFLICT (content): Merge conflict in tests/TestNetcdf.cpp
-Auto-merging tests/TestMomentsD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestMomentsD3Q19.cpp
-Auto-merging tests/TestMicroCTReader.cpp
-CONFLICT (content): Merge conflict in tests/TestMicroCTReader.cpp
-Auto-merging tests/TestMassConservationD3Q7.cpp
-CONFLICT (content): Merge conflict in tests/TestMassConservationD3Q7.cpp
-Auto-merging tests/TestMap.cpp
-CONFLICT (content): Merge conflict in tests/TestMap.cpp
-Auto-merging tests/TestMRT.cpp
-CONFLICT (content): Merge conflict in tests/TestMRT.cpp
-Auto-merging tests/TestInterfaceSpeed.cpp
-CONFLICT (content): Merge conflict in tests/TestInterfaceSpeed.cpp
-Auto-merging tests/TestForceMoments.cpp
-CONFLICT (content): Merge conflict in tests/TestForceMoments.cpp
-Auto-merging tests/TestForceD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestForceD3Q19.cpp
-Auto-merging tests/TestFluxBC.cpp
-CONFLICT (content): Merge conflict in tests/TestFluxBC.cpp
-Auto-merging tests/TestDatabase.cpp
-CONFLICT (content): Merge conflict in tests/TestDatabase.cpp
-Auto-merging tests/TestCommD3Q19.cpp
-CONFLICT (content): Merge conflict in tests/TestCommD3Q19.cpp
-Auto-merging tests/TestColorSquareTube.cpp
-CONFLICT (content): Merge conflict in tests/TestColorSquareTube.cpp
-Auto-merging tests/TestColorMassBounceback.cpp
-CONFLICT (content): Merge conflict in tests/TestColorMassBounceback.cpp
-Auto-merging tests/TestColorGradDFH.cpp
-CONFLICT (content): Merge conflict in tests/TestColorGradDFH.cpp
-Auto-merging tests/TestColorGrad.cpp
-CONFLICT (content): Merge conflict in tests/TestColorGrad.cpp
-Auto-merging tests/TestColorBubble.cpp
-CONFLICT (content): Merge conflict in tests/TestColorBubble.cpp
-Auto-merging tests/TestBubbleDFH.cpp
-CONFLICT (content): Merge conflict in tests/TestBubbleDFH.cpp
-Auto-merging tests/TestBubble.cpp
-CONFLICT (content): Merge conflict in tests/TestBubble.cpp
-Auto-merging tests/TestBlobIdentifyCorners.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobIdentifyCorners.cpp
-Auto-merging tests/TestBlobIdentify.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobIdentify.cpp
-Auto-merging tests/TestBlobAnalyze.cpp
-CONFLICT (content): Merge conflict in tests/TestBlobAnalyze.cpp
-Auto-merging tests/GenerateSphereTest.cpp
-CONFLICT (content): Merge conflict in tests/GenerateSphereTest.cpp
-Auto-merging tests/ComponentLabel.cpp
-CONFLICT (content): Merge conflict in tests/ComponentLabel.cpp
-Auto-merging tests/ColorToBinary.cpp
-CONFLICT (content): Merge conflict in tests/ColorToBinary.cpp
-Auto-merging tests/CMakeLists.txt
-Auto-merging tests/BlobIdentifyParallel.cpp
-CONFLICT (content): Merge conflict in tests/BlobIdentifyParallel.cpp
-Auto-merging tests/BlobAnalyzeParallel.cpp
-CONFLICT (content): Merge conflict in tests/BlobAnalyzeParallel.cpp
-Auto-merging models/MRTModel.cpp
-CONFLICT (content): Merge conflict in models/MRTModel.cpp
-Auto-merging models/DFHModel.cpp
-CONFLICT (content): Merge conflict in models/DFHModel.cpp
-Auto-merging models/ColorModel.cpp
-CONFLICT (content): Merge conflict in models/ColorModel.cpp
-Auto-merging cuda/exe/lb2_Color_pBC_wia_mpi.cpp
-Auto-merging cuda/exe/lb2_Color_mpi.cpp
-Auto-merging cuda/exe/lb2_Color.cu
-Auto-merging cuda/exe/lb1_MRT_mpi.cu
-Auto-merging cuda/exe/lb1_MRT_mpi.cpp
-Auto-merging cuda/D3Q7.cu
-Auto-merging cuda/D3Q19.cu
-Auto-merging cuda/Color.cu
-Auto-merging common/Utilities.cpp
-Auto-merging common/ScaLBL.h
-Auto-merging common/ScaLBL.cpp
-Auto-merging common/MPI_Helpers.hpp
-CONFLICT (modify/delete): common/MPI.h deleted in electrokinetic and modified in HEAD. Version HEAD of common/MPI.h left in tree.
-CONFLICT (modify/delete): common/MPI.cpp deleted in electrokinetic and modified in HEAD. Version HEAD of common/MPI.cpp left in tree.
-Removing common/MPI.I
-Auto-merging common/Domain.h
-CONFLICT (content): Merge conflict in common/Domain.h
-Auto-merging common/Domain.cpp
-CONFLICT (content): Merge conflict in common/Domain.cpp
-Auto-merging common/Communication.h
-Removing cmake/FindHIP.cmake
-Auto-merging analysis/runAnalysis.h
-Auto-merging analysis/runAnalysis.cpp
-CONFLICT (content): Merge conflict in analysis/runAnalysis.cpp
-Auto-merging analysis/morphology.cpp
-CONFLICT (content): Merge conflict in analysis/morphology.cpp
-Auto-merging IO/silo.h
-Auto-merging IO/netcdf.h
-Removing IO/PackData.h
-Removing IO/PackData.cpp
-Auto-merging CMakeLists.txt
-CONFLICT (content): Merge conflict in CMakeLists.txt
-Automatic merge failed; fix conflicts and then commit the result.
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	{
 		// parallel domain size (# of sub-domains)
 		int nprocx,nprocy,nprocz;
@@ -743,14 +389,14 @@ Automatic merge failed; fix conflicts and then commit the result.
 		//.......................................................................
 		if (rank == 0)	printf("Reading the sphere packing \n");
 		if (rank == 0)	ReadSpherePacking(nspheres,cx,cy,cz,rad);
-		MPI_Barrier(comm);
+		comm.barrier();
 		// Broadcast the sphere packing to all processes
-		MPI_Bcast(cx,nspheres,MPI_DOUBLE,0,comm);
-		MPI_Bcast(cy,nspheres,MPI_DOUBLE,0,comm);
-		MPI_Bcast(cz,nspheres,MPI_DOUBLE,0,comm);
-		MPI_Bcast(rad,nspheres,MPI_DOUBLE,0,comm);
+		comm.bcast(cx,nspheres,0);
+		comm.bcast(cy,nspheres,0);
+		comm.bcast(cz,nspheres,0);
+		comm.bcast(rad,nspheres,0);
 		//...........................................................................
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 0) cout << "Domain set." << endl;
 		if (rank == 0){
 			// Compute the Sauter mean diameter
@@ -764,7 +410,7 @@ Automatic merge failed; fix conflicts and then commit the result.
 			D = 6.0*(Nx-2)*nprocx*totVol / totArea / Lx;
 			printf("Sauter Mean Diameter (computed from sphere packing) = %f \n",D);
 		}
-		MPI_Bcast(&D,1,MPI_DOUBLE,0,comm);
+		comm.bcast(&D,1,0);
 
 		//.......................................................................
 		SignedDistance(SignDist.data(),nspheres,cx,cy,cz,rad,Lx,Ly,Lz,Nx,Ny,Nz,
@@ -796,7 +442,7 @@ Automatic merge failed; fix conflicts and then commit the result.
 			}
 		}
 		sum_local = 1.0*sum;
-		MPI_Allreduce(&sum_local,&porosity,1,MPI_DOUBLE,MPI_SUM,comm);
+		porosity = comm.sumReduce(sum_local);
 		porosity = porosity*iVol_global;
 		if (rank==0) printf("Media porosity = %f \n",porosity);
 

tests/TestBlobIdentify.cpp

@@ -23,21 +23,19 @@ inline double rand2()
 
 
 // Test if all ranks agree on a value
-bool allAgree( int x, MPI_Comm comm ) {
+bool allAgree( int x, const Utilities::MPI& comm ) {
     int x2 = x;
-    MPI_Bcast(&x2,1,MPI_INT,0,comm);
+    comm.bcast(&x2,1,0);
     int diff = x==x2 ? 0:1;
-    int diff2 = 0;
-    MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm);
+    int diff2 = comm.sumReduce( diff );
     return diff2==0;
 }
 template<class T>
-bool allAgree( const std::vector<T>& x, MPI_Comm comm ) {
+bool allAgree( const std::vector<T>& x, const Utilities::MPI& comm ) {
     std::vector<T> x2 = x;
-    MPI_Bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),MPI_INT,0,comm);
+    comm.bcast(&x2[0],x.size()*sizeof(T)/sizeof(int),0);
     int diff = x==x2 ? 0:1;
-    int diff2 = 0;
-    MPI_Allreduce(&diff,&diff2,1,MPI_INT,MPI_SUM,comm);
+    int diff2 = comm.sumReduce( diff );
     return diff2==0;
 }
 
@@ -74,9 +72,9 @@ struct bubble_struct {
 
 
 // Create a random set of bubles
-std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, MPI_Comm comm )
+std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly, double Lz, const Utilities::MPI& comm )
 {
-    int rank = comm_rank(comm);
+    int rank = comm.getRank();
     std::vector<bubble_struct> bubbles(N_bubbles);
     if ( rank == 0 ) {
         double R0 = 0.2*Lx*Ly*Lz/pow((double)N_bubbles,0.333);
@@ -91,7 +89,7 @@ std::vector<bubble_struct> create_bubbles( int N_bubbles, double Lx, double Ly,
         }
     }
     size_t N_bytes = N_bubbles*sizeof(bubble_struct);
-    MPI_Bcast((char*)&bubbles[0],N_bytes,MPI_CHAR,0,comm);
+    comm.bcast((char*)&bubbles[0],N_bytes,0);
     return bubbles;
 }
 
@@ -124,7 +122,7 @@ void fillBubbleData( const std::vector<bubble_struct>& bubbles, DoubleArray& Pha
 
 
 // Shift all of the data by the given number of cells
-void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, MPI_Comm comm )
+void shift_data( DoubleArray& data, int sx, int sy, int sz, const RankInfoStruct& rank_info, const Utilities::MPI& comm )
 {
     int nx = data.size(0)-2;
     int ny = data.size(1)-2;
@@ -296,7 +294,7 @@ int main(int argc, char **argv)
             velocity[i].z = bubbles[i].radius*(2*rand2()-1);
         }
     }
-    MPI_Bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),MPI_CHAR,0,comm);
+    comm.bcast((char*)&velocity[0],bubbles.size()*sizeof(Point),0);
     fillBubbleData( bubbles, Phase, SignDist, Lx, Ly, Lz, rank_info );
     fillData.fill(Phase);
     fillData.fill(SignDist);
@@ -390,8 +388,8 @@ int main(int argc, char **argv)
                 printf("\n");
             }
         }
-        MPI_Bcast(&N1,1,MPI_INT,0,comm);
-        MPI_Bcast(&N2,1,MPI_INT,0,comm);
+        comm.bcast(&N1,1,0);
+        comm.bcast(&N2,1,0);
         if ( N1!=nblobs || N2!=nblobs2 ) {
             if ( rank==0 )
                 printf("Error, blob ids do not map in moving bubble test (%i,%i,%i,%i)\n",

tests/TestBubbleDFH.cpp

@@ -9,7 +9,7 @@
 #include "common/Communication.h"
 #include "analysis/TwoPhase.h"
 #include "analysis/runAnalysis.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "ProfilerApp.h"
 #include "threadpool/thread_pool.h"
 
@@ -43,7 +43,7 @@ int main(int argc, char **argv)
 		int device=ScaLBL_SetDevice(rank);
 		printf("Using GPU ID %i for rank %i \n",device,rank);
 		ScaLBL_DeviceBarrier();
-		MPI_Barrier(comm);
+		comm.barrier();
 
 		PROFILE_ENABLE(1);
 		//PROFILE_ENABLE_TRACE();
@@ -70,7 +70,7 @@ int main(int argc, char **argv)
         // Initialize compute device
         //        int device=ScaLBL_SetDevice(rank);
         ScaLBL_DeviceBarrier();
-        MPI_Barrier(comm);
+        comm.barrier();
 
         Utilities::setErrorHandlers();
 
@@ -116,7 +116,7 @@ int main(int argc, char **argv)
         // Get the rank info
         const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
 
-        MPI_Barrier(comm);
+        comm.barrier();
 
         if (nprocs != nprocx*nprocy*nprocz){
             printf("nprocx =  %i \n",nprocx);
@@ -165,7 +165,7 @@ int main(int argc, char **argv)
 
         // Mask that excludes the solid phase
         auto Mask = std::make_shared<Domain>(domain_db,comm);
-        MPI_Barrier(comm);
+        comm.barrier();
 
         Nx+=2; Ny+=2; Nz += 2;
         int N = Nx*Ny*Nz;
@@ -249,6 +249,7 @@ int main(int argc, char **argv)
 		auto neighborList= new int[18*Npad];
 		Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np);
 		comm.barrier();
+
 		//...........................................................................
 		//				MAIN  VARIABLES ALLOCATED HERE
 		//...........................................................................
@@ -384,8 +385,8 @@ int main(int argc, char **argv)
 		//.......create and start timer............
 		double starttime,stoptime,cputime;
 		ScaLBL_DeviceBarrier();
-		MPI_Barrier(comm);
-		starttime = MPI_Wtime();
+		comm.barrier();
+		starttime = Utilities::MPI::time();
 		//.........................................
 
 		err = 1.0; 	
@@ -434,7 +435,7 @@ int main(int argc, char **argv)
 			}
 			ScaLBL_D3Q19_AAodd_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
 					alpha, beta, Fx, Fy, Fz, 0, ScaLBL_Comm->next, Np);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+			ScaLBL_DeviceBarrier(); comm.barrier();
 
 			// *************EVEN TIMESTEP*************
 			timestep++;
@@ -470,9 +471,9 @@ int main(int argc, char **argv)
 			}
 			ScaLBL_D3Q19_AAeven_DFH(NeighborList, fq, Aq, Bq, Den, Phi, Gradient, SolidPotential, rhoA, rhoB, tauA, tauB,
 					alpha, beta, Fx, Fy, Fz,  0, ScaLBL_Comm->next, Np);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+			ScaLBL_DeviceBarrier(); comm.barrier();
 			//************************************************************************
-			MPI_Barrier(comm);
+			comm.barrier();
 			PROFILE_STOP("Update");
 
 			// Run the analysis
@@ -484,8 +485,8 @@ int main(int argc, char **argv)
 		PROFILE_SAVE("lbpm_color_simulator",1);
 		//************************************************************************
 		ScaLBL_DeviceBarrier();
-		MPI_Barrier(comm);
-		stoptime = MPI_Wtime();
+		comm.barrier();
+		stoptime = Utilities::MPI::time();
 		if (rank==0) printf("-------------------------------------------------------------------\n");
 		// Compute the walltime per timestep
 		cputime = (stoptime - starttime)/timestep;
@@ -544,9 +545,9 @@ int main(int argc, char **argv)
 		PROFILE_STOP("Main");
 		PROFILE_SAVE("lbpm_color_simulator",1);
 		// ****************************************************
-		MPI_Barrier(comm);
+		comm.barrier();
 	} // Limit scope so variables that contain communicators will free before MPI_Finialize
-        Utilities::shutdown();
+    Utilities::shutdown();
 	return check;
 }
 

tests/TestColorBubble.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "models/ColorModel.h"
 
 using namespace std;
@@ -64,14 +64,11 @@ inline void InitializeBubble(ScaLBL_ColorModel &ColorModel, double BubbleRadius)
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	int check=0;
 	{
 		if (rank == 0){
@@ -95,7 +92,8 @@ int main(int argc, char **argv)
 		ColorModel.Run();	       
 		ColorModel.WriteDebug();
 	}
-        Utilities::shutdown();
+    Utilities::shutdown();
+
 	return check;
 }
 

tests/TestColorGradDFH.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 
 using namespace std;
 
@@ -25,14 +25,11 @@ std::shared_ptr<Database> loadInputs( int nprocs )
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	int check=0;
 	{
 		// parallel domain size (# of sub-domains)
@@ -81,7 +78,7 @@ int main(int argc, char **argv)
 			}
 		}
 		Dm->CommInit();
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 0) cout << "Domain set." << endl;
 		if (rank==0)	printf ("Create ScaLBL_Communicator \n");
 
@@ -208,7 +205,9 @@ int main(int argc, char **argv)
     		}
     	}
 
-       Utilities::shutdown();
-       return check;
+	}
+    Utilities::shutdown();
+
+	return check;
 }
 

tests/TestFluxBC.cpp

@@ -90,7 +90,7 @@ int main (int argc, char **argv)
 
 		Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
 		comm.barrier();
-		//......................device distributions.................................
+		//......................device distributions.................................
 		int dist_mem_size = Np*sizeof(double);
 		if (rank==0)	printf ("Allocating distributions \n");
 
@@ -148,7 +148,7 @@ int main (int argc, char **argv)
     	double *VEL;
     	VEL= new double [3*Np];
     	int SIZE=3*Np*sizeof(double);
-    	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+ 	ScaLBL_Comm->Barrier();
     	ScaLBL_CopyToHost(&VEL[0],&dvc_vel[0],SIZE);
 
     	double Q = 0.f;    	
@@ -191,7 +191,8 @@ int main (int argc, char **argv)
 			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
 			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
 			ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		    	ScaLBL_Comm->Barrier();
+
 			timestep++;
 
 			ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
@@ -200,7 +201,7 @@ int main (int argc, char **argv)
 			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
 			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
 			ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		    	ScaLBL_Comm->Barrier();
 			timestep++;
 			//************************************************************************/
 

tests/TestForceD3Q19.cpp

@@ -1,5 +1,5 @@
 #include <iostream>
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Utilities.h"
 #include <math.h>
 
@@ -443,14 +443,14 @@ inline void MRT_Transform(double *dist, int Np, double Fx, double Fy, double Fz)
 int main (int argc, char **argv)
 {
 	Utilities::startup( argc, argv );
-        Utilities::MPI comm( MPI_COMM_WORLD );
+    Utilities::MPI comm( MPI_COMM_WORLD );
 	int rank = comm.getRank();
 	int nprocs = comm.getSize();
 
 	for (int i=0; i<nprocs; i++) {
 		if ( rank==i )
 			printf("%i of %i: Testing force term \n",rank,nprocs);
-		MPI_Barrier(MPI_COMM_WORLD);
+	    comm.barrier();
 	}
 
 	// Create a memory leak for valgrind to find

tests/TestForceMoments.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 
 using namespace std;
 
@@ -46,14 +46,11 @@ std::shared_ptr<Database> loadInputs( int nprocs )
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	int check=0;
 	{
 		// parallel domain size (# of sub-domains)
@@ -97,7 +94,7 @@ int main(int argc, char **argv)
 			printf("********************************************************\n");
 		}
 
-		MPI_Barrier(comm);
+		comm.barrier();
 		kproc = rank/(nprocx*nprocy);
 		jproc = (rank-nprocx*nprocy*kproc)/nprocx;
 		iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -105,7 +102,7 @@ int main(int argc, char **argv)
 		if (rank == 0) {
 			printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
 		}
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 1){
 			printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
 			printf("\n\n");
@@ -142,7 +139,7 @@ int main(int argc, char **argv)
 			}
 		}
 		Dm->CommInit();
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 0) cout << "Domain set." << endl;
 
 		int Np=0;  // number of local pore nodes
@@ -187,7 +184,7 @@ int main(int argc, char **argv)
 
 	        if (rank == 0) PrintNeighborList(neighborList,Np, rank);
 
-		MPI_Barrier(comm);
+		comm.barrier();
 
 		//......................device distributions.................................
 		int dist_mem_size = Np*sizeof(double);
@@ -212,13 +209,13 @@ int main(int argc, char **argv)
 		//.......create and start timer............
 		double starttime,stoptime,cputime;
 
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
-		starttime = MPI_Wtime();
+		ScaLBL_DeviceBarrier(); comm.barrier();
+		starttime = Utilities::MPI::time();
 
 		/************ MAIN ITERATION LOOP (timing communications)***************************************/
 		//ScaLBL_Comm->SendD3Q19(dist, &dist[10*Np]);
 		//ScaLBL_Comm->RecvD3Q19(dist, &dist[10*Np]);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); comm.barrier();
 
 		if (rank==0) printf("Beginning AA timesteps...\n");
 		if (rank==0) printf("********************************************************\n");
@@ -230,14 +227,14 @@ int main(int argc, char **argv)
 			ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist,  ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 			ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
 			ScaLBL_D3Q19_AAodd_MRT(NeighborList, dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+			ScaLBL_DeviceBarrier(); comm.barrier();
 			timestep++;
 
 			ScaLBL_Comm->SendD3Q19AA(dist); //READ FORM NORMAL
 			ScaLBL_D3Q19_AAeven_MRT(dist, ScaLBL_Comm->first_interior, ScaLBL_Comm->last_interior, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 			ScaLBL_Comm->RecvD3Q19AA(dist); //WRITE INTO OPPOSITE
 			ScaLBL_D3Q19_AAeven_MRT(dist, 0, ScaLBL_Comm->next, Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
-			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+			ScaLBL_DeviceBarrier(); comm.barrier();
 			timestep++;
 			//************************************************************************/
 			
@@ -247,7 +244,7 @@ int main(int argc, char **argv)
 
 
 		//************************************************************************/
-		stoptime = MPI_Wtime();
+		stoptime = Utilities::MPI::time();
 		//	cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl;
 		cputime = stoptime - starttime;
 		//	cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 <<  " MLUPS" << endl;
@@ -329,6 +326,7 @@ int main(int argc, char **argv)
 
 
 	}
-        Utilities::shutdown();
+    Utilities::shutdown();
+
 	return check;
 }

tests/TestInterfaceSpeed.cpp

@@ -2,7 +2,7 @@
 #include <math.h>
 
 #include "analysis/TwoPhase.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Communication.h"
 #include "IO/Mesh.h"
 #include "IO/Writer.h"
@@ -21,6 +21,7 @@ int main (int argc, char *argv[])
     Utilities::startup( argc, argv );
     Utilities::MPI comm( MPI_COMM_WORLD );
     int rank = comm.getRank();
+
     // Load inputs
 	string FILENAME = argv[1];
     // Load inputs
@@ -35,7 +36,7 @@ int main (int argc, char *argv[])
 
     Nx+=2; Ny+=2; Nz+=2;
 
-	for (i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;
 
 	Dm->CommInit();
 
@@ -46,9 +47,9 @@ int main (int argc, char *argv[])
 	double dist1,dist2;
 
 	Cx = Cy = Cz = N*0.5;
-	for (k=0; k<Nz; k++){
-		for (j=0; j<Ny; j++){
-			for (i=0; i<Nx; i++){
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
 				dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
 				dist2 = fabs(Cz-k)-HEIGHT;
 
@@ -57,9 +58,9 @@ int main (int argc, char *argv[])
 		} 
 	}
 	Cz += SPEED;
-	for (k=0; k<Nz; k++){
-		for (j=0; j<Ny; j++){
-			for (i=0; i<Nx; i++){
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
 				
 				dist1 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)) - RADIUS;
 				dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
@@ -72,9 +73,9 @@ int main (int argc, char *argv[])
 		}   
 	}
 	Cz += SPEED;
-	for (k=0; k<Nz; k++){
-		for (j=0; j<Ny; j++){
-			for (i=0; i<Nx; i++){
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
 				dist2 = sqrt((i-Cx)*(i-Cx)+(j-Cy)*(j-Cy)+(k-Cz)*(k-Cz)) - CAPRAD;
 				dist2 = fabs(Cz-k)-HEIGHT;
 
@@ -144,7 +145,8 @@ int main (int argc, char *argv[])
 	}
 
 	return toReturn;
+
 	comm.barrier();
-        Utilities::shutdown();
 	return 0;
+    Utilities::shutdown();
 }

tests/TestMap.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 
 using namespace std;
 
@@ -26,11 +26,8 @@ std::shared_ptr<Database> loadInputs( int nprocs )
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
 	int check=0;
 	{
@@ -42,6 +39,7 @@ int main(int argc, char **argv)
 				{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
 				{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
 
+        int rank = comm.getRank();
 		if (rank == 0){
 			printf("********************************************************\n");
 			printf("Running unit test: TestMap	\n");
@@ -49,7 +47,7 @@ int main(int argc, char **argv)
 		}
 		
 	    // Load inputs
-	    auto db = loadInputs( nprocs );
+	    auto db = loadInputs( comm.getSize() );
 	    int Nx = db->getVector<int>( "n" )[0];
 	    int Ny = db->getVector<int>( "n" )[1];
 	    int Nz = db->getVector<int>( "n" )[2];
@@ -92,7 +90,7 @@ int main(int argc, char **argv)
 
 		Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Dm->id.data(),Np);
 		comm.barrier();
-
+		
 		// Check the neighborlist
 		printf("Check neighborlist: exterior %i, first interior %i last interior %i \n",ScaLBL_Comm->LastExterior(),ScaLBL_Comm->FirstInterior(),ScaLBL_Comm->LastInterior());
 		for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
@@ -193,7 +191,7 @@ int main(int argc, char **argv)
 		delete [] TmpMap;
 
 	}
-        Utilities::shutdown();
+    Utilities::shutdown();
 
 	return check;
 }

tests/TestMomentsD3Q19.cpp

@@ -1,5 +1,5 @@
 #include <iostream>
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Utilities.h"
 #include <math.h>
 
@@ -470,7 +470,7 @@ int main (int argc, char **argv)
     for (int i=0; i<nprocs; i++) {
         if ( rank==i )
             printf("%i of %i: TestMoments\n",rank,nprocs);
-        MPI_Barrier(MPI_COMM_WORLD);
+        comm.barrier();
     }
 
     // Create a memory leak for valgrind to find

tests/TestNernstPlanck.cpp

@@ -20,26 +20,23 @@ using namespace std;
 
 int main(int argc, char **argv)
 {
-    // Initialize MPI
+	// Initialize MPI
     Utilities::startup( argc, argv );
-
+    Utilities::MPI comm( MPI_COMM_WORLD );
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
     { // Limit scope so variables that contain communicators will free before MPI_Finialize
 
-        MPI_Comm comm;
-        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
-        int rank = comm_rank(comm);
-        int nprocs = comm_size(comm);
-
         if (rank == 0){
             printf("********************************************************\n");
             printf("Running Test for LB-Poisson-Ion Coupling \n");
             printf("********************************************************\n");
         }
-
-        // Initialize compute device
-        ScaLBL_SetDevice(rank);
-        ScaLBL_DeviceBarrier();
-        MPI_Barrier(comm);
+		// Initialize compute device
+		int device=ScaLBL_SetDevice(rank);
+        NULL_USE( device );
+		ScaLBL_DeviceBarrier();
+		comm.barrier();
 
         PROFILE_ENABLE(1);
         //PROFILE_ENABLE_TRACE();
@@ -97,9 +94,6 @@ int main(int argc, char **argv)
         PROFILE_STOP("Main");
         PROFILE_SAVE("TestNernstPlanck",1);
         // ****************************************************
-        //
-        MPI_Barrier(comm);
-        MPI_Comm_free(&comm);
     } // Limit scope so variables that contain communicators will free before MPI_Finialize
 
     Utilities::shutdown();

tests/TestPNP_Stokes.cpp

@@ -21,25 +21,25 @@ using namespace std;
 
 int main(int argc, char **argv)
 {
-    // Initialize MPI and error handlers
+	// Initialize MPI
     Utilities::startup( argc, argv );
+    Utilities::MPI comm( MPI_COMM_WORLD );
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 
     { // Limit scope so variables that contain communicators will free before MPI_Finialize
 
-        MPI_Comm comm;
-        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
-        int rank = comm_rank(comm);
-        int nprocs = comm_size(comm);
 
         if (rank == 0){
             printf("********************************************************\n");
             printf("Running Test for LB-Poisson-Ion Coupling \n");
             printf("********************************************************\n");
         }
-        // Initialize compute device
-        ScaLBL_SetDevice(rank);
-        ScaLBL_DeviceBarrier();
-        MPI_Barrier(comm);
+		// Initialize compute device
+		int device=ScaLBL_SetDevice(rank);
+        NULL_USE( device );
+		ScaLBL_DeviceBarrier();
+		comm.barrier();
 
         PROFILE_ENABLE(1);
         //PROFILE_ENABLE_TRACE();
@@ -118,9 +118,7 @@ int main(int argc, char **argv)
         PROFILE_STOP("Main");
         PROFILE_SAVE("TestPNP_Stokes",1);
         // ****************************************************
-        
-        MPI_Barrier(comm);
-        MPI_Comm_free(&comm);
+
 
     } // Limit scope so variables that contain communicators will free before MPI_Finialize
 

tests/TestPoiseuille.cpp

@@ -7,7 +7,7 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "models/MRTModel.h"
 
 void ParallelPlates(ScaLBL_MRTModel &MRT){
@@ -47,14 +47,11 @@ void ParallelPlates(ScaLBL_MRTModel &MRT){
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	int check=0;
 	{
 		if (rank == 0){
@@ -76,7 +73,7 @@ int main(int argc, char **argv)
 
 		int SIZE=MRT.Np*sizeof(double);
 		ScaLBL_D3Q19_Momentum(MRT.fq,MRT.Velocity, MRT.Np);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); comm.barrier();
 		ScaLBL_CopyToHost(&Vz[0],&MRT.Velocity[0],3*SIZE);
 		
 		if (rank == 0) printf("Force: %f,%f,%f \n",MRT.Fx,MRT.Fy,MRT.Fz);
@@ -90,7 +87,7 @@ int main(int argc, char **argv)
 		j=Ny/2; k=Nz/2;
 		if (rank == 0) printf("Channel width=%f \n",W);
 		if (rank == 0) printf("ID flag vz       analytical\n");
-		MPI_Barrier(comm);
+		comm.barrier();
 
 		if (rank == 0) {
 			for (i=0;i<Nx;i++){
@@ -128,6 +125,7 @@ int main(int argc, char **argv)
 		}
 	}
 
-        Utilities::shutdown();
+    Utilities::shutdown();
+
 	return check;
 }

tests/TestPoissonSolver.cpp

@@ -20,23 +20,21 @@ int main(int argc, char **argv)
 {
     // Initialize MPI
     Utilities::startup( argc, argv );
-    
+    Utilities::MPI comm( MPI_COMM_WORLD );
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();    
     {// Limit scope so variables that contain communicators will free before MPI_Finialize
 
-        MPI_Comm comm;
-        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
-        int rank = comm_rank(comm);
-        int nprocs = comm_size(comm);
-
         if (rank == 0){
             printf("********************************************************\n");
             printf("Running Test for LB-Poisson Solver \n");
             printf("********************************************************\n");
         }
-        // Initialize compute device
-        ScaLBL_SetDevice(rank);
-        ScaLBL_DeviceBarrier();
-        MPI_Barrier(comm);
+		// Initialize compute device
+		int device=ScaLBL_SetDevice(rank);
+        NULL_USE( device );
+		ScaLBL_DeviceBarrier();
+		comm.barrier();
 
         PROFILE_ENABLE(1);
         //PROFILE_ENABLE_TRACE();
@@ -69,9 +67,6 @@ int main(int argc, char **argv)
         PROFILE_SAVE("TestPoissonSolver",1);
         // ****************************************************
         
-        MPI_Barrier(comm);
-        MPI_Comm_free(&comm);
-
     } // Limit scope so variables that contain communicators will free before MPI_Finialize
 
     Utilities::shutdown();

tests/TestPressVel.cpp

@@ -7,19 +7,16 @@
 #include <iostream>
 #include <fstream>
 #include "common/ScaLBL.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 
 
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
+    int rank = comm.getRank();
 	int check=0;
 	{
 		if (rank == 0){
@@ -48,7 +45,7 @@ int main(int argc, char **argv)
 			printf("********************************************************\n");
 		}
 
-		MPI_Barrier(comm);
+		comm.barrier();
 		int kproc = rank/(nprocx*nprocy);
 		int jproc = (rank-nprocx*nprocy*kproc)/nprocx;
 		int iproc = rank-nprocx*nprocy*kproc-nprocz*jproc;
@@ -56,7 +53,7 @@ int main(int argc, char **argv)
 		if (rank == 0) {
 			printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
 		}
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 1){
 			printf("i,j,k proc=%d %d %d \n",iproc,jproc,kproc);
 			printf("\n\n");
@@ -100,11 +97,11 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,comm);
+        sum = comm.sumReduce( sum_local );
 		porosity = sum*iVol_global;
 		if (rank==0) printf("Media porosity = %f \n",porosity);
 
-		MPI_Barrier(comm);
+		comm.barrier();
 		if (rank == 0) cout << "Domain set." << endl;
 		if (rank==0)	printf ("Create ScaLBL_Communicator \n");
 
@@ -191,6 +188,7 @@ int main(int argc, char **argv)
 	     }
 	   }
 	}
-        Utilities::shutdown();
+    Utilities::shutdown();
 	return check;
+
 }

tests/hello_world.cpp

@@ -1,5 +1,5 @@
 #include <iostream>
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Utilities.h"
 
 
@@ -13,7 +13,7 @@ int main (int argc, char **argv)
     for (int i=0; i<nprocs; i++) {
         if ( rank==i )
             printf("%i of %i: Hello world\n",rank,nprocs);
-        MPI_Barrier(MPI_COMM_WORLD);
+        comm.barrier();
     }
 
     // Create a memory leak for valgrind to find

tests/lbpm_dfh_simulator.cpp

@@ -58,7 +58,7 @@ int main(int argc, char **argv)
     PROFILE_STOP("Main");
     PROFILE_SAVE("lbpm_color_simulator",1);
 	// ****************************************************
-	MPI_Barrier(comm);
+	comm.barrier();
   } // Limit scope so variables that contain communicators will free before MPI_Finialize
     Utilities::shutdown();
 }

tests/lbpm_minkowski_scalar.cpp

@@ -33,7 +33,7 @@ int main(int argc, char **argv)
         Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
         int rank = comm.getRank();
-        int nprocs = comm.getSize();
+        //int nprocs = comm.getSize();
 	{
 		Utilities::setErrorHandlers();
 		PROFILE_START("Main");
@@ -64,108 +64,21 @@ int main(int argc, char **argv)
 		auto nx = size[0];
 		auto ny = size[1];
 		auto nz = size[2];
-		auto nprocx = nproc[0];
+		/*auto nprocx = nproc[0];
 		auto nprocy = nproc[1];
 		auto nprocz = nproc[2];
 		auto Nx = SIZE[0];
 		auto Ny = SIZE[1];
 		auto Nz = SIZE[2];
-		
+		*/
 		int i,j,k,n;
 
-		char *SegData = NULL;
-		// Rank=0 reads the entire segmented data and distributes to worker processes
-		if (rank==0){
-			printf("Dimensions of segmented image: %i x %i x %i \n",Nx,Ny,Nz);
-			SegData = new char[Nx*Ny*Nz];
-			FILE *SEGDAT = fopen(Filename.c_str(),"rb");
-			if (SEGDAT==NULL) ERROR("Error reading segmented data");
-			size_t ReadSeg;
-			ReadSeg=fread(SegData,1,Nx*Ny*Nz,SEGDAT);
-			if (ReadSeg != size_t(Nx*Ny*Nz)) printf("lbpm_segmented_decomp: Error reading segmented data (rank=%i)\n",rank);
-			fclose(SEGDAT);
-			printf("Read segmented data from %s \n",Filename.c_str());
-		}
-		MPI_Barrier(comm);
-
-		// Get the rank info
-		int N = (nx+2)*(ny+2)*(nz+2);
-		
-		std::shared_ptr<Domain> Dm (new Domain(domain_db,comm));
-		for (k=0;k<nz+2;k++){
-			for (j=0;j<ny+2;j++){
-				for (i=0;i<nx+2;i++){
-					n = k*(nx+2)*(ny+2)+j*(nx+2)+i;
-					Dm->id[n] = 1;
-				}
-			}
-		}
+		std::shared_ptr<Domain> Dm  = std::shared_ptr<Domain>(new Domain(domain_db,comm));      // full domain for analysis
+		comm.barrier();
 		Dm->CommInit();
-
-		int z_transition_size = 0;
-		int xStart = 0;
-		int yStart = 0;
-		int zStart = 0;
-		// Set up the sub-domains
-		if (rank==0){
-			printf("Distributing subdomain across %i processors \n",nprocs);
-			printf("Process grid: %i x %i x %i \n",Dm->nprocx(),Dm->nprocy(),Dm->nprocz());
-			printf("Subdomain size: %i \n",N);
-			//printf("Size of transition region: %i \n", z_transition_size);
-			char *tmp;
-			tmp = new char[N];
-			for (int kp=0; kp<nprocz; kp++){
-				for (int jp=0; jp<nprocy; jp++){
-					for (int ip=0; ip<nprocx; ip++){
-						// rank of the process that gets this subdomain
-						int rnk = kp*Dm->nprocx()*Dm->nprocy() + jp*Dm->nprocx() + ip;
-						// Pack and send the subdomain for rnk
-						for (k=0;k<nz+2;k++){
-							for (j=0;j<ny+2;j++){
-								for (i=0;i<nx+2;i++){
-									int x = xStart + ip*nx + i-1;
-									int y = yStart + jp*ny + j-1;
-									//		int z = zStart + kp*nz + k-1;
-									int z = zStart + kp*nz + k-1 - z_transition_size;
-									if (x<xStart) 	x=xStart;
-									if (!(x<Nx))	x=Nx-1;
-									if (y<yStart) 	y=yStart;
-									if (!(y<Ny))	y=Ny-1;
-									if (z<zStart) 	z=zStart;
-									if (!(z<Nz))	z=Nz-1;
-									int nlocal = k*(nx+2)*(ny+2) + j*(nx+2) + i;
-									int nglobal = z*Nx*Ny+y*Nx+x;
-									tmp[nlocal] = SegData[nglobal];
-								}
-							}
-						}
-						if (rnk==0){
-							for (k=0;k<nz+2;k++){
-								for (j=0;j<ny+2;j++){
-									for (i=0;i<nx+2;i++){
-										int nlocal = k*(nx+2)*(ny+2) + j*(nx+2) + i;
-										Dm->id[nlocal] = tmp[nlocal];
-									}
-								}
-							}
-						}
-						else{
-							printf("Sending data to process %i \n", rnk);
-							MPI_Send(tmp,N,MPI_CHAR,rnk,15,comm);
-						}
-					}
-				}
-			}
-		}
-		else{
-			// Recieve the subdomain from rank = 0
-			printf("Ready to recieve data %i at process %i \n", N,rank);
-			comm.recv(Dm->id.data(),N,0,15);
-		}
-		MPI_Barrier(comm);
 		
 		// Compute the Minkowski functionals
-		MPI_Barrier(comm);
+		comm.barrier();
 		std::shared_ptr<Minkowski> Averages(new Minkowski(Dm));
 
 		// Calculate the distance		

tests/lbpm_morph_pp.cpp

@@ -23,9 +23,9 @@
 int main(int argc, char **argv)
 {
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
+    int rank = comm.getRank();
 	{
 		//.......................................................................
 		// Reading the domain information file
@@ -125,13 +125,13 @@ int main(int argc, char **argv)
 
 		if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
 		CalcDist(SignDist,id_solid,*Dm);
-		MPI_Barrier(comm);
+		comm.barrier();
 		
 		// Extract only the connected part of NWP
 		BlobIDstruct new_index;
 		double vF=0.0; double vS=0.0;
 		ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
-		MPI_Barrier(Dm->Comm);
+		Dm->Comm.barrier();
 			
 		int count_connected=0;
 		int count_porespace=0;
@@ -153,9 +153,9 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		count_connected=sumReduce( Dm->Comm, count_connected);
-		count_porespace=sumReduce( Dm->Comm, count_porespace);
-		count_water=sumReduce( Dm->Comm, count_water);
+		count_connected = Dm->Comm.sumReduce( count_connected );
+		count_porespace = Dm->Comm.sumReduce( count_porespace );
+		count_water = Dm->Comm.sumReduce( count_water );
 		
 		for (int k=0; k<nz; k++){
 			for (int j=0; j<ny; j++){
@@ -213,7 +213,7 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		count_water=sumReduce( Dm->Comm, count_water);
+		count_water = Dm->Comm.sumReduce( count_water );
 		
 		SW = double(count_water) / count_porespace;
 		if(rank==0) printf("Final saturation: %f \n", SW);
@@ -234,12 +234,12 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		MPI_Barrier(comm);
+		comm.barrier();
 
         auto filename2 = READFILE + ".morph.raw";
 		if (rank==0) printf("Writing file to: %s \n", filename2.c_str());
 		Mask->AggregateLabels(filename2);
 	}
 
-        Utilities::shutdown();
+    Utilities::shutdown();
 }

tests/lbpm_morphdrain_pp.cpp

@@ -23,9 +23,9 @@
 int main(int argc, char **argv)
 {
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
+    int rank = comm.getRank();
 	{
 		//.......................................................................
 		// Reading the domain information file
@@ -119,7 +119,7 @@ int main(int argc, char **argv)
 		if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
 		CalcDist(SignDist,id_solid,*Dm);
 
-		MPI_Barrier(comm);
+		comm.barrier();
 
 		// Run the morphological opening
 		MorphDrain(SignDist, id, Dm, SW);
@@ -194,7 +194,7 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		MPI_Barrier(comm);
+		comm.barrier();
 
         auto filename2 = READFILE + ".morphdrain.raw";
 		if (rank==0) printf("Writing file to: %s \n", filename2.data() );

tests/lbpm_morphopen_pp.cpp

@@ -23,9 +23,9 @@
 int main(int argc, char **argv)
 {
 	// Initialize MPI
-        Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
+    int rank = comm.getRank();
 	{
 		//.......................................................................
 		// Reading the domain information file
@@ -121,7 +121,7 @@ int main(int argc, char **argv)
 		if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
 		CalcDist(SignDist,id_solid,*Dm);
 
-		MPI_Barrier(comm);
+		comm.barrier();
 
 		// Run the morphological opening
 		MorphOpen(SignDist, id, Dm, SW, ErodeLabel, OpenLabel);
@@ -196,7 +196,7 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-		MPI_Barrier(comm);
+		comm.barrier();
 
         auto filename2 = READFILE + ".morphopen.raw";
 		if (rank==0) printf("Writing file to: %s \n", filename2.data());

tests/lbpm_permeability_simulator.cpp

@@ -9,7 +9,7 @@
 #include "common/ScaLBL.h"
 #include "common/Communication.h"
 #include "analysis/TwoPhase.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "models/MRTModel.h"
 //#define WRITE_SURFACES
 
@@ -24,10 +24,10 @@ using namespace std;
 int main(int argc, char **argv)
 {
 	// Initialize MPI
-        Utilities::startup( argc, argv );
- 	Utilities::MPI comm( MPI_COMM_WORLD );
-        int rank = comm.getRank();
-        int nprocs = comm.getSize();
+    Utilities::startup( argc, argv );
+    Utilities::MPI comm( MPI_COMM_WORLD );
+    int rank = comm.getRank();
+    int nprocs = comm.getSize();
 	{
 		if (rank == 0){
 			printf("********************************************************\n");
@@ -38,7 +38,7 @@ int main(int argc, char **argv)
 		int device=ScaLBL_SetDevice(rank);
         NULL_USE( device );
 		ScaLBL_DeviceBarrier();
-		MPI_Barrier(comm);
+		comm.barrier();
 		
 		ScaLBL_MRTModel MRT(rank,nprocs,comm);
 		auto filename = argv[1];

tests/testCommunication.cpp

@@ -6,7 +6,7 @@
 #include <fstream>
 
 #include "common/Communication.h"
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
 #include "common/Array.h"
 
 using namespace std;
@@ -15,11 +15,9 @@ using namespace std;
 
 //***************************************************************************************
 
-int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
+int test_communication( const Utilities::MPI& comm, int nprocx, int nprocy, int nprocz )
 {
-    int rank,nprocs;
-    MPI_Comm_rank(comm,&rank);
-    MPI_Comm_size(comm,&nprocs);
+    int rank = comm.getRank();
     int iproc,jproc,kproc;
     int sendtag,recvtag;
     if (rank==0)    printf("\nRunning test %i %i %i\n",nprocx,nprocy,nprocz);
@@ -38,7 +36,7 @@ int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
 	    rank_xy, rank_XY, rank_xY, rank_Xy,
 	    rank_xz, rank_XZ, rank_xZ, rank_Xz,
 	    rank_yz, rank_YZ, rank_yZ, rank_Yz );
-    MPI_Barrier(comm);
+    comm.barrier();
 
     //**********************************
 
@@ -85,7 +83,7 @@ int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
     sendCount_xy = sendCount_yz = sendCount_xz = sendCount_Xy = sendCount_Yz = sendCount_xZ = 0;
     sendCount_xY = sendCount_yZ = sendCount_Xz = sendCount_XY = sendCount_YZ = sendCount_XZ = 0;
 
-    MPI_Barrier(comm);
+    comm.barrier();
     if (rank==0)    printf ("SendLists are ready on host\n");
     //......................................................................................
     // Use MPI to fill in the recvCounts form the associated processes
@@ -158,7 +156,7 @@ int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
         recvCount_yz, recvCount_YZ, recvCount_yZ, recvCount_Yz,
         rank_x, rank_y, rank_z, rank_X, rank_Y, rank_Z, rank_xy, rank_XY, rank_xY,
         rank_Xy, rank_xz, rank_XZ, rank_xZ, rank_Xz, rank_yz, rank_YZ, rank_yZ, rank_Yz );
-    MPI_Barrier(comm);
+    comm.barrier();
     if (rank==0)    printf ("RecvLists finished\n");
     
     // Free memory
@@ -181,11 +179,9 @@ int test_communication( MPI_Comm comm, int nprocx, int nprocy, int nprocz )
 
 
 template<class TYPE>
-int testHalo( MPI_Comm comm, int nprocx, int nprocy, int nprocz, int depth )
+int testHalo( const Utilities::MPI& comm, int nprocx, int nprocy, int nprocz, int depth )
 {
-    int rank,nprocs;
-    MPI_Comm_rank(comm,&rank);
-    MPI_Comm_size(comm,&nprocs);
+    int rank = comm.getRank();
     if ( rank==0 )
         printf("\nRunning Halo test %i %i %i %i\n",nprocx,nprocy,nprocz,depth);
 
@@ -292,7 +288,6 @@ int main(int argc, char **argv)
     int N_errors_global = comm.sumReduce( N_errors );
     comm.barrier();
     Utilities::shutdown();
-
     if ( rank==0 ) {
         if ( N_errors_global==0 )
             std::cout << "All tests passed\n";