merge scaLBL routines

Merge branch 'electrokinetic' of github.com:JamesEMcClure/LBPM-WIA into electrokinetic

analysis/ElectroChemistry.cpp

@@ -0,0 +1,228 @@
+#include "analysis/ElectroChemistry.h"
+
+ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr <Domain> dm):
+	Dm(dm),
+	fillData(dm->Comm,dm->rank_info,{dm->Nx-2,dm->Ny-2,dm->Nz-2},{1,1,1},0,1)
+{
+	
+    MPI_Comm_dup(dm->Comm,&comm);
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	ChemicalPotential.resize(Nx,Ny,Nz);       ChemicalPotential.fill(0);
+	ElectricalPotential.resize(Nx,Ny,Nz);      ElectricalPotential.fill(0);
+	Pressure.resize(Nx,Ny,Nz);     	Pressure.fill(0);
+	Rho.resize(Nx,Ny,Nz);       	Rho.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("electrokinetic.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("electrokinetic.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"TBD TBD\n");				
+		}
+	}
+
+}
+
+ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){
+	if (Dm->rank()==0){
+		fclose(TIMELOG);
+	}
+}
+
+void ElectroChemistryAnalyzer::SetParams(){
+	
+}
+
+void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep){
+
+	int i,j,k;
+	Poisson.getElectricPotential(ElectricalPotential);
+	
+	/* local sub-domain averages */
+	double rho_avg_local[Ion.number_ion_species];
+	double rho_mu_avg_local[Ion.number_ion_species];
+	double rho_mu_fluctuation_local[Ion.number_ion_species];	
+	double rho_psi_avg_local[Ion.number_ion_species];
+	double rho_psi_fluctuation_local[Ion.number_ion_species];
+	/* global averages */
+	double rho_avg_global[Ion.number_ion_species];
+	double rho_mu_avg_global[Ion.number_ion_species];
+	double rho_mu_fluctuation_global[Ion.number_ion_species];
+	double rho_psi_avg_global[Ion.number_ion_species];
+	double rho_psi_fluctuation_global[Ion.number_ion_species];
+	
+	for (int ion=0; ion<Ion.number_ion_species; ion++){
+		rho_avg_local[ion] = 0.0;
+		rho_mu_avg_local[ion] = 0.0;
+		rho_psi_avg_local[ion] = 0.0;
+		Ion.getIonConcentration(Rho,ion);
+		/* Compute averages for each ion */
+		for (k=1; k<Nz; k++){
+			for (j=1; j<Ny; j++){
+				for (i=1; i<Nx; i++){
+					rho_avg_local[ion] += Rho(i,j,k);
+					rho_mu_avg_local[ion] += Rho(i,j,k)*Rho(i,j,k);
+					rho_psi_avg_local[ion] += Rho(i,j,k)*ElectricalPotential(i,j,k);
+				}
+			}
+		}
+		rho_avg_global[ion]=sumReduce( Dm->Comm, rho_avg_local[ion]);
+		rho_mu_avg_global[ion]=sumReduce( Dm->Comm, rho_mu_avg_local[ion]);
+		rho_psi_avg_global[ion]=sumReduce( Dm->Comm, rho_psi_avg_local[ion]);
+		
+		rho_mu_avg_global[ion] /= rho_avg_global[ion];
+		rho_psi_avg_global[ion] /= rho_avg_global[ion];
+	}
+	
+	for (int ion=0; ion<Ion.number_ion_species; ion++){
+		rho_mu_fluctuation_local[ion] = 0.0;
+		rho_psi_fluctuation_local[ion] = 0.0;
+		/* Compute averages for each ion */
+		for (k=1; k<Nz; k++){
+			for (j=1; j<Ny; j++){
+				for (i=1; i<Nx; i++){
+					rho_mu_fluctuation_local[ion] += (Rho(i,j,k)*Rho(i,j,k) - rho_mu_avg_global[ion]);
+					rho_psi_fluctuation_local[ion] += (Rho(i,j,k)*ElectricalPotential(i,j,k) - rho_psi_avg_global[ion]);
+				}
+			}
+		}
+		rho_mu_fluctuation_global[ion]=sumReduce( Dm->Comm, rho_mu_fluctuation_local[ion]);
+		rho_psi_fluctuation_global[ion]=sumReduce( Dm->Comm, rho_psi_fluctuation_local[ion]);
+	}
+	
+	if (Dm->rank()==0){	
+		fprintf(TIMELOG,"%i ",timestep); 
+		for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
+		}
+		fflush(TIMELOG);
+	}
+/*	else{
+		fprintf(TIMELOG,"%i ",timestep); 
+		for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
+		}
+		fflush(TIMELOG);
+	} */
+}
+
+void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep){
+	
+	auto vis_db =  input_db->getDatabase( "Visualization" );
+    char VisName[40];
+
+    IO::initialize("","silo","false");
+    // Create the MeshDataStruct    
+    visData.resize(1);
+
+    visData[0].meshName = "domain";
+    visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+    auto ElectricPotential = std::make_shared<IO::Variable>();
+    std::vector<shared_ptr<IO::Variable>> IonConcentration;
+    for (int ion=0; ion<Ion.number_ion_species; ion++){
+        IonConcentration.push_back(std::make_shared<IO::Variable>());
+    }
+    auto VxVar = std::make_shared<IO::Variable>();
+    auto VyVar = std::make_shared<IO::Variable>();
+    auto VzVar = std::make_shared<IO::Variable>();
+    
+    if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
+    	ElectricPotential->name = "ElectricPotential";
+    	ElectricPotential->type = IO::VariableType::VolumeVariable;
+    	ElectricPotential->dim = 1;
+    	ElectricPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricPotential);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_concentration", true )){
+    	for (int ion=0; ion<Ion.number_ion_species; ion++){
+    		sprintf(VisName,"IonConcentration_%i",ion+1);
+    		IonConcentration[ion]->name = VisName;
+    		IonConcentration[ion]->type = IO::VariableType::VolumeVariable;
+    		IonConcentration[ion]->dim = 1;
+    		IonConcentration[ion]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonConcentration[ion]);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim = 1;
+        VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VxVar);
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim = 1;
+        VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VyVar);
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim = 1;
+        VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VzVar);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
+    	ASSERT(visData[0].vars[0]->name=="ElectricPotential");
+    	Poisson.getElectricPotential(ElectricalPotential);
+    	Array<double>& ElectricPotentialData = visData[0].vars[0]->data;
+    	fillData.copy(ElectricalPotential,ElectricPotentialData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_concentration", true )){
+    	for (int ion=0; ion<Ion.number_ion_species; ion++){
+    		sprintf(VisName,"IonConcentration_%i",ion+1);
+    		IonConcentration[ion]->name = VisName;
+    		ASSERT(visData[0].vars[1+ion]->name==VisName);
+    		Array<double>& IonConcentrationData = visData[0].vars[1+ion]->data;
+    		Ion.getIonConcentration(Rho,ion);
+    		fillData.copy(Rho,IonConcentrationData);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+0]->name=="Velocity_x");
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+1]->name=="Velocity_y");
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+2]->name=="Velocity_z");
+    	Stokes.getVelocity(Vel_x,Vel_y,Vel_z);
+    	Array<double>& VelxData = visData[0].vars[1+Ion.number_ion_species+0]->data;
+    	Array<double>& VelyData = visData[0].vars[1+Ion.number_ion_species+1]->data;
+    	Array<double>& VelzData = visData[0].vars[1+Ion.number_ion_species+2]->data;
+    	fillData.copy(Vel_x,VelxData);
+    	fillData.copy(Vel_y,VelyData);
+    	fillData.copy(Vel_z,VelzData);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "write_silo", true ))
+    	IO::writeData( timestep, visData, comm );
+
+/*    if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
+    	char CurrentIDFilename[40];
+    	sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
+    	Averages.AggregateLabels(CurrentIDFilename);
+    }
+*/	
+}

analysis/ElectroChemistry.h

@@ -0,0 +1,59 @@
+/*
+ *  averaging tools for electrochemistry
+ */
+
+#ifndef ElectroChem_INC
+#define ElectroChem_INC
+
+#include <vector>
+#include "common/Domain.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "analysis/distance.h"
+#include "analysis/Minkowski.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+#include "models/IonModel.h"
+#include "models/PoissonSolver.h"
+#include "models/StokesModel.h"
+
+class ElectroChemistryAnalyzer{
+public:
+    MPI_Comm comm;
+    int tag;
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+
+    //...........................................................................
+    int Nx,Ny,Nz;
+	DoubleArray Rho;	           // density field 
+	DoubleArray ChemicalPotential;	   // density field 
+	DoubleArray ElectricalPotential;	// density field 
+	DoubleArray Pressure; 	// pressure field
+	DoubleArray Vel_x;		// velocity field
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+	DoubleArray SDs;
+
+	ElectroChemistryAnalyzer(std::shared_ptr <Domain> Dm);
+	~ElectroChemistryAnalyzer();
+	
+	void SetParams();
+	void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep);
+	void WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep);
+
+private:
+    std::vector<IO::MeshDataStruct> visData;
+    fillHalo<double> fillData;
+	FILE *TIMELOG;
+};
+#endif
+

analysis/Minkowski.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/Minkowski.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/PointList.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/TwoPhase.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/TwoPhase.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/analysis.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/analysis.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/distance.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/distance.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/filters.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/filters.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/histogram.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/imfilter.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/imfilter.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/pmmc.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/runAnalysis.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/runAnalysis.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/uCT.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/uCT.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Array.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Array.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Communication.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Communication.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Communication.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Database.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Database.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Database.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Domain.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -643,38 +642,38 @@ void Domain::Decomp( const std::string& Filename )
 	if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6));
 	//.........................................................
 	// If external boundary conditions are applied remove solid
-	if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == 0){
-    	if (inlet_layers_z < 4){
-            inlet_layers_z=4;
-            if(RANK==0){
-                printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase);
-            } 
-        }	
-		for (int k=0; k<inlet_layers_z; k++){
-			for (int j=0;j<Ny;j++){
-				for (int i=0;i<Nx;i++){
-					int n = k*Nx*Ny+j*Nx+i;
-					id[n] = inlet_layers_phase;
-				}                    
-			}
- 		}
- 	}
-    if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == nprocz-1){
-    	if (outlet_layers_z < 4){
-            outlet_layers_z=4;
-            if(RANK==nprocs-1){
-                printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase);
-            } 
-        }	
- 		for (int k=Nz-outlet_layers_z; k<Nz; k++){
- 			for (int j=0;j<Ny;j++){
- 				for (int i=0;i<Nx;i++){
- 					int n = k*Nx*Ny+j*Nx+i;
- 					id[n] = outlet_layers_phase;
- 				}                    
- 			}
- 		}
- 	}
+//	if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == 0){
+//    	if (inlet_layers_z < 4){
+//            inlet_layers_z=4;
+//            if(RANK==0){
+//                printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase);
+//            } 
+//        }	
+//		for (int k=0; k<inlet_layers_z; k++){
+//			for (int j=0;j<Ny;j++){
+//				for (int i=0;i<Nx;i++){
+//					int n = k*Nx*Ny+j*Nx+i;
+//					id[n] = inlet_layers_phase;
+//				}                    
+//			}
+// 		}
+// 	}
+//    if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == nprocz-1){
+//    	if (outlet_layers_z < 4){
+//            outlet_layers_z=4;
+//            if(RANK==nprocs-1){
+//                printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase);
+//            } 
+//        }	
+// 		for (int k=Nz-outlet_layers_z; k<Nz; k++){
+// 			for (int j=0;j<Ny;j++){
+// 				for (int i=0;i<Nx;i++){
+// 					int n = k*Nx*Ny+j*Nx+i;
+// 					id[n] = outlet_layers_phase;
+// 				}                    
+// 			}
+// 		}
+// 	}
     for (int k=inlet_layers_z+1; k<Nz-outlet_layers_z-1;k++){
         for (int j=1;j<Ny-1;j++){
             for (int i=1;i<Nx-1;i++){
@@ -1304,7 +1303,6 @@ void ReadBinaryFile(char *FILENAME, double *Data, size_t N)
   File.close();
 }
 
-
 void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData)
 {
 	//........................................................................................
@@ -1452,3 +1450,94 @@ void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatyp
 	//Comm.barrier();
 	MPI_Barrier(Comm);
 }
+
+void Domain::AggregateLabels( const std::string& filename, DoubleArray &UserData ){
+	
+	int nx = Nx;
+	int ny = Ny;
+	int nz = Nz;
+	
+	int npx = nprocx();
+	int npy = nprocy();
+	int npz = nprocz();
+	
+	int ipx = iproc();
+	int ipy = jproc();
+	int ipz = kproc();		
+	
+	int nprocs = nprocx()*nprocy()*nprocz();
+		
+	int full_nx = npx*(nx-2);
+	int full_ny = npy*(ny-2);
+	int full_nz = npz*(nz-2);
+	int local_size = (nx-2)*(ny-2)*(nz-2);
+	unsigned long int full_size = long(full_nx)*long(full_ny)*long(full_nz);
+	
+	double *LocalID;
+	LocalID = new double [local_size];
+		
+	//printf("aggregate labels: local size=%i, global size = %i",local_size, full_size);
+	// assign the ID for the local sub-region
+	for (int k=1; k<nz-1; k++){
+		for (int j=1; j<ny-1; j++){
+			for (int i=1; i<nx-1; i++){
+				int n = k*nx*ny+j*nx+i;
+				double local_id_val = UserData(i,j,k); 
+				LocalID[(k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1] = local_id_val;
+			}
+		}
+	}
+	MPI_Barrier(Comm);
+
+	// populate the FullID 
+	if (rank() == 0){
+		double *FullID;
+		FullID = new double [full_size];
+		// first handle local ID for rank 0
+		for (int k=1; k<nz-1; k++){
+			for (int j=1; j<ny-1; j++){
+				for (int i=1; i<nx-1; i++){
+					int x = i-1;
+					int y = j-1;
+					int z = k-1;
+					int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1;
+					unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x;
+					FullID[n_full] = LocalID[n_local];
+				}
+			}
+		}
+		// next get the local ID from the other ranks
+		for (int rnk = 1; rnk<nprocs; rnk++){
+			ipz = rnk / (npx*npy);
+			ipy = (rnk - ipz*npx*npy) / npx;
+			ipx = (rnk - ipz*npx*npy - ipy*npx); 
+			//printf("ipx=%i ipy=%i ipz=%i\n", ipx, ipy, ipz);
+			int tag = 15+rnk;
+			MPI_Recv(LocalID,local_size,MPI_DOUBLE,rnk,tag,Comm,MPI_STATUS_IGNORE);
+			for (int k=1; k<nz-1; k++){
+				for (int j=1; j<ny-1; j++){
+					for (int i=1; i<nx-1; i++){
+						int x = i-1 + ipx*(nx-2);
+						int y = j-1 + ipy*(ny-2);
+						int z = k-1 + ipz*(nz-2);
+						int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1;
+						unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x;
+						FullID[n_full] = LocalID[n_local];
+					}
+				}
+			}
+		}
+		// write the output
+		FILE *OUTFILE = fopen(filename.c_str(),"wb");
+		fwrite(FullID,8,full_size,OUTFILE);
+		fclose(OUTFILE);
+	}
+	else{
+		// send LocalID to rank=0
+		int tag = 15+ rank();
+		int dstrank = 0;
+		MPI_Send(LocalID,local_size,MPI_DOUBLE,dstrank,tag,Comm);
+	}
+	MPI_Barrier(Comm);
+
+}

common/Domain.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -199,6 +198,7 @@ public: // Public variables (need to create accessors instead)
     void CommInit(); 
     int PoreCount();
     void AggregateLabels( const std::string& filename );
+    void AggregateLabels( const std::string& filename, DoubleArray &UserData );
 
 private:
 

common/FunctionTable.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/FunctionTable.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/MPI_Helpers.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/MPI_Helpers.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/MPI_Helpers.hpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/ScaLBL.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -94,6 +93,7 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
 	nprocy = Dm->nprocy();
 	nprocz = Dm->nprocz();
 	BoundaryCondition = Dm->BoundaryCondition;
+	//BoundaryCondition = 0; // default to periodic BC
 	//......................................................................................
 
 	ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 2*5*sendCount_x*sizeof(double));	// Allocate device memory
@@ -388,7 +388,11 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 	for (k=0;k<Nz;k++){
 		for (j=0;j<Ny;j++){
 			for (i=0;i<Nx;i++){
-				Map(i,j,k) = -2;
+				n = k*Nx*Ny + j*Nx + i;
+				if (id[n] > 0)
+					Map(i,j,k) = -2; // this label is for parallel communication sites
+				else
+					Map(i,j,k) = -1; // this label is for solid bounce-back sites
 			}
 		}
 	}
@@ -538,7 +542,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 			}
 		}
 	}
-
+	
 	//for (idx=0; idx<Np; idx++)	printf("%i: %i %i\n", idx, neighborList[Np],  neighborList[Np+idx]);
 	//.......................................................................
 	// Now map through  SendList and RecvList to update indices
@@ -865,6 +869,239 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 	return(Np);
 }
 
+
+void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, int Np)
+{
+
+    int idx,i,j,k;
+    int neighbor;    
+	// save list of bounce-back distributions and interaction sites
+	n_bb_d3q7 = 0; n_bb_d3q19 = 0;
+	
+	int local_count = 0;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					neighbor=Map(i-1,j,k);
+					if (neighbor==-1)	  local_count++;
+
+					neighbor=Map(i+1,j,k);
+					if (neighbor==-1)	  local_count++;
+
+					neighbor=Map(i,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j,k-1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i+1,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i+1,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j,k-1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i+1,j,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j,k+1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i+1,j,k-1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i,j-1,k-1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j-1,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k-1);
+					if (neighbor==-1)	    local_count++;
+				}
+			}
+		}
+	}
+	
+	int *bb_dist_tmp = new int [local_count];	
+	int *bb_interactions_tmp = new int [local_count];	
+	ScaLBL_AllocateDeviceMemory((void **) &bb_dist, sizeof(int)*local_count);
+	ScaLBL_AllocateDeviceMemory((void **) &bb_interactions, sizeof(int)*local_count);
+	
+	local_count=0;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					int neighbor;    // cycle through the neighbors of lattice site idx
+					neighbor=Map(i-1,j,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 2*Np;
+					}
+
+					neighbor=Map(i+1,j,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++] = idx + 1*Np;
+					}
+
+					neighbor=Map(i,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 4*Np;
+					}
+
+					neighbor=Map(i,j+1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 3*Np;
+					}
+
+					neighbor=Map(i,j,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 6*Np;
+					}
+
+					neighbor=Map(i,j,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 5*Np;
+					}
+				}
+			}
+		}
+	}
+	n_bb_d3q7 = local_count;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					neighbor=Map(i-1,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 8*Np;
+					}
+
+					neighbor=Map(i+1,j+1,k);
+					if (neighbor==-1)	{
+						bb_interactions_tmp[local_count] = (i+1) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 7*Np;
+					}
+
+					neighbor=Map(i-1,j+1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 10*Np;
+					}
+
+					neighbor=Map(i+1,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 9*Np;
+					}
+
+					neighbor=Map(i-1,j,k-1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 12*Np;
+					}
+
+					neighbor=Map(i+1,j,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 11*Np;
+					}
+
+					neighbor=Map(i-1,j,k+1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 14*Np;
+					}
+
+					neighbor=Map(i+1,j,k-1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 13*Np;
+					}
+
+					neighbor=Map(i,j-1,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 16*Np;
+					}
+
+					neighbor=Map(i,j+1,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 15*Np;
+					}
+
+					neighbor=Map(i,j-1,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 18*Np;
+					}
+
+					neighbor=Map(i,j+1,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 17*Np;
+					}
+				}
+			}
+		}
+	}
+	n_bb_d3q19 = local_count; // this gives the d3q19 distributions not part of d3q7 model
+	ScaLBL_CopyToDevice(bb_dist, bb_dist_tmp, local_count*sizeof(int));
+	ScaLBL_CopyToDevice(bb_interactions, bb_interactions_tmp, local_count*sizeof(int));
+	ScaLBL_DeviceBarrier();
+    
+	delete [] bb_dist_tmp;
+	delete [] bb_interactions_tmp;
+}
+
+void ScaLBL_Communicator::SolidDirichletD3Q7(double *fq, double *BoundaryValue){
+	// fq is a D3Q7 distribution
+	// BoundaryValues is a list of values to assign at bounce-back solid sites
+    ScaLBL_Solid_Dirichlet_D3Q7(fq,BoundaryValue,bb_dist,bb_interactions,n_bb_d3q7);
+}
+
+void ScaLBL_Communicator::SolidNeumannD3Q7(double *fq, double *BoundaryValue){
+	// fq is a D3Q7 distribution
+	// BoundaryValues is a list of values to assign at bounce-back solid sites
+    ScaLBL_Solid_Neumann_D3Q7(fq,BoundaryValue,bb_dist,bb_interactions,n_bb_d3q7);
+}
+
+
 void ScaLBL_Communicator::SendD3Q19AA(double *dist){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
@@ -1192,12 +1429,120 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
 	//...................................................................................
 
 }
+/**
+ * 
+ */
+void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){
+
+	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
+	if (Lock==true){
+		ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+	}
+	else{
+		Lock=true;
+	}
+	// assign tag of 19 to D3Q19 communication
+	sendtag = recvtag = 7;
+	ScaLBL_DeviceBarrier();
+	// Pack the distributions
+	//...Packing for x face(2,8,10,12,14)................................
+	ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
+	MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
+	
+	//...Packing for X face(1,7,9,11,13)................................
+	ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
+	
+	MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
+	MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
+
+	//...Packing for y face(4,8,9,16,18).................................
+	ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
+	MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
+	
+	//...Packing for Y face(3,7,10,15,17).................................
+	ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
+	MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
+	
+	//...Packing for z face(6,12,13,16,17)................................
+	ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
+	
+	MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
+	MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
+	
+	//...Packing for Z face(5,11,14,15,18)................................
+	ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
+
+	//...................................................................................
+	// Send all the distributions
+	MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
+	MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
+
+}
+
+
+void ScaLBL_Communicator::RecvD3Q7AA(double *Aq, int Component){
+
+	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
+	//...................................................................................
+	// Wait for completion of D3Q19 communication
+	MPI_Waitall(6,req1,stat1);
+	MPI_Waitall(6,req2,stat2);
+	ScaLBL_DeviceBarrier();
+
+	//...................................................................................
+	// NOTE: AA Routine writes to opposite
+	// Unpack the distributions on the device
+	//...................................................................................
+	//...Unpacking for x face(2,8,10,12,14)................................
+	ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for X face(1,7,9,11,13)................................
+	ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for y face(4,8,9,16,18).................................
+	ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for Y face(3,7,10,15,17).................................
+	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,&Aq[Component*7*N],N);
+	//...................................................................................
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N);
+		}
+	}
+	else {
+		//...Packing for z face(6,12,13,16,17)................................
+		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N);
+		//...Packing for Z face(5,11,14,15,18)................................
+		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N);
+	}
+
+	//...................................................................................
+	Lock=false; // unlock the communicator after communications complete
+	//...................................................................................
+
+}
+/* 
+ */
+
 
 void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
 	if (Lock==true){
-		ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+		ERROR("ScaLBL Error (BiSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
 	}
 	else{
 		Lock=true;
@@ -1324,7 +1669,7 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
 	if (Lock==true){
-		ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+		ERROR("ScaLBL Error (TriSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
 	}
 	else{
 		Lock=true;
@@ -1725,3 +2070,80 @@ void ScaLBL_Communicator::PrintD3Q19(){
 	delete [] TempBuffer;
 }
 
+void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(dvcSendList_z, fq, Vin, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(neighborList, dvcSendList_z, fq, Vin, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(dvcSendList_Z, fq, Vout, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(neighborList, dvcSendList_Z, fq, Vout, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin){
+	if (kproc == 0) {
+		ScaLBL_Poisson_D3Q7_BC_z(dvcSendList_z, Map, Psi, Vin, sendCount_z);
+	}
+}
+
+void ScaLBL_Communicator::Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout){
+	if (kproc == nprocz-1){
+		ScaLBL_Poisson_D3Q7_BC_Z(dvcSendList_Z, Map, Psi, Vout, sendCount_Z);
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(dvcSendList_z, fq, Cin, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(neighborList, dvcSendList_z, fq, Cin, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(dvcSendList_Z, fq, Cout, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(neighborList, dvcSendList_Z, fq, Cout, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+	}
+}

common/ScaLBL.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -62,7 +61,6 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int
 
 extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np);
 
-
 extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np);
 
 extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np);
@@ -87,12 +85,108 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int
 
 extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+// ION TRANSPORT MODEL
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
+
+// LBM Poisson solver
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+//maybe deprecated
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np);
+
+// LBM Stokes Model (adapted from MRT model)
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
                                               double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
 
 extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+        double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
+
+// LBM Poisson solver
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+//maybe deprecated
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np);
+
+// LBM Stokes Model (adapted from MRT model)
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
+
 // GREYSCALE FREE-ENERGY MODEL (Two-component)
 
 //extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den,
@@ -202,6 +296,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
         double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
 
+extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np);
+
 // MRT MODEL
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
 		double Fy, double Fz);
@@ -210,7 +306,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int st
 		double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz);
 
 // COLOR MODEL
-
 extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
@@ -278,6 +373,38 @@ extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int
 
 extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination);
 
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np);
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count);
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+
 class ScaLBL_Communicator{
 public:
 	//......................................................................................
@@ -289,6 +416,7 @@ public:
 	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator
 	unsigned long int CommunicationCount,SendCount,RecvCount;
 	int Nx,Ny,Nz,N;
+	int n_bb_d3q7, n_bb_d3q19; 
 	int BoundaryCondition;
 	
 	int next;
@@ -314,8 +442,8 @@ public:
 	int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np);
 	void SendD3Q19AA(double *dist);
 	void RecvD3Q19AA(double *dist);
-//	void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
-//	void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
+	void SendD3Q7AA(double *fq, int Component);
+	void RecvD3Q7AA(double *fq, int Component);
 	void BiSendD3Q7AA(double *Aq, double *Bq);
 	void BiRecvD3Q7AA(double *Aq, double *Bq);
 	void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
@@ -324,6 +452,9 @@ public:
 	void RecvHalo(double *data);
 	void RecvGrad(double *Phi, double *Gradient);
 	void RegularLayout(IntArray map, const double *data, DoubleArray &regdata);
+	void SetupBounceBackList(IntArray &Map, signed char *id, int Np);
+    void SolidDirichletD3Q7(double *fq, double *BoundaryValue);
+    void SolidNeumannD3Q7(double *fq, double *BoundaryValue);
 
 	// Routines to set boundary conditions
 	void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
@@ -333,13 +464,22 @@ public:
 	void D3Q19_Reflection_BC_z(double *fq);
 	void D3Q19_Reflection_BC_Z(double *fq);
 	double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
+	void D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time);
+	void D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time);
+	void Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin);
+	void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout);
+	void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time);
+	void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time);
+	void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
+	void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
 	void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
 	void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
-    void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
-    void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
-//	void TestSendD3Q19(double *f_even, double *f_odd);
-//	void TestRecvD3Q19(double *f_even, double *f_odd);
-
+        void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
+        void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
+	void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
+	void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
+        void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
+        void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
 	// Debugging and unit testing functions
 	void PrintD3Q19();
 
@@ -393,6 +533,9 @@ private:
 	int *dvcRecvDist_xy, *dvcRecvDist_yz, *dvcRecvDist_xz, *dvcRecvDist_Xy, *dvcRecvDist_Yz, *dvcRecvDist_xZ;
 	int *dvcRecvDist_xY, *dvcRecvDist_yZ, *dvcRecvDist_Xz, *dvcRecvDist_XY, *dvcRecvDist_YZ, *dvcRecvDist_XZ;
 	//......................................................................................
+	int *bb_dist;
+	int *bb_interactions;
+	//......................................................................................
 
 };
 

common/SpherePack.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/SpherePack.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/UnitTest.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/UnitTest.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Units.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Units.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Utilities.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Utilities.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/UtilityMacros.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/BGK.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/Color.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/D3Q19.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/D3Q7.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/D3Q7BC.cpp

@@ -0,0 +1,347 @@
+// CPU Functions for D3Q7 Lattice Boltzmann Methods
+// Boundary Conditions 
+
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	for (idx=0; idx<N; idx++){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
+	}
+}
+
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	for (idx=0; idx<N; idx++){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = value_q + value_b;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f6 = dist[5*Np+n];
+		//...................................................
+		double f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f5 = dist[6*Np+n];
+		//...................................................
+		double f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
+    int nread,nr5;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		double f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		double f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
+    int nread,nr6;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		double f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		double f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
+{
+	int idx,n,nm;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vin;
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
+{
+	int idx,n,nm;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vout;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f6 = dist[5*Np+n];
+		//...................................................
+		double f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f5 = dist[6*Np+n];
+		//...................................................
+		double f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
+    int nread,nr5;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		double f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		double f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
+    int nread,nr6;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		double f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		double f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    int nread,nr5;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    int nread,nr6;
+    double uz;
+
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}

cpu/Greyscale.cpp

@@ -1974,7 +1974,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist
         }
 
         //Calculate pressure for MRT model
-        pressure=rho/3.f/porosity;
+        //pressure=rho/3.f/porosity;
+        pressure=rho/3.f;
 
         //-------------------- MRT collison where body force has NO higher-order terms -------------//
 		m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);
@@ -2472,7 +2473,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int f
         }
 
         //Calculate pressure for Incompressible-MRT model
-        pressure=rho/3.f/porosity;
+        //pressure=rho/3.f/porosity;
+        pressure=rho/3.f;
 
         //-------------------- IMRT collison where body force has NO higher-order terms -------------//
 		m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);

cpu/GreyscaleColor.cpp

@@ -494,7 +494,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, d
 		Velocity[n] = ux;
 		Velocity[Np+n] = uy;
 		Velocity[2*Np+n] = uz;
-        Pressure[n] = rho/3.f/porosity;
+        //Pressure[n] = rho/3.f/porosity;
+        Pressure[n] = rho/3.f;
 
 		//........................................................................
 		//..............carry out relaxation process..............................
@@ -1149,7 +1150,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
 		Velocity[n] = ux;
 		Velocity[Np+n] = uy;
 		Velocity[2*Np+n] = uz;
-        Pressure[n] = rho/3.f/porosity;
+        //Pressure[n] = rho/3.f/porosity;
+        Pressure[n] = rho/3.f;
 
 		//........................................................................
 		//..............carry out relaxation process..............................
@@ -1336,6 +1338,32 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
 	}
 }
 
+extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
+	int idx;
+	double nA,nB;
+
+	for (idx=start; idx<finish; idx++){
+
+		nA = Den[idx];
+		nB = Den[Np+idx];
+		
+		Aq[idx]=0.3333333333333333*nA;
+		Aq[Np+idx]=0.1111111111111111*nA;
+		Aq[2*Np+idx]=0.1111111111111111*nA;
+		Aq[3*Np+idx]=0.1111111111111111*nA;
+		Aq[4*Np+idx]=0.1111111111111111*nA;
+		Aq[5*Np+idx]=0.1111111111111111*nA;
+		Aq[6*Np+idx]=0.1111111111111111*nA;
+		
+		Bq[idx]=0.3333333333333333*nB;
+		Bq[Np+idx]=0.1111111111111111*nB;
+		Bq[2*Np+idx]=0.1111111111111111*nB;
+		Bq[3*Np+idx]=0.1111111111111111*nB;
+		Bq[4*Np+idx]=0.1111111111111111*nB;
+		Bq[5*Np+idx]=0.1111111111111111*nB;
+		Bq[6*Np+idx]=0.1111111111111111*nB;
+	}
+}
 
 //extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np){
 //	int n;

cpu/Ion.cpp

@@ -0,0 +1,254 @@
+#include <stdio.h>
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+    int n,nread;
+    double fq,Ci;
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+        Ci = fq;
+
+		// q=1
+		nread = neighborList[n]; 
+		fq = dist[nread]; 
+		Ci += fq;
+        
+        // q=2
+		nread = neighborList[n+Np]; 
+		fq = dist[nread];  
+		Ci += fq;
+		
+        // q=3
+		nread = neighborList[n+2*Np]; 
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=4
+		nread = neighborList[n+3*Np]; 
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		Ci += fq;
+
+        Den[n]=Ci;
+    }
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+    int n;
+    double fq,Ci;
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+		Ci = fq;
+		
+		// q=1
+		fq = dist[2*Np+n];
+		Ci += fq;
+
+		// q=2
+		fq = dist[1*Np+n];
+		Ci += fq;
+
+		// q=3
+		fq = dist[4*Np+n];
+		Ci += fq;
+
+		// q=4
+		fq = dist[3*Np+n];
+		Ci += fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		Ci += fq;
+
+		// q=6
+		fq = dist[5*Np+n];
+		Ci += fq;
+
+        Den[n]=Ci;
+    }
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+
+	for (n=start; n<finish; n++){
+		
+        //Load data
+        Ci=Den[n];
+        Ex=ElectricField[n+0*Np];
+        Ey=ElectricField[n+1*Np];
+        Ez=ElectricField[n+2*Np];
+        ux=Velocity[n+0*Np];
+        uy=Velocity[n+1*Np];
+        uz=Velocity[n+2*Np];
+        uEPx=zi*Di/Vt*Ex;
+        uEPy=zi*Di/Vt*Ey;
+        uEPz=zi*Di/Vt*Ez;
+
+		// q=0
+		f0 = dist[n];
+		// q=1
+		nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		f1 = dist[nr1]; // reading the f1 data into register fq
+		// q=2
+		nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		f2 = dist[nr2];  // reading the f2 data into register fq
+		// q=3
+		nr3 = neighborList[n+2*Np]; // neighbor 4
+		f3 = dist[nr3];
+		// q=4
+		nr4 = neighborList[n+3*Np]; // neighbor 3
+		f4 = dist[nr4];
+		// q=5
+		nr5 = neighborList[n+4*Np];
+		f5 = dist[nr5];
+		// q=6
+		nr6 = neighborList[n+5*Np];
+		f6 = dist[nr6];
+		
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+
+		// q = 1
+		dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+
+		// q=2
+		dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+
+		// q = 3
+		dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+
+		// q = 4
+		dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+
+		// q = 5
+		dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+
+		// q = 6
+		dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+
+	for (n=start; n<finish; n++){
+		
+        //Load data
+        Ci=Den[n];
+        Ex=ElectricField[n+0*Np];
+        Ey=ElectricField[n+1*Np];
+        Ez=ElectricField[n+2*Np];
+        ux=Velocity[n+0*Np];
+        uy=Velocity[n+1*Np];
+        uz=Velocity[n+2*Np];
+        uEPx=zi*Di/Vt*Ex;
+        uEPy=zi*Di/Vt*Ey;
+        uEPz=zi*Di/Vt*Ez;
+
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		f6 = dist[5*Np+n];
+		
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+
+		// q = 1
+		dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+
+		// q=2
+		dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+
+		// q = 3
+		dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+
+		// q = 4
+		dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+
+		// q = 5
+		dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+
+		// q = 6
+		dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+
+	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np)
+{
+	int n;
+	for (n=0; n<Np; n++){
+		dist[0*Np+n] = 0.25*DenInit;
+		dist[1*Np+n] = 0.125*DenInit;		
+		dist[2*Np+n] = 0.125*DenInit;	
+		dist[3*Np+n] = 0.125*DenInit;	
+		dist[4*Np+n] = 0.125*DenInit;	
+		dist[5*Np+n] = 0.125*DenInit;	
+		dist[6*Np+n] = 0.125*DenInit;	
+        Den[n] = DenInit;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np)
+{
+	int n;
+    double DenInit;
+	for (n=0; n<Np; n++){
+        DenInit = Den[n];
+		dist[0*Np+n] = 0.25*DenInit;
+		dist[1*Np+n] = 0.125*DenInit;		
+		dist[2*Np+n] = 0.125*DenInit;	
+		dist[3*Np+n] = 0.125*DenInit;	
+		dist[4*Np+n] = 0.125*DenInit;	
+		dist[5*Np+n] = 0.125*DenInit;	
+		dist[6*Np+n] = 0.125*DenInit;	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+    
+    int n;
+    double Ci;//ion concentration of species i
+    double CD;//charge density
+    double CD_tmp;
+    double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
+
+	for (n=start; n<finish; n++){
+        Ci = Den[n+ion_component*Np];
+        CD = ChargeDensity[n];
+        CD_tmp = F*IonValence*Ci;
+        ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
+    }
+}
+

cpu/Poisson.cpp

@@ -0,0 +1,372 @@
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+	int nread;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+        psi = fq;
+
+		// q=1
+		nread = neighborList[n]; 
+		fq = dist[nread];
+		psi += fq;
+		
+		// q=2
+		nread = neighborList[n+Np]; 
+		fq = dist[nread];  
+		psi += fq;
+
+		// q=3
+		nread = neighborList[n+2*Np]; 
+		fq = dist[nread];
+		psi += fq;
+
+		// q = 4
+		nread = neighborList[n+3*Np]; 
+		fq = dist[nread];
+		psi += fq;
+
+		// q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		psi += fq;
+
+		// q = 6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		psi += fq;
+        
+        idx=Map[n];
+        Psi[idx] = psi;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+		psi = fq;
+		
+		// q=1
+		fq = dist[2*Np+n];
+		psi += fq;
+
+		// q=2 
+		fq = dist[1*Np+n];
+		psi += fq;
+
+		// q=3
+		fq = dist[4*Np+n];
+		psi += fq;
+
+		// q=4
+		fq = dist[3*Np+n];
+		psi += fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		psi += fq;
+
+		// q=6
+		fq = dist[5*Np+n];
+		psi += fq;
+
+        idx=Map[n];
+        Psi[idx] = psi;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+    double rlx=1.0/tau;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+        //Load data
+        rho_e = Den_charge[n];
+        rho_e = rho_e/epsilon_LB;
+        idx=Map[n];
+        psi = Psi[idx];
+		
+		// q=0
+		f0 = dist[n];
+		// q=1
+		nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		f1 = dist[nr1]; // reading the f1 data into register fq
+
+		nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		f2 = dist[nr2];  // reading the f2 data into register fq
+
+		// q=3
+		nr3 = neighborList[n+2*Np]; // neighbor 4
+		f3 = dist[nr3];
+
+		// q = 4
+		nr4 = neighborList[n+3*Np]; // neighbor 3
+		f4 = dist[nr4];
+
+		// q=5
+		nr5 = neighborList[n+4*Np];
+		f5 = dist[nr5];
+
+		// q = 6
+		nr6 = neighborList[n+5*Np];
+		f6 = dist[nr6];
+		
+		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ez = (f5-f6)*rlx*4.0;
+        ElectricField[n+0*Np] = Ex;
+        ElectricField[n+1*Np] = Ey;
+        ElectricField[n+2*Np] = Ez;
+
+		// q = 0
+		dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+		// q = 1
+		dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 2
+		dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 3
+		dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 4
+		dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 5
+		dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 6
+		dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+    double rlx=1.0/tau;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+        //Load data
+        rho_e = Den_charge[n];
+        rho_e = rho_e/epsilon_LB;
+        idx=Map[n];
+        psi = Psi[idx];
+
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		f6 = dist[5*Np+n];
+
+
+		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ez = (f5-f6)*rlx*4.0;
+        ElectricField[n+0*Np] = Ex;
+        ElectricField[n+1*Np] = Ey;
+        ElectricField[n+2*Np] = Ez;
+
+		// q = 0
+		dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+		// q = 1
+		dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 2
+		dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 3
+		dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 4
+		dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 5
+		dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 6
+		dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np)
+{
+	int n;
+    int ijk;
+	for (n=start; n<finish; n++){
+        ijk = Map[n];
+		dist[0*Np+n] = 0.25*Psi[ijk];
+		dist[1*Np+n] = 0.125*Psi[ijk];		
+		dist[2*Np+n] = 0.125*Psi[ijk];	
+		dist[3*Np+n] = 0.125*Psi[ijk];	
+		dist[4*Np+n] = 0.125*Psi[ijk];	
+		dist[5*Np+n] = 0.125*Psi[ijk];	
+		dist[6*Np+n] = 0.125*Psi[ijk];	
+	}
+}
+
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np){
+//
+//	int n,nn;
+//    int ijk;
+//    int id;
+//	// distributions
+//	double m1,m2,m3,m4,m5,m6,m7,m8,m9;
+//	double m10,m11,m12,m13,m14,m15,m16,m17,m18;
+//	double nx,ny,nz;
+//
+//	for (n=start; n<finish; n++){
+//
+//		// Get the 1D index based on regular data layout
+//		ijk = Map[n];
+//		//					COMPUTE THE COLOR GRADIENT
+//		//........................................................................
+//		//.................Read Phase Indicator Values............................
+//		//........................................................................
+//		nn = ijk-1;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m1 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1
+//		//........................................................................
+//		nn = ijk+1;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2
+//		//........................................................................
+//		nn = ijk-strideY;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3
+//		//........................................................................
+//		nn = ijk+strideY;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4
+//		//........................................................................
+//		nn = ijk-strideZ;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5
+//		//........................................................................
+//		nn = ijk+strideZ;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6
+//		//........................................................................
+//		nn = ijk-strideY-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7
+//		//........................................................................
+//		nn = ijk+strideY+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8
+//		//........................................................................
+//		nn = ijk+strideY-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9
+//		//........................................................................
+//		nn = ijk-strideY+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10
+//		//........................................................................
+//		nn = ijk-strideZ-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11
+//		//........................................................................
+//		nn = ijk+strideZ+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12
+//		//........................................................................
+//		nn = ijk+strideZ-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13
+//		//........................................................................
+//		nn = ijk-strideZ+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14
+//		//........................................................................
+//		nn = ijk-strideZ-strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15
+//		//........................................................................
+//		nn = ijk+strideZ+strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16
+//		//........................................................................
+//		nn = ijk+strideZ-strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17
+//		//........................................................................
+//		nn = ijk-strideZ+strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18
+//		//............Compute the Color Gradient...................................
+//		//nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14));
+//		//ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18));
+//		//nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18));
+//		nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3
+//		ny = 1.f/6.f*(m3-m4);
+//		nz = 1.f/6.f*(m5-m6);
+//		
+//		ElectricField[n] = nx;
+//		ElectricField[Np+n] = ny;
+//		ElectricField[2*Np+n] = nz;
+//	}
+//}
+
+//extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np){
+//	int n;
+//	// distributions
+//	double f1,f2,f3,f4,f5,f6;
+//	double Ex,Ey,Ez;
+//    double rlx=1.0/tau;
+//
+//	for (n=0; n<Np; n++){
+//		//........................................................................
+//		// Registers to store the distributions
+//		//........................................................................
+//		f1 = dist[Np+n];
+//		f2 = dist[2*Np+n];
+//		f3 = dist[3*Np+n];
+//		f4 = dist[4*Np+n];
+//		f5 = dist[5*Np+n];
+//		f6 = dist[6*Np+n];
+//		//.................Compute the Electric Field...................................
+//		//Ex = (f1-f2)*rlx*4.5;//NOTE the unit of electric field here is V/lu
+//		//Ey = (f3-f4)*rlx*4.5;
+//		//Ez = (f5-f6)*rlx*4.5;
+//		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+//		Ey = (f3-f4)*rlx*4.0;
+//		Ez = (f5-f6)*rlx*4.0;
+//		//..................Write the Electric Field.....................................
+//		ElectricField[0*Np+n] = Ex;
+//		ElectricField[1*Np+n] = Ey;
+//		ElectricField[2*Np+n] = Ez;
+//		//........................................................................
+//	}
+//}

cpu/Stokes.cpp

@@ -0,0 +1,999 @@
+#include <stdio.h>
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np)
+{
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	constexpr double mrt_V1=0.05263157894736842;
+	constexpr double mrt_V2=0.012531328320802;
+	constexpr double mrt_V3=0.04761904761904762;
+	constexpr double mrt_V4=0.004594820384294068;
+	constexpr double mrt_V5=0.01587301587301587;
+	constexpr double mrt_V6=0.0555555555555555555555555;
+	constexpr double mrt_V7=0.02777777777777778;
+	constexpr double mrt_V8=0.08333333333333333;
+	constexpr double mrt_V9=0.003341687552213868;
+	constexpr double mrt_V10=0.003968253968253968;
+	constexpr double mrt_V11=0.01388888888888889;
+	constexpr double mrt_V12=0.04166666666666666;
+
+	for (int n=start; n<finish; n++){
+        
+        //Load data
+        rhoE = ChargeDensity[n];
+        Ex = ElectricField[n+0*Np];
+        Ey = ElectricField[n+1*Np];
+        Ez = ElectricField[n+2*Np];
+        //compute total body force, including input body force (Gx,Gy,Gz)
+        Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
+        Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+		// q=0
+		fq = dist[n];
+		rho = fq;
+		m1  = -30.0*fq;
+		m2  = 12.0*fq;
+
+		// q=1
+		fq = dist[2*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jx = fq;
+		m4 = -4.0*fq;
+		m9 = 2.0*fq;
+		m10 = -4.0*fq;
+
+		// f2 = dist[10*Np+n];
+		fq = dist[1*Np+n];
+		rho += fq;
+		m1 -= 11.0*(fq);
+		m2 -= 4.0*(fq);
+		jx -= fq;
+		m4 += 4.0*(fq);
+		m9 += 2.0*(fq);
+		m10 -= 4.0*(fq);
+
+		// q=3
+		fq = dist[4*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy = fq;
+		m6 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 = fq;
+		m12 = -2.0*fq;
+
+		// q = 4
+		fq = dist[3*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy -= fq;
+		m6 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 += fq;
+		m12 -= 2.0*fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz = fq;
+		m8 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q = 6
+		fq = dist[5*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz -= fq;
+		m8 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q=7
+		fq = dist[8*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy += fq;
+		m6 += fq;
+		m9  += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 = fq;
+		m16 = fq;
+		m17 = -fq;
+
+		// q = 8
+		fq = dist[7*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 += fq;
+		m16 -= fq;
+		m17 += fq;
+
+		// q=9
+		fq = dist[10*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 += fq;
+		m17 += fq;
+
+		// q = 10
+		fq = dist[9*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy += fq;
+		m6 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 -= fq;
+		m17 -= fq;
+
+		// q=11
+		fq = dist[12*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 = fq;
+		m16 -= fq;
+		m18 = fq;
+
+		// q=12
+		fq = dist[11*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 += fq;
+		m16 += fq;
+		m18 -= fq;
+
+		// q=13
+		fq = dist[14*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 -= fq;
+		m18 -= fq;
+
+		// q=14
+		fq = dist[13*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 += fq;
+		m18 += fq;
+
+		// q=15
+		fq = dist[16*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 = fq;
+		m17 += fq;
+		m18 -= fq;
+
+		// q=16
+		fq = dist[15*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 += fq;
+		m17 -= fq;
+		m18 += fq;
+
+		// q=17
+		fq = dist[18*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 += fq;
+		m18 += fq;
+
+		// q=18
+		fq = dist[17*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 -= fq;
+		m18 -= fq;
+
+		// write the velocity 
+		ux = jx / rho0;
+		uy = jy / rho0;
+		uz = jz / rho0;
+		Velocity[n] = ux;
+		Velocity[Np+n] = uy;
+		Velocity[2*Np+n] = uz;
+
+
+		//........................................................................
+		//					READ THE DISTRIBUTIONS
+		//		(read from opposite array due to previous swap operation)
+		//........................................................................
+
+		//..............incorporate external force................................................
+		//..............carry out relaxation process...............................................
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+		m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+		m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+		m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+		m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+		m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+		m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+		m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+		m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+		m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+		m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+		m16 = m16 + rlx_setB*( - m16);
+		m17 = m17 + rlx_setB*( - m17);
+		m18 = m18 + rlx_setB*( - m18);
+		//.......................................................................................................
+		//.................inverse transformation......................................................
+
+		// q=0
+		fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+		dist[n] = fq;
+
+		// q = 1
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
+		dist[1*Np+n] = fq;
+
+		// q=2
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+		dist[2*Np+n] = fq;
+
+		// q = 3
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+		dist[3*Np+n] = fq;
+
+		// q = 4
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+		dist[4*Np+n] = fq;
+
+		// q = 5
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+		dist[5*Np+n] = fq;
+
+		// q = 6
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+		dist[6*Np+n] = fq;
+
+		// q = 7
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+		dist[7*Np+n] = fq;
+
+
+		// q = 8
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+				+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+		dist[8*Np+n] = fq;
+
+		// q = 9
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+		dist[9*Np+n] = fq;
+
+		// q = 10
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+		dist[10*Np+n] = fq;
+
+
+		// q = 11
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+		dist[11*Np+n] = fq;
+
+		// q = 12
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                        								+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                        								-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+		dist[12*Np+n] = fq;
+
+		// q = 13
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+		dist[13*Np+n] = fq;
+
+		// q= 14
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+
+		dist[14*Np+n] = fq;
+
+		// q = 15
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+		dist[15*Np+n] = fq;
+
+		// q = 16
+		fq =  mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+		dist[16*Np+n] = fq;
+
+
+		// q = 17
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+		dist[17*Np+n] = fq;
+
+		// q = 18
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+		dist[18*Np+n] = fq;
+
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np)
+{
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+	int nread;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	constexpr double mrt_V1=0.05263157894736842;
+	constexpr double mrt_V2=0.012531328320802;
+	constexpr double mrt_V3=0.04761904761904762;
+	constexpr double mrt_V4=0.004594820384294068;
+	constexpr double mrt_V5=0.01587301587301587;
+	constexpr double mrt_V6=0.0555555555555555555555555;
+	constexpr double mrt_V7=0.02777777777777778;
+	constexpr double mrt_V8=0.08333333333333333;
+	constexpr double mrt_V9=0.003341687552213868;
+	constexpr double mrt_V10=0.003968253968253968;
+	constexpr double mrt_V11=0.01388888888888889;
+	constexpr double mrt_V12=0.04166666666666666;
+
+	for (int n=start; n<finish; n++){
+
+        //Load data
+        rhoE = ChargeDensity[n];
+        Ex = ElectricField[n+0*Np];
+        Ey = ElectricField[n+1*Np];
+        Ez = ElectricField[n+2*Np];
+        //compute total body force, including input body force (Gx,Gy,Gz)
+        Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+		// q=0
+		fq = dist[n];
+		rho = fq;
+		m1  = -30.0*fq;
+		m2  = 12.0*fq;
+
+		// q=1
+		nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		fq = dist[nread]; // reading the f1 data into register fq
+		//fp = dist[10*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jx = fq;
+		m4 = -4.0*fq;
+		m9 = 2.0*fq;
+		m10 = -4.0*fq;
+
+		// f2 = dist[10*Np+n];
+		nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		fq = dist[nread];  // reading the f2 data into register fq
+		//fq = dist[Np+n];
+		rho += fq;
+		m1 -= 11.0*(fq);
+		m2 -= 4.0*(fq);
+		jx -= fq;
+		m4 += 4.0*(fq);
+		m9 += 2.0*(fq);
+		m10 -= 4.0*(fq);
+
+		// q=3
+		nread = neighborList[n+2*Np]; // neighbor 4
+		fq = dist[nread];
+		//fq = dist[11*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy = fq;
+		m6 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 = fq;
+		m12 = -2.0*fq;
+
+		// q = 4
+		nread = neighborList[n+3*Np]; // neighbor 3
+		fq = dist[nread];
+		//fq = dist[2*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy -= fq;
+		m6 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 += fq;
+		m12 -= 2.0*fq;
+
+		// q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		//fq = dist[12*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz = fq;
+		m8 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+
+		// q = 6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		//fq = dist[3*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz -= fq;
+		m8 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q=7
+		nread = neighborList[n+6*Np];
+		fq = dist[nread];
+		//fq = dist[13*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy += fq;
+		m6 += fq;
+		m9  += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 = fq;
+		m16 = fq;
+		m17 = -fq;
+
+		// q = 8
+		nread = neighborList[n+7*Np];
+		fq = dist[nread];
+		//fq = dist[4*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 += fq;
+		m16 -= fq;
+		m17 += fq;
+
+		// q=9
+		nread = neighborList[n+8*Np];
+		fq = dist[nread];
+		//fq = dist[14*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 += fq;
+		m17 += fq;
+
+		// q = 10
+		nread = neighborList[n+9*Np];
+		fq = dist[nread];
+		//fq = dist[5*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy += fq;
+		m6 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 -= fq;
+		m17 -= fq;
+
+		// q=11
+		nread = neighborList[n+10*Np];
+		fq = dist[nread];
+		//fq = dist[15*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 = fq;
+		m16 -= fq;
+		m18 = fq;
+
+		// q=12
+		nread = neighborList[n+11*Np];
+		fq = dist[nread];
+		//fq = dist[6*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 += fq;
+		m16 += fq;
+		m18 -= fq;
+
+		// q=13
+		nread = neighborList[n+12*Np];
+		fq = dist[nread];
+		//fq = dist[16*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 -= fq;
+		m18 -= fq;
+
+		// q=14
+		nread = neighborList[n+13*Np];
+		fq = dist[nread];
+		//fq = dist[7*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 += fq;
+		m18 += fq;
+
+		// q=15
+		nread = neighborList[n+14*Np];
+		fq = dist[nread];
+		//fq = dist[17*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 = fq;
+		m17 += fq;
+		m18 -= fq;
+
+		// q=16
+		nread = neighborList[n+15*Np];
+		fq = dist[nread];
+		//fq = dist[8*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 += fq;
+		m17 -= fq;
+		m18 += fq;
+
+		// q=17
+		//fq = dist[18*Np+n];
+		nread = neighborList[n+16*Np];
+		fq = dist[nread];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 += fq;
+		m18 += fq;
+
+		// q=18
+		nread = neighborList[n+17*Np];
+		fq = dist[nread];
+		//fq = dist[9*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 -= fq;
+		m18 -= fq;
+
+		// write the velocity 
+		ux = jx / rho0;
+		uy = jy / rho0;
+		uz = jz / rho0;
+		Velocity[n] = ux;
+		Velocity[Np+n] = uy;
+		Velocity[2*Np+n] = uz;
+
+		//..............incorporate external force................................................
+		//..............carry out relaxation process...............................................
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+		m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+		m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+		m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+		m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+		m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+		m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+		m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+		m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+		m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+		m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+		m16 = m16 + rlx_setB*( - m16);
+		m17 = m17 + rlx_setB*( - m17);
+		m18 = m18 + rlx_setB*( - m18);
+		//.......................................................................................................
+		//.................inverse transformation......................................................
+
+		// q=0
+		fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+		dist[n] = fq;
+
+		// q = 1
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
+		nread = neighborList[n+Np];
+		dist[nread] = fq;
+
+		// q=2
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+		nread = neighborList[n];
+		dist[nread] = fq;
+
+		// q = 3
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+		nread = neighborList[n+3*Np];
+		dist[nread] = fq;
+
+		// q = 4
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+		nread = neighborList[n+2*Np];
+		dist[nread] = fq;
+
+		// q = 5
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+		nread = neighborList[n+5*Np];
+		dist[nread] = fq;
+
+		// q = 6
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+		nread = neighborList[n+4*Np];
+		dist[nread] = fq;
+
+		// q = 7
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+		nread = neighborList[n+7*Np];
+		dist[nread] = fq;
+
+		// q = 8
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+				+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+		nread = neighborList[n+6*Np];
+		dist[nread] = fq;
+
+		// q = 9
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+		nread = neighborList[n+9*Np];
+		dist[nread] = fq;
+
+		// q = 10
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+		nread = neighborList[n+8*Np];
+		dist[nread] = fq;
+
+		// q = 11
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+		nread = neighborList[n+11*Np];
+		dist[nread] = fq;
+
+		// q = 12
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                        								+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                        								-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+		nread = neighborList[n+10*Np];
+		dist[nread]= fq;
+
+		// q = 13
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+		nread = neighborList[n+13*Np];
+		dist[nread] = fq;
+
+		// q= 14
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+		nread = neighborList[n+12*Np];
+		dist[nread] = fq;
+
+
+		// q = 15
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+		nread = neighborList[n+15*Np];
+		dist[nread] = fq;
+
+		// q = 16
+		fq =  mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+		nread = neighborList[n+14*Np];
+		dist[nread] = fq;
+
+
+		// q = 17
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+		nread = neighborList[n+17*Np];
+		dist[nread] = fq;
+
+		// q = 18
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+		nread = neighborList[n+16*Np];
+		dist[nread] = fq;
+
+	}
+}
+
+//extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np)
+//{
+//    //h: resolution [um/lu]
+//    //time_conv: time conversion factor [sec/lt]
+//	int n;
+//	// distributions
+//	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
+//	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
+//	double vx,vy,vz;
+//
+//	for (n=0; n<Np; n++){
+//		//........................................................................
+//		// Registers to store the distributions
+//		//........................................................................
+//		f2 = dist[2*Np+n];
+//		f4 = dist[4*Np+n];
+//		f6 = dist[6*Np+n];
+//		f8 = dist[8*Np+n];
+//		f10 = dist[10*Np+n];
+//		f12 = dist[12*Np+n];
+//		f14 = dist[14*Np+n];
+//		f16 = dist[16*Np+n];
+//		f18 = dist[18*Np+n];
+//		//........................................................................
+//		f1 = dist[Np+n];
+//		f3 = dist[3*Np+n];
+//		f5 = dist[5*Np+n];
+//		f7 = dist[7*Np+n];
+//		f9 = dist[9*Np+n];
+//		f11 = dist[11*Np+n];
+//		f13 = dist[13*Np+n];
+//		f15 = dist[15*Np+n];
+//		f17 = dist[17*Np+n];
+//		//.................Compute the velocity...................................
+//		vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+//		vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+//		vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+//		//..................Write the velocity.....................................
+//		vel[0*Np+n] = vx*(h*1.0e-6)/time_conv;
+//		vel[1*Np+n] = vy*(h*1.0e-6)/time_conv;
+//		vel[2*Np+n] = vz*(h*1.0e-6)/time_conv;
+//		//........................................................................
+//	}
+//}

cpu/dfh.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/BGK.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/Color.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/CudaExtras.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/D3Q19.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/D3Q7.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/D3Q7BC.cu

@@ -0,0 +1,537 @@
+#include <math.h>
+#include <stdio.h>
+#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+
+__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
+{
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
+	}
+}
+
+__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
+{
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = value_q + value_b;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+		//...................................................
+		f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		//...................................................
+		f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np)
+{
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np)
+{
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
+{
+	int idx,n,nm;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vin;
+	}
+}
+
+
+__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
+{
+	int idx,n,nm;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vout;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+		//...................................................
+		f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		//...................................................
+		f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np)
+{
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np)
+{
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
+//*************************************************************************
+
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_Solid_Dirichlet_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_Solid_Neumann_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<<GRID,512>>>(list, dist, Vin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<<GRID,512>>>(list, dist, Vout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Vin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Vout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){
+	int GRID = count / 512 + 1;
+    dvc_ScaLBL_Poisson_D3Q7_BC_z<<<GRID,512>>>(list, Map, Psi, Vin, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){
+	int GRID = count / 512 + 1;
+    dvc_ScaLBL_Poisson_D3Q7_BC_Z<<<GRID,512>>>(list, Map, Psi, Vout, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<<GRID,512>>>(list, dist, Cin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<<GRID,512>>>(list, dist, Cout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Cin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Cout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}

gpu/Greyscale.cu

@@ -2006,7 +2006,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *
             }
 
             //Calculate pressure for MRT model
-            pressure=rho/3.f/porosity;
+            //pressure=rho/3.f/porosity;
+            pressure=rho/3.f;
 
             //-------------------- MRT collison where body force has NO higher-order terms -------------//
 			m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);
@@ -2512,7 +2513,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, i
             }
 
             //Calculate pressure for Incompressible-MRT model
-            pressure=rho/3.f/porosity;
+            //pressure=rho/3.f/porosity;
+            pressure=rho/3.f;
 
             //-------------------- IMRT collison where body force has NO higher-order terms -------------//
 			m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);

gpu/GreyscaleColor.cu

@@ -512,7 +512,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma
 			Velocity[n] = ux;
 			Velocity[Np+n] = uy;
 			Velocity[2*Np+n] = uz;
-            Pressure[n] = rho/3.f/porosity;
+            //Pressure[n] = rho/3.f/porosity;
+            Pressure[n] = rho/3.f;
 
 			//........................................................................
 			//..............carry out relaxation process..............................
@@ -1218,7 +1219,8 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist,
 			Velocity[n] = ux;
 			Velocity[Np+n] = uy;
 			Velocity[2*Np+n] = uz;
-            Pressure[n] = rho/3.f/porosity;
+            //Pressure[n] = rho/3.f/porosity;
+            Pressure[n] = rho/3.f;
 
 			//........................................................................
 			//..............carry out relaxation process..............................
@@ -1445,6 +1447,37 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist,
 	}
 }
 
+__global__ void dvc_ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
+	int idx;
+	double nA,nB;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		idx =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (idx<finish) {
+
+			nA = Den[idx];
+			nB = Den[Np+idx];
+
+			Aq[idx]=0.3333333333333333*nA;
+			Aq[Np+idx]=0.1111111111111111*nA;
+			Aq[2*Np+idx]=0.1111111111111111*nA;
+			Aq[3*Np+idx]=0.1111111111111111*nA;
+			Aq[4*Np+idx]=0.1111111111111111*nA;
+			Aq[5*Np+idx]=0.1111111111111111*nA;
+			Aq[6*Np+idx]=0.1111111111111111*nA;
+
+			Bq[idx]=0.3333333333333333*nB;
+			Bq[Np+idx]=0.1111111111111111*nB;
+			Bq[2*Np+idx]=0.1111111111111111*nB;
+			Bq[3*Np+idx]=0.1111111111111111*nB;
+			Bq[4*Np+idx]=0.1111111111111111*nB;
+			Bq[5*Np+idx]=0.1111111111111111*nB;
+			Bq[6*Np+idx]=0.1111111111111111*nB;
+		}
+	}
+}
 ////Model-2&3
 //__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
 //		 double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity, 
@@ -2957,6 +2990,13 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
 	//cudaProfilerStop();
 }
 
+extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
+	dvc_ScaLBL_PhaseField_InitFromRestart<<<NBLOCKS,NTHREADS >>>(Den, Aq, Bq, start, finish, Np); 
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_PhaseField_InitFromRestart: %s \n",cudaGetErrorString(err));
+	}
+}
 ////Model-2&3
 //extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
 //        double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 

gpu/Ion.cu

@@ -0,0 +1,391 @@
+#include <stdio.h>
+#include <math.h>
+//#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+    int n,nread;
+    double fq,Ci;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            Ci = fq;
+
+            // q=1
+            nread = neighborList[n]; 
+            fq = dist[nread]; 
+            Ci += fq;
+            
+            // q=2
+            nread = neighborList[n+Np]; 
+            fq = dist[nread];  
+            Ci += fq;
+            
+            // q=3
+            nread = neighborList[n+2*Np]; 
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=4
+            nread = neighborList[n+3*Np]; 
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=5
+            nread = neighborList[n+4*Np];
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=6
+            nread = neighborList[n+5*Np];
+            fq = dist[nread];
+            Ci += fq;
+
+            Den[n]=Ci;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+    int n;
+    double fq,Ci;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            Ci = fq;
+            
+            // q=1
+            fq = dist[2*Np+n];
+            Ci += fq;
+
+            // q=2
+            fq = dist[1*Np+n];
+            Ci += fq;
+
+            // q=3
+            fq = dist[4*Np+n];
+            Ci += fq;
+
+            // q=4
+            fq = dist[3*Np+n];
+            Ci += fq;
+
+            // q=5
+            fq = dist[6*Np+n];
+            Ci += fq;
+
+            // q=6
+            fq = dist[5*Np+n];
+            Ci += fq;
+
+            Den[n]=Ci;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                           double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            Ci=Den[n];
+            Ex=ElectricField[n+0*Np];
+            Ey=ElectricField[n+1*Np];
+            Ez=ElectricField[n+2*Np];
+            ux=Velocity[n+0*Np];
+            uy=Velocity[n+1*Np];
+            uz=Velocity[n+2*Np];
+            uEPx=zi*Di/Vt*Ex;
+            uEPy=zi*Di/Vt*Ey;
+            uEPz=zi*Di/Vt*Ez;
+
+            // q=0
+            f0 = dist[n];
+            // q=1
+            nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+            f1 = dist[nr1]; // reading the f1 data into register fq
+            // q=2
+            nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+            f2 = dist[nr2];  // reading the f2 data into register fq
+            // q=3
+            nr3 = neighborList[n+2*Np]; // neighbor 4
+            f3 = dist[nr3];
+            // q=4
+            nr4 = neighborList[n+3*Np]; // neighbor 3
+            f4 = dist[nr4];
+            // q=5
+            nr5 = neighborList[n+4*Np];
+            f5 = dist[nr5];
+            // q=6
+            nr6 = neighborList[n+5*Np];
+            f6 = dist[nr6];
+            
+            // q=0
+            dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+            //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 1
+            dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+            //dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q=2
+            dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+            //dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 3
+            dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+            //dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 4
+            dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+            //dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 5
+            dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+            //dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 6
+            dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+            //dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                            double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            Ci=Den[n];
+            Ex=ElectricField[n+0*Np];
+            Ey=ElectricField[n+1*Np];
+            Ez=ElectricField[n+2*Np];
+            ux=Velocity[n+0*Np];
+            uy=Velocity[n+1*Np];
+            uz=Velocity[n+2*Np];
+            uEPx=zi*Di/Vt*Ex;
+            uEPy=zi*Di/Vt*Ey;
+            uEPz=zi*Di/Vt*Ez;
+
+            f0 = dist[n];
+            f1 = dist[2*Np+n];
+            f2 = dist[1*Np+n];
+            f3 = dist[4*Np+n];
+            f4 = dist[3*Np+n];
+            f5 = dist[6*Np+n];
+            f6 = dist[5*Np+n];
+            
+            // q=0
+            dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+            //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 1
+            dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+            //dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q=2
+            dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+            //dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 3
+            dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+            //dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 4
+            dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+            //dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 5
+            dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+            //dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 6
+            dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+            //dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
+
+	int n;
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
+		if (n<Np) {
+            dist[0*Np+n] = 0.25*DenInit;
+            dist[1*Np+n] = 0.125*DenInit;		
+            dist[2*Np+n] = 0.125*DenInit;	
+            dist[3*Np+n] = 0.125*DenInit;	
+            dist[4*Np+n] = 0.125*DenInit;	
+            dist[5*Np+n] = 0.125*DenInit;	
+            dist[6*Np+n] = 0.125*DenInit;	
+            Den[n] = DenInit;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
+
+	int n;
+    double DenInit;
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
+		if (n<Np) {
+            DenInit = Den[n];
+            dist[0*Np+n] = 0.25*DenInit;
+            dist[1*Np+n] = 0.125*DenInit;		
+            dist[2*Np+n] = 0.125*DenInit;	
+            dist[3*Np+n] = 0.125*DenInit;	
+            dist[4*Np+n] = 0.125*DenInit;	
+            dist[5*Np+n] = 0.125*DenInit;	
+            dist[6*Np+n] = 0.125*DenInit;	
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+
+    int n;
+    double Ci;//ion concentration of species i
+    double CD;//charge density
+    double CD_tmp;
+    double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+            Ci = Den[n+ion_component*Np];
+            CD = ChargeDensity[n];
+            CD_tmp = F*IonValence*Ci;
+            ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
+		}
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<<NBLOCKS,NTHREADS >>>(dist,Den,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,  
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_Init<<<NBLOCKS,NTHREADS >>>(dist,Den,DenInit,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_Init: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_Init_FromFile<<<NBLOCKS,NTHREADS >>>(dist,Den,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<<NBLOCKS,NTHREADS >>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}

gpu/Poisson.cu

@@ -0,0 +1,330 @@
+#include <stdio.h>
+#include <math.h>
+//#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+	int nread;
+    int idx;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            psi = fq;
+
+            // q=1
+            nread = neighborList[n]; 
+            fq = dist[nread];
+            psi += fq;
+            
+            // q=2
+            nread = neighborList[n+Np]; 
+            fq = dist[nread];  
+            psi += fq;
+
+            // q=3
+            nread = neighborList[n+2*Np]; 
+            fq = dist[nread];
+            psi += fq;
+
+            // q = 4
+            nread = neighborList[n+3*Np]; 
+            fq = dist[nread];
+            psi += fq;
+
+            // q=5
+            nread = neighborList[n+4*Np];
+            fq = dist[nread];
+            psi += fq;
+
+            // q = 6
+            nread = neighborList[n+5*Np];
+            fq = dist[nread];
+            psi += fq;
+            
+            idx=Map[n];
+            Psi[idx] = psi;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+    int idx;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            psi = fq;
+            
+            // q=1
+            fq = dist[2*Np+n];
+            psi += fq;
+
+            // q=2 
+            fq = dist[1*Np+n];
+            psi += fq;
+
+            // q=3
+            fq = dist[4*Np+n];
+            psi += fq;
+
+            // q=4
+            fq = dist[3*Np+n];
+            psi += fq;
+
+            // q=5
+            fq = dist[6*Np+n];
+            psi += fq;
+
+            // q=6
+            fq = dist[5*Np+n];
+            psi += fq;
+
+            idx=Map[n];
+            Psi[idx] = psi;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+    double rlx=1.0/tau;
+    int idx;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            rho_e = Den_charge[n];
+            rho_e = rho_e/epsilon_LB;
+            idx=Map[n];
+            psi = Psi[idx];
+            
+            // q=0
+            f0 = dist[n];
+            // q=1
+            nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+            f1 = dist[nr1]; // reading the f1 data into register fq
+
+            nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+            f2 = dist[nr2];  // reading the f2 data into register fq
+
+            // q=3
+            nr3 = neighborList[n+2*Np]; // neighbor 4
+            f3 = dist[nr3];
+
+            // q = 4
+            nr4 = neighborList[n+3*Np]; // neighbor 3
+            f4 = dist[nr4];
+
+            // q=5
+            nr5 = neighborList[n+4*Np];
+            f5 = dist[nr5];
+
+            // q = 6
+            nr6 = neighborList[n+5*Np];
+            f6 = dist[nr6];
+            
+            Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+            Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+            Ez = (f5-f6)*rlx*4.0;
+            ElectricField[n+0*Np] = Ex;
+            ElectricField[n+1*Np] = Ey;
+            ElectricField[n+2*Np] = Ez;
+
+            // q = 0
+            dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+            // q = 1
+            dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 2
+            dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 3
+            dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 4
+            dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 5
+            dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 6
+            dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+            //........................................................................
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+    double rlx=1.0/tau;
+    int idx;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            rho_e = Den_charge[n];
+            rho_e = rho_e/epsilon_LB;
+            idx=Map[n];
+            psi = Psi[idx];
+
+            f0 = dist[n];
+            f1 = dist[2*Np+n];
+            f2 = dist[1*Np+n];
+            f3 = dist[4*Np+n];
+            f4 = dist[3*Np+n];
+            f5 = dist[6*Np+n];
+            f6 = dist[5*Np+n];
+
+
+            Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+            Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+            Ez = (f5-f6)*rlx*4.0;
+            ElectricField[n+0*Np] = Ex;
+            ElectricField[n+1*Np] = Ey;
+            ElectricField[n+2*Np] = Ez;
+
+            // q = 0
+            dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+            // q = 1
+            dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+            // q = 2
+            dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+            // q = 3
+            dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+            // q = 4
+            dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+            // q = 5
+            dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+            // q = 6
+            dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+            //........................................................................
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+
+	int n;
+    int ijk;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+            ijk = Map[n];
+            dist[0*Np+n] = 0.25*Psi[ijk];
+            dist[1*Np+n] = 0.125*Psi[ijk];		
+            dist[2*Np+n] = 0.125*Psi[ijk];	
+            dist[3*Np+n] = 0.125*Psi[ijk];	
+            dist[4*Np+n] = 0.125*Psi[ijk];	
+            dist[5*Np+n] = 0.125*Psi[ijk];	
+            dist[6*Np+n] = 0.125*Psi[ijk];	
+		}
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Psi,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_Poisson<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_Poisson<<<NBLOCKS,NTHREADS >>>(Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Poisson_Init<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Poisson_Init: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}

gpu/Stokes.cu

@@ -0,0 +1,995 @@
+#include <stdio.h>
+#include <math.h>
+//#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+__global__  void dvc_ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np){
+
+    int n;
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+	int nread;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	const double mrt_V1=0.05263157894736842;
+	const double mrt_V2=0.012531328320802;
+	const double mrt_V3=0.04761904761904762;
+	const double mrt_V4=0.004594820384294068;
+	const double mrt_V5=0.01587301587301587;
+	const double mrt_V6=0.0555555555555555555555555;
+	const double mrt_V7=0.02777777777777778;
+	const double mrt_V8=0.08333333333333333;
+	const double mrt_V9=0.003341687552213868;
+	const double mrt_V10=0.003968253968253968;
+	const double mrt_V11=0.01388888888888889;
+	const double mrt_V12=0.04166666666666666;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            rhoE = ChargeDensity[n];
+            Ex = ElectricField[n+0*Np];
+            Ey = ElectricField[n+1*Np];
+            Ez = ElectricField[n+2*Np];
+            //compute total body force, including input body force (Gx,Gy,Gz)
+            Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+            Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+            Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+            // q=0
+            fq = dist[n];
+            rho = fq;
+            m1  = -30.0*fq;
+            m2  = 12.0*fq;
+
+            // q=1
+            nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+            fq = dist[nread]; // reading the f1 data into register fq
+            //fp = dist[10*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jx = fq;
+            m4 = -4.0*fq;
+            m9 = 2.0*fq;
+            m10 = -4.0*fq;
+
+            // f2 = dist[10*Np+n];
+            nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+            fq = dist[nread];  // reading the f2 data into register fq
+            //fq = dist[Np+n];
+            rho += fq;
+            m1 -= 11.0*(fq);
+            m2 -= 4.0*(fq);
+            jx -= fq;
+            m4 += 4.0*(fq);
+            m9 += 2.0*(fq);
+            m10 -= 4.0*(fq);
+
+            // q=3
+            nread = neighborList[n+2*Np]; // neighbor 4
+            fq = dist[nread];
+            //fq = dist[11*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jy = fq;
+            m6 = -4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 = fq;
+            m12 = -2.0*fq;
+
+            // q = 4
+            nread = neighborList[n+3*Np]; // neighbor 3
+            fq = dist[nread];
+            //fq = dist[2*Np+n];
+            rho+= fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jy -= fq;
+            m6 += 4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 += fq;
+            m12 -= 2.0*fq;
+
+            // q=5
+            nread = neighborList[n+4*Np];
+            fq = dist[nread];
+            //fq = dist[12*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jz = fq;
+            m8 = -4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 -= fq;
+            m12 += 2.0*fq;
+
+
+            // q = 6
+            nread = neighborList[n+5*Np];
+            fq = dist[nread];
+            //fq = dist[3*Np+n];
+            rho+= fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jz -= fq;
+            m8 += 4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 -= fq;
+            m12 += 2.0*fq;
+
+            // q=7
+            nread = neighborList[n+6*Np];
+            fq = dist[nread];
+            //fq = dist[13*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jy += fq;
+            m6 += fq;
+            m9  += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 = fq;
+            m16 = fq;
+            m17 = -fq;
+
+            // q = 8
+            nread = neighborList[n+7*Np];
+            fq = dist[nread];
+            //fq = dist[4*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jy -= fq;
+            m6 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 += fq;
+            m16 -= fq;
+            m17 += fq;
+
+            // q=9
+            nread = neighborList[n+8*Np];
+            fq = dist[nread];
+            //fq = dist[14*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jy -= fq;
+            m6 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 -= fq;
+            m16 += fq;
+            m17 += fq;
+
+            // q = 10
+            nread = neighborList[n+9*Np];
+            fq = dist[nread];
+            //fq = dist[5*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jy += fq;
+            m6 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 -= fq;
+            m16 -= fq;
+            m17 -= fq;
+
+            // q=11
+            nread = neighborList[n+10*Np];
+            fq = dist[nread];
+            //fq = dist[15*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jz += fq;
+            m8 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 = fq;
+            m16 -= fq;
+            m18 = fq;
+
+            // q=12
+            nread = neighborList[n+11*Np];
+            fq = dist[nread];
+            //fq = dist[6*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 += fq;
+            m16 += fq;
+            m18 -= fq;
+
+            // q=13
+            nread = neighborList[n+12*Np];
+            fq = dist[nread];
+            //fq = dist[16*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 -= fq;
+            m16 -= fq;
+            m18 -= fq;
+
+            // q=14
+            nread = neighborList[n+13*Np];
+            fq = dist[nread];
+            //fq = dist[7*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jz += fq;
+            m8 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 -= fq;
+            m16 += fq;
+            m18 += fq;
+
+            // q=15
+            nread = neighborList[n+14*Np];
+            fq = dist[nread];
+            //fq = dist[17*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy += fq;
+            m6 += fq;
+            jz += fq;
+            m8 += fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 = fq;
+            m17 += fq;
+            m18 -= fq;
+
+            // q=16
+            nread = neighborList[n+15*Np];
+            fq = dist[nread];
+            //fq = dist[8*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy -= fq;
+            m6 -= fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 += fq;
+            m17 -= fq;
+            m18 += fq;
+
+            // q=17
+            //fq = dist[18*Np+n];
+            nread = neighborList[n+16*Np];
+            fq = dist[nread];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy += fq;
+            m6 += fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 -= fq;
+            m17 += fq;
+            m18 += fq;
+
+            // q=18
+            nread = neighborList[n+17*Np];
+            fq = dist[nread];
+            //fq = dist[9*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy -= fq;
+            m6 -= fq;
+            jz += fq;
+            m8 += fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 -= fq;
+            m17 -= fq;
+            m18 -= fq;
+
+            // write the velocity 
+            ux = jx / rho0;
+            uy = jy / rho0;
+            uz = jz / rho0;
+            Velocity[n] = ux;
+            Velocity[Np+n] = uy;
+            Velocity[2*Np+n] = uz;
+
+            //..............incorporate external force................................................
+            //..............carry out relaxation process...............................................
+            m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+            m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+            m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+            m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+            m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+            m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+            m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+            m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+            m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+            m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+            m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+            m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+            m16 = m16 + rlx_setB*( - m16);
+            m17 = m17 + rlx_setB*( - m17);
+            m18 = m18 + rlx_setB*( - m18);
+            //.......................................................................................................
+            //.................inverse transformation......................................................
+
+            // q=0
+            fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+            dist[n] = fq;
+
+            // q = 1
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
+            nread = neighborList[n+Np];
+            dist[nread] = fq;
+
+            // q=2
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+            nread = neighborList[n];
+            dist[nread] = fq;
+
+            // q = 3
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+            nread = neighborList[n+3*Np];
+            dist[nread] = fq;
+
+            // q = 4
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+            nread = neighborList[n+2*Np];
+            dist[nread] = fq;
+
+            // q = 5
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+            nread = neighborList[n+5*Np];
+            dist[nread] = fq;
+
+            // q = 6
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+            nread = neighborList[n+4*Np];
+            dist[nread] = fq;
+
+            // q = 7
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+            nread = neighborList[n+7*Np];
+            dist[nread] = fq;
+
+            // q = 8
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                    +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+            nread = neighborList[n+6*Np];
+            dist[nread] = fq;
+
+            // q = 9
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+            nread = neighborList[n+9*Np];
+            dist[nread] = fq;
+
+            // q = 10
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+            nread = neighborList[n+8*Np];
+            dist[nread] = fq;
+
+            // q = 11
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+            nread = neighborList[n+11*Np];
+            dist[nread] = fq;
+
+            // q = 12
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                                                            +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                                                            -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+            nread = neighborList[n+10*Np];
+            dist[nread]= fq;
+
+            // q = 13
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+            nread = neighborList[n+13*Np];
+            dist[nread] = fq;
+
+            // q= 14
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+            nread = neighborList[n+12*Np];
+            dist[nread] = fq;
+
+
+            // q = 15
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+                    -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+            nread = neighborList[n+15*Np];
+            dist[nread] = fq;
+
+            // q = 16
+            fq =  mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+                    -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+            nread = neighborList[n+14*Np];
+            dist[nread] = fq;
+
+
+            // q = 17
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+                    -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+            nread = neighborList[n+17*Np];
+            dist[nread] = fq;
+
+            // q = 18
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+                    -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+            nread = neighborList[n+16*Np];
+            dist[nread] = fq;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
+
+    int n;
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	const double mrt_V1=0.05263157894736842;
+	const double mrt_V2=0.012531328320802;
+	const double mrt_V3=0.04761904761904762;
+	const double mrt_V4=0.004594820384294068;
+	const double mrt_V5=0.01587301587301587;
+	const double mrt_V6=0.0555555555555555555555555;
+	const double mrt_V7=0.02777777777777778;
+	const double mrt_V8=0.08333333333333333;
+	const double mrt_V9=0.003341687552213868;
+	const double mrt_V10=0.003968253968253968;
+	const double mrt_V11=0.01388888888888889;
+	const double mrt_V12=0.04166666666666666;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            rhoE = ChargeDensity[n];
+            Ex = ElectricField[n+0*Np];
+            Ey = ElectricField[n+1*Np];
+            Ez = ElectricField[n+2*Np];
+            //compute total body force, including input body force (Gx,Gy,Gz)
+            Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
+            Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+            Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+            // q=0
+            fq = dist[n];
+            rho = fq;
+            m1  = -30.0*fq;
+            m2  = 12.0*fq;
+
+            // q=1
+            fq = dist[2*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jx = fq;
+            m4 = -4.0*fq;
+            m9 = 2.0*fq;
+            m10 = -4.0*fq;
+
+            // f2 = dist[10*Np+n];
+            fq = dist[1*Np+n];
+            rho += fq;
+            m1 -= 11.0*(fq);
+            m2 -= 4.0*(fq);
+            jx -= fq;
+            m4 += 4.0*(fq);
+            m9 += 2.0*(fq);
+            m10 -= 4.0*(fq);
+
+            // q=3
+            fq = dist[4*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jy = fq;
+            m6 = -4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 = fq;
+            m12 = -2.0*fq;
+
+            // q = 4
+            fq = dist[3*Np+n];
+            rho+= fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jy -= fq;
+            m6 += 4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 += fq;
+            m12 -= 2.0*fq;
+
+            // q=5
+            fq = dist[6*Np+n];
+            rho += fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jz = fq;
+            m8 = -4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 -= fq;
+            m12 += 2.0*fq;
+
+            // q = 6
+            fq = dist[5*Np+n];
+            rho+= fq;
+            m1 -= 11.0*fq;
+            m2 -= 4.0*fq;
+            jz -= fq;
+            m8 += 4.0*fq;
+            m9 -= fq;
+            m10 += 2.0*fq;
+            m11 -= fq;
+            m12 += 2.0*fq;
+
+            // q=7
+            fq = dist[8*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jy += fq;
+            m6 += fq;
+            m9  += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 = fq;
+            m16 = fq;
+            m17 = -fq;
+
+            // q = 8
+            fq = dist[7*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jy -= fq;
+            m6 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 += fq;
+            m16 -= fq;
+            m17 += fq;
+
+            // q=9
+            fq = dist[10*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jy -= fq;
+            m6 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 -= fq;
+            m16 += fq;
+            m17 += fq;
+
+            // q = 10
+            fq = dist[9*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jy += fq;
+            m6 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 += fq;
+            m12 += fq;
+            m13 -= fq;
+            m16 -= fq;
+            m17 -= fq;
+
+            // q=11
+            fq = dist[12*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jz += fq;
+            m8 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 = fq;
+            m16 -= fq;
+            m18 = fq;
+
+            // q=12
+            fq = dist[11*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 += fq;
+            m16 += fq;
+            m18 -= fq;
+
+            // q=13
+            fq = dist[14*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx += fq;
+            m4 += fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 -= fq;
+            m16 -= fq;
+            m18 -= fq;
+
+            // q=14
+            fq = dist[13*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jx -= fq;
+            m4 -= fq;
+            jz += fq;
+            m8 += fq;
+            m9 += fq;
+            m10 += fq;
+            m11 -= fq;
+            m12 -= fq;
+            m15 -= fq;
+            m16 += fq;
+            m18 += fq;
+
+            // q=15
+            fq = dist[16*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy += fq;
+            m6 += fq;
+            jz += fq;
+            m8 += fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 = fq;
+            m17 += fq;
+            m18 -= fq;
+
+            // q=16
+            fq = dist[15*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy -= fq;
+            m6 -= fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 += fq;
+            m17 -= fq;
+            m18 += fq;
+
+            // q=17
+            fq = dist[18*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy += fq;
+            m6 += fq;
+            jz -= fq;
+            m8 -= fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 -= fq;
+            m17 += fq;
+            m18 += fq;
+
+            // q=18
+            fq = dist[17*Np+n];
+            rho += fq;
+            m1 += 8.0*fq;
+            m2 += fq;
+            jy -= fq;
+            m6 -= fq;
+            jz += fq;
+            m8 += fq;
+            m9 -= 2.0*fq;
+            m10 -= 2.0*fq;
+            m14 -= fq;
+            m17 -= fq;
+            m18 -= fq;
+
+            // write the velocity 
+            ux = jx / rho0;
+            uy = jy / rho0;
+            uz = jz / rho0;
+            Velocity[n] = ux;
+            Velocity[Np+n] = uy;
+            Velocity[2*Np+n] = uz;
+
+
+            //........................................................................
+            //					READ THE DISTRIBUTIONS
+            //		(read from opposite array due to previous swap operation)
+            //........................................................................
+
+            //..............incorporate external force................................................
+            //..............carry out relaxation process...............................................
+            m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+            m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+            m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+            m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+            m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+            m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+            m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+            m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+            m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+            m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+            m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+            m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+            m16 = m16 + rlx_setB*( - m16);
+            m17 = m17 + rlx_setB*( - m17);
+            m18 = m18 + rlx_setB*( - m18);
+            //.......................................................................................................
+            //.................inverse transformation......................................................
+
+            // q=0
+            fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+            dist[n] = fq;
+
+            // q = 1
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
+            dist[1*Np+n] = fq;
+
+            // q=2
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+            dist[2*Np+n] = fq;
+
+            // q = 3
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+            dist[3*Np+n] = fq;
+
+            // q = 4
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+            dist[4*Np+n] = fq;
+
+            // q = 5
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+            dist[5*Np+n] = fq;
+
+            // q = 6
+            fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+            dist[6*Np+n] = fq;
+
+            // q = 7
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+            dist[7*Np+n] = fq;
+
+
+            // q = 8
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                    +mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+            dist[8*Np+n] = fq;
+
+            // q = 9
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+            dist[9*Np+n] = fq;
+
+            // q = 10
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                                                    +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                                                    +mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+            dist[10*Np+n] = fq;
+
+
+            // q = 11
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+            dist[11*Np+n] = fq;
+
+            // q = 12
+            fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                                                            +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                                                            -mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+            dist[12*Np+n] = fq;
+
+            // q = 13
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+            dist[13*Np+n] = fq;
+
+            // q= 14
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+                    +mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                    -mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+
+            dist[14*Np+n] = fq;
+
+            // q = 15
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+                    -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+            dist[15*Np+n] = fq;
+
+            // q = 16
+            fq =  mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+                    -mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+            dist[16*Np+n] = fq;
+
+
+            // q = 17
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+                    -mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+            dist[17*Np+n] = fq;
+
+            // q = 18
+            fq = mrt_V1*rho+mrt_V9*m1
+                    +mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+                    -mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+            dist[18*Np+n] = fq;
+
+            //........................................................................
+		}
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
+                
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q19_AAodd_StokesMRT<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_StokesMRT: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
+                
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q19_AAeven_StokesMRT<<<NBLOCKS,NTHREADS >>>(dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_StokesMRT: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+

gpu/dfh.cu

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -16,7 +15,6 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

models/ColorModel.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -137,7 +136,10 @@ void ScaLBL_ColorModel::ReadParams(string filename){
 	//if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details)
 
 	BoundaryCondition = 0;
-	if (domain_db->keyExists( "BC" )){
+	if (color_db->keyExists( "BC" )){
+		BoundaryCondition = color_db->getScalar<int>( "BC" );
+	}
+	else if (domain_db->keyExists( "BC" )){
 		BoundaryCondition = domain_db->getScalar<int>( "BC" );
 	}
 	

models/ColorModel.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

models/DFHModel.cpp

@@ -81,13 +81,18 @@ void ScaLBL_DFHModel::ReadParams(string filename){
 	outletA=0.f;
 	outletB=1.f;
 
-	if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details)
+	BoundaryCondition = domain_db->getScalar<int>( "BC" );
+	if (color_db->keyExists( "BC" )){
+		BoundaryCondition = color_db->getScalar<int>( "BC" );
+	}
+	else if (domain_db->keyExists( "BC" )){
+		BoundaryCondition = domain_db->getScalar<int>( "BC" );
+	}
 
 	// Read domain parameters
 	auto L = domain_db->getVector<double>( "L" );
 	auto size = domain_db->getVector<int>( "n" );
 	auto nproc = domain_db->getVector<int>( "nproc" );
-	BoundaryCondition = domain_db->getScalar<int>( "BC" );
 	Nx = size[0];
 	Ny = size[1];
 	Nz = size[2];
@@ -97,6 +102,8 @@ void ScaLBL_DFHModel::ReadParams(string filename){
 	nprocx = nproc[0];
 	nprocy = nproc[1];
 	nprocz = nproc[2];
+	
+	if (BoundaryCondition==4) flux = din*rhoA; // mass flux must adjust for density (see formulation for details)
 
 }
 void ScaLBL_DFHModel::SetDomain(){

models/GreyscaleColorModel.cpp

@@ -9,6 +9,12 @@ Two-fluid greyscale color lattice boltzmann model
 #include <stdlib.h>
 #include <time.h>
 
+template<class TYPE>
+void DeleteArray( const TYPE *p )
+{
+    delete [] p;
+}
+
 ScaLBL_GreyscaleColorModel::ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauA_eff(0),tauB_eff(0),rhoA(0),rhoB(0),alpha(0),beta(0),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),GreyPorosity(0),
@@ -667,13 +673,17 @@ void ScaLBL_GreyscaleColorModel::Create(){
 }        
 
 void ScaLBL_GreyscaleColorModel::Initialize(){
-	
-	if (rank==0)	printf ("Initializing distributions \n");
-	ScaLBL_D3Q19_Init(fq, Np);
-    //ScaLBL_D3Q19_GreyscaleColor_Init(fq, Porosity_dvc, Np);
 	/*
 	 * This function initializes model
 	 */
+	if (rank==0)	printf ("Initializing distributions \n");
+	ScaLBL_D3Q19_Init(fq, Np);
+    //ScaLBL_D3Q19_GreyscaleColor_Init(fq, Porosity_dvc, Np);
+
+	if (rank==0)	printf ("Initializing phase field \n");
+	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
+	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+
 	if (Restart == true){
 		if (rank==0){
 			printf("Reading restart file! \n");
@@ -732,11 +742,11 @@ void ScaLBL_GreyscaleColorModel::Initialize(){
 		ScaLBL_DeviceBarrier();
 
 		MPI_Barrier(comm);
-	}
 
-	if (rank==0)	printf ("Initializing phase field \n");
-	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
-	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+        if (rank==0)	printf ("Initializing phase field from Restart\n");
+        ScaLBL_PhaseField_InitFromRestart(Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
+        ScaLBL_PhaseField_InitFromRestart(Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	}
 
 	// establish reservoirs for external bC
 	if (BoundaryCondition == 1 || BoundaryCondition == 2 ||  BoundaryCondition == 3 || BoundaryCondition == 4 ){
@@ -786,6 +796,17 @@ void ScaLBL_GreyscaleColorModel::Run(){
 	double initial_volume = 0.0;
 	double delta_volume = 0.0;
 	double delta_volume_target = 0.0;
+
+    //TODO -------- For temporary use - should be included in the analysis framework later -------------
+    int visualization_interval = 50000;
+    int restart_interval = 100000;
+	if (analysis_db->keyExists( "visualization_interval" )){
+		visualization_interval = analysis_db->getScalar<int>( "visualization_interval" );
+	}
+	if (analysis_db->keyExists( "restart_interval" )){
+		restart_interval = analysis_db->getScalar<int>( "restart_interval" );
+	}
+    //-------------------------------------------------------------------------------------------------
 	
 	/* history for morphological algoirthm */
 	double KRA_MORPH_FACTOR=0.5;
@@ -1015,6 +1036,51 @@ void ScaLBL_GreyscaleColorModel::Run(){
 		//************************************************************************
 		PROFILE_STOP("Update");
 
+        //TODO For temporary use - writing Restart and Vis files should be included in the analysis framework in the future
+		if (timestep%restart_interval==0){
+            //Use rank=0 write out Restart.db
+            if (rank==0) {
+                greyscaleColor_db->putScalar<int>("timestep",timestep);    		
+                greyscaleColor_db->putScalar<bool>( "Restart", true );
+                current_db->putDatabase("Color", greyscaleColor_db);
+                std::ofstream OutStream("Restart.db");
+                current_db->print(OutStream, "");
+                OutStream.close();
+      
+            }
+            //Write out Restart data.
+            std::shared_ptr<double> cDen;
+            std::shared_ptr<double> cfq;
+            cDen = std::shared_ptr<double>(new double[2*Np], DeleteArray<double>);
+            cfq  = std::shared_ptr<double>(new double[19*Np],DeleteArray<double>);
+            ScaLBL_CopyToHost(cDen.get(),Den,2*Np*sizeof(double));// Copy restart data to the CPU
+            ScaLBL_CopyToHost(cfq.get(), fq,19*Np*sizeof(double));// Copy restart data to the CPU
+
+            ofstream RESTARTFILE(LocalRestartFile,ios::binary);
+            double value;
+            for (int n=0; n<Np; n++){
+                // Write the two density values
+                value = cDen.get()[n];
+                RESTARTFILE.write((char*) &value, sizeof(value));
+                value = cDen.get()[Np+n];
+                RESTARTFILE.write((char*) &value, sizeof(value));
+                
+            }
+            for (int n=0; n<Np; n++){
+                // Write the distributions
+                for (int q=0; q<19; q++){
+                    value = cfq.get()[q*Np+n];
+                    RESTARTFILE.write((char*) &value, sizeof(value));
+                }
+            }
+            RESTARTFILE.close();
+		    MPI_Barrier(comm);
+        }
+		if (timestep%visualization_interval==0){
+            WriteVisFiles();
+        }
+        //-----------------------------------------------------------------------------------------------------------------
+
 		if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){
 			printf("%i %.5g \n",timestep,din);
 		}
@@ -1393,6 +1459,115 @@ double ScaLBL_GreyscaleColorModel::SeedPhaseField(const double seed_water_in_oil
   return(mass_loss);
 }
 
+//TODO for temporary use - writing visualization files should be included in the analysis framework in the future
+void ScaLBL_GreyscaleColorModel::WriteVisFiles(){
+    //NOTE: write_silo is always true
+    
+	std::vector<IO::MeshDataStruct> visData;
+	fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
+
+	auto VxVar = std::make_shared<IO::Variable>();
+	auto VyVar = std::make_shared<IO::Variable>();
+	auto VzVar = std::make_shared<IO::Variable>();
+	auto SignDistVar = std::make_shared<IO::Variable>();
+	auto PressureVar = std::make_shared<IO::Variable>();
+	auto PhaseVar    = std::make_shared<IO::Variable>();
+
+	// Create the MeshDataStruct	
+	IO::initialize("","silo","false");
+	visData.resize(1);
+	visData[0].meshName = "domain";
+	visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+
+    // create a temp data for copy from device
+	DoubleArray DataTemp(Nx,Ny,Nz);
+
+    if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
+
+        PhaseVar->name = "Phase";
+        PhaseVar->type = IO::VariableType::VolumeVariable;
+        PhaseVar->dim = 1;
+        PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(PhaseVar);
+
+        ASSERT(visData[0].vars[0]->name=="Phase");
+        Array<double>& PhaseData = visData[0].vars[0]->data;
+     	ScaLBL_CopyToHost(DataTemp.data(), Phi, sizeof(double)*Nx*Ny*Nz);
+        fillData.copy(DataTemp,PhaseData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_pressure", false )){
+
+        PressureVar->name = "Pressure";
+        PressureVar->type = IO::VariableType::VolumeVariable;
+        PressureVar->dim = 1;
+        PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(PressureVar);
+
+        ASSERT(visData[0].vars[1]->name=="Pressure");
+        Array<double>& PressData = visData[0].vars[1]->data;
+	    ScaLBL_Comm->RegularLayout(Map,Pressure,DataTemp);
+        fillData.copy(DataTemp,PressData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim = 1;
+        VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VxVar);
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim = 1;
+        VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VyVar);
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim = 1;
+        VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VzVar);
+
+        ASSERT(visData[0].vars[2]->name=="Velocity_x");
+        ASSERT(visData[0].vars[3]->name=="Velocity_y");
+        ASSERT(visData[0].vars[4]->name=="Velocity_z");
+        Array<double>& VelxData = visData[0].vars[2]->data;
+        Array<double>& VelyData = visData[0].vars[3]->data;
+        Array<double>& VelzData = visData[0].vars[4]->data;
+	    ScaLBL_Comm->RegularLayout(Map,&Velocity[0],DataTemp);
+        fillData.copy(DataTemp,VelxData);
+	    ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],DataTemp);
+        fillData.copy(DataTemp,VelyData);
+	    ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],DataTemp);
+        fillData.copy(DataTemp,VelzData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_distance", false )){
+
+        SignDistVar->name = "SignDist";
+        SignDistVar->type = IO::VariableType::VolumeVariable;
+        SignDistVar->dim = 1;
+        SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(SignDistVar);
+
+        ASSERT(visData[0].vars[5]->name=="SignDist");
+        Array<double>& SignData  = visData[0].vars[5]->data;
+        fillData.copy(Averages->SDs,SignData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "write_silo", true )){
+        IO::writeData( timestep, visData, Dm->Comm );
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
+        //TODO
+        //char CurrentIDFilename[40];
+        //sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
+        //Averages.AggregateLabels(CurrentIDFilename);
+    }
+
+}
+
 void ScaLBL_GreyscaleColorModel::WriteDebug(){
 	// Copy back final phase indicator field and convert to regular layout
 	DoubleArray PhaseField(Nx,Ny,Nz);

models/GreyscaleColorModel.h

@@ -90,5 +90,6 @@ private:
     double MorphInit(const double beta, const double morph_delta);
     double SeedPhaseField(const double seed_water_in_oil);
     double MorphOpenConnected(double target_volume_change);
+    void WriteVisFiles();
 };
 

models/GreyscaleModel.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2020 Equinor ASA
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -106,7 +105,10 @@ void ScaLBL_GreyscaleModel::ReadParams(string filename){
     
     //------------------------ Other Domain parameters ------------------------//
 	BoundaryCondition = 0;
-	if (domain_db->keyExists( "BC" )){
+	if (greyscale_db->keyExists( "BC" )){
+		BoundaryCondition = greyscale_db->getScalar<int>( "BC" );
+	}
+	else if (domain_db->keyExists( "BC" )){
 		BoundaryCondition = domain_db->getScalar<int>( "BC" );
 	}
 	// ------------------------------------------------------------------------//

models/GreyscaleModel.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2020 Equinor ASA                                                          
-  Copyright Equnior ASA
                                                                                       
   This file is part of the Open Porous Media project (OPM).                           
   OPM is free software: you can redistribute it and/or modify                         

models/IonModel.h

@@ -0,0 +1,102 @@
+/*
+ * Ion transporte LB Model
+ */
+
+#ifndef ScaLBL_IonModel_INC
+#define ScaLBL_IonModel_INC
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <vector>
+
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "analysis/Minkowski.h"
+#include "ProfilerApp.h"
+
+class ScaLBL_IonModel{
+public:
+	ScaLBL_IonModel(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_IonModel();	
+	
+	// functions in they should be run
+	void ReadParams(string filename,vector<int> &num_iter);
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+	void SetDomain();
+	void ReadInput();
+	void Create();
+	void Initialize();
+	void Run(double *Velocity, double *ElectricField);
+    void getIonConcentration(DoubleArray &IonConcentration, const int ic);
+    void getIonConcentration_debug(int timestep);
+    void DummyFluidVelocity();
+    void DummyElectricField();
+    double CalIonDenConvergence(vector<double> &ci_avg_previous);
+
+	//bool Restart,pBC;
+	int timestep;
+    vector<int> timestepMax;
+	int BoundaryConditionSolid;
+    double h;//domain resolution, unit [um/lu]
+    double kb,electron_charge,T,Vt;
+    double k2_inv;
+    double tolerance;
+    double fluidVelx_dummy,fluidVely_dummy,fluidVelz_dummy;
+    double Ex_dummy,Ey_dummy,Ez_dummy;
+	
+	int number_ion_species;
+	vector<int> BoundaryConditionInlet;
+	vector<int> BoundaryConditionOutlet;
+    vector<double> IonDiffusivity;//User input unit [m^2/sec]
+    vector<int> IonValence;
+    vector<double> IonConcentration;//unit [mol/m^3]
+    vector<double> Cin;//inlet boundary value, can be either concentration [mol/m^3] or flux [mol/m^2/sec]
+    vector<double> Cout;//outlet boundary value, can be either concentration [mol/m^3] or flux [mol/m^2/sec]
+	vector<double> tau;
+	vector<double> time_conv;
+	
+	int Nx,Ny,Nz,N,Np;
+	int rank,nprocx,nprocy,nprocz,nprocs;
+	double Lx,Ly,Lz;
+
+	std::shared_ptr<Domain> Dm;   // this domain is for analysis
+	std::shared_ptr<Domain> Mask; // this domain is for lbm
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> domain_db;
+    std::shared_ptr<Database> ion_db;
+
+    IntArray Map;
+    DoubleArray Distance;
+    int *NeighborList;
+    double *fq;
+    double *Ci; 
+    double *ChargeDensity; 
+    double *IonSolid;
+    double *FluidVelocityDummy;
+    double *ElectricFieldDummy;
+
+private:
+	MPI_Comm comm;
+	
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+    char OutputFilename[200];
+   
+    //int rank,nprocs;
+    void LoadParams(std::shared_ptr<Database> db0);    	
+    void AssignSolidBoundary(double *ion_solid);
+    void AssignIonConcentration_FromFile(double *Ci,const vector<std::string> &File_ion);
+    void IonConcentration_LB_to_Phys(DoubleArray &Den_reg);
+};
+#endif

models/MRTModel.cpp

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -73,7 +72,10 @@ void ScaLBL_MRTModel::ReadParams(string filename){
 	}	
 	
 	// Read domain parameters
-	if (domain_db->keyExists( "BC" )){
+	if (mrt_db->keyExists( "BoundaryCondition" )){
+		BoundaryCondition = mrt_db->getScalar<int>( "BC" );
+	}
+	else if (domain_db->keyExists( "BC" )){
 		BoundaryCondition = domain_db->getScalar<int>( "BC" );
 	}
 

models/MRTModel.h

@@ -1,6 +1,5 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
-  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

models/MultiPhysController.cpp

@@ -0,0 +1,147 @@
+#include "models/MultiPhysController.h"
+
+ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM):
+rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),
+analysis_interval(0),visualization_interval(0),tolerance(0),time_conv_max(0),comm(COMM)
+{
+
+}
+ScaLBL_Multiphys_Controller::~ScaLBL_Multiphys_Controller(){
+
+}
+
+void ScaLBL_Multiphys_Controller::ReadParams(string filename){
+    
+    // read the input database 
+	db = std::make_shared<Database>( filename );
+	study_db = db->getDatabase( "MultiphysController" );
+    
+
+    // Default parameters
+    timestepMax = 10000;
+    Restart = false;
+    num_iter_Stokes=1;
+    num_iter_Ion.push_back(1);
+    analysis_interval = 500;
+    visualization_interval = 10000;
+    tolerance = 1.0e-6;
+    time_conv_max = 0.0;
+	
+    // load input parameters
+	if (study_db->keyExists( "timestepMax" )){
+		timestepMax = study_db->getScalar<int>( "timestepMax" );
+	}
+	if (study_db->keyExists( "analysis_interval" )){
+		analysis_interval = study_db->getScalar<int>( "analysis_interval" );
+	}
+	if (study_db->keyExists( "visualization_interval" )){
+		visualization_interval = study_db->getScalar<int>( "visualization_interval" );
+	}
+	if (study_db->keyExists( "tolerance" )){
+		tolerance = study_db->getScalar<double>( "tolerance" );
+	}
+	//if (study_db->keyExists( "time_conv" )){
+	//	time_conv = study_db->getScalar<double>( "time_conv" );
+	//}
+	//if (study_db->keyExists( "Schmidt_Number" )){
+	//	SchmidtNum = study_db->getScalar<double>( "Schmidt_Number" );
+	//}
+
+    // recalculate relevant parameters
+    //if (SchmidtNum>1){
+    //    num_iter_Stokes = int(round(SchmidtNum/2)*2);
+    //    num_iter_Ion = 1;
+    //}
+    //else if (SchmidtNum>0 && SchmidtNum<1){
+    //    num_iter_Ion = int(round((1.0/SchmidtNum)/2)*2);
+    //    num_iter_Stokes = 1;
+    //}
+    //else if (SchmidtNum==1){
+    //    num_iter_Stokes = 1;
+    //    num_iter_Ion = 1;
+    //}
+    //else{
+	//	ERROR("Error: SchmidtNum (Schmidt number) must be a positive number! \n");
+    //}
+
+    // load input parameters
+    // in case user wants to have an absolute control over the iternal iteration
+    if (study_db->keyExists( "num_iter_Ion_List" )){
+        num_iter_Ion.clear();
+        num_iter_Ion = study_db->getVector<int>( "num_iter_Ion_List" );
+    }
+    if (study_db->keyExists( "num_iter_Stokes" )){
+        num_iter_Stokes = study_db->getScalar<int>( "num_iter_Stokes" );
+    }
+
+}
+
+int ScaLBL_Multiphys_Controller::getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv){
+    //Return number of internal iterations for the Stokes solver
+    int num_iter_stokes;
+    vector<double> TimeConv;
+
+    TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end());
+    TimeConv.insert(TimeConv.begin(),StokesTimeConv);
+    vector<double>::iterator it_max = max_element(TimeConv.begin(),TimeConv.end());
+    int idx_max = distance(TimeConv.begin(),it_max);
+    if (idx_max==0){
+        num_iter_stokes = 2;
+    }
+    else{
+        double temp = 2*TimeConv[idx_max]/StokesTimeConv;//the factor 2 is the number of iterations for the element has max time_conv
+        num_iter_stokes = int(round(temp/2)*2);
+    }
+    return num_iter_stokes;
+}
+
+vector<int> ScaLBL_Multiphys_Controller::getIonNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv){
+    //Return number of internal iterations for the Ion transport solver
+    vector<int> num_iter_ion;
+    vector<double> TimeConv;
+    TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end());
+    TimeConv.insert(TimeConv.begin(),StokesTimeConv);
+    vector<double>::iterator it_max = max_element(TimeConv.begin(),TimeConv.end());
+    unsigned int idx_max = distance(TimeConv.begin(),it_max);
+    if (idx_max==0){
+        for (unsigned int idx=1;idx<TimeConv.size();idx++){
+            double temp = 2*StokesTimeConv/TimeConv[idx];//the factor 2 is the number of iterations for the element has max time_conv
+            num_iter_ion.push_back(int(round(temp/2)*2));
+        }
+    }
+    else if (idx_max==1){
+        num_iter_ion.push_back(2);
+        for (unsigned int idx=2;idx<TimeConv.size();idx++){
+            double temp = 2*TimeConv[idx_max]/TimeConv[idx];//the factor 2 is the number of iterations for the element has max time_conv
+            num_iter_ion.push_back(int(round(temp/2)*2));
+        }
+    }
+    else if (idx_max==TimeConv.size()-1){
+        for (unsigned int idx=1;idx<TimeConv.size()-1;idx++){
+            double temp = 2*TimeConv[idx_max]/TimeConv[idx];//the factor 2 is the number of iterations for the element has max time_conv
+            num_iter_ion.push_back(int(round(temp/2)*2));
+        }
+        num_iter_ion.push_back(2);
+    }
+    else {
+        for (unsigned int idx=1;idx<idx_max;idx++){
+            double temp = 2*TimeConv[idx_max]/TimeConv[idx];//the factor 2 is the number of iterations for the element has max time_conv
+            num_iter_ion.push_back(int(round(temp/2)*2));
+        }
+        num_iter_ion.push_back(2);
+        for (unsigned int idx=idx_max+1;idx<TimeConv.size();idx++){
+            double temp = 2*TimeConv[idx_max]/TimeConv[idx];//the factor 2 is the number of iterations for the element has max time_conv
+            num_iter_ion.push_back(int(round(temp/2)*2));
+        }
+    }
+    return num_iter_ion;
+}
+
+void ScaLBL_Multiphys_Controller::getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv){
+    //Return maximum of the time converting factor from Stokes and ion solvers
+    vector<double> TimeConv;
+
+    TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end());
+    TimeConv.insert(TimeConv.begin(),StokesTimeConv);
+    time_conv_max = *max_element(TimeConv.begin(),TimeConv.end());
+}

models/MultiPhysController.h

@@ -0,0 +1,58 @@
+/*
+ * Multiphysics controller that coordinates the coupling between different models
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <vector>
+#include <algorithm>
+
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "analysis/Minkowski.h"
+#include "ProfilerApp.h"
+
+class ScaLBL_Multiphys_Controller{
+public:
+	ScaLBL_Multiphys_Controller(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_Multiphys_Controller();	
+	
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+    int getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
+    vector<int> getIonNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
+    //void getIonNumIter_PNP_coupling(double StokesTimeConv,vector<double> &IonTimeConv,vector<int> &IonTimeMax);
+    void getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
+	
+	bool Restart;
+    int timestepMax;
+    int num_iter_Stokes;
+    vector<int> num_iter_Ion;
+    int analysis_interval;
+    int visualization_interval;
+    double tolerance;
+    double time_conv_max;
+    //double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity
+
+	int rank,nprocs;
+
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> study_db;
+
+private:
+	MPI_Comm comm;
+	
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+   
+    //int rank,nprocs;
+    void LoadParams(std::shared_ptr<Database> db0);    	
+};

models/PoissonSolver.cpp

@@ -0,0 +1,936 @@
+/*
+ * Multi-relaxation time LBM Model
+ */
+#include "models/PoissonSolver.h"
+#include "analysis/distance.h"
+#include "common/ReadMicroCT.h"
+
+ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM):
+rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
+epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
+chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
+BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
+Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
+TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),   
+comm(COMM)
+{
+
+}
+ScaLBL_Poisson::~ScaLBL_Poisson(){
+
+}
+
+void ScaLBL_Poisson::ReadParams(string filename){
+	// read the input database 
+	db = std::make_shared<Database>( filename );
+	domain_db = db->getDatabase( "Domain" );
+	electric_db = db->getDatabase( "Poisson" );
+	
+    k2_inv = 4.0;//speed of sound for D3Q7 lattice 
+	tau = 0.5+k2_inv;
+	timestepMax = 100000;
+	tolerance = 1.0e-6;//stopping criterion for obtaining steady-state electricla potential
+    h = 1.0;//resolution; unit: um/lu
+    epsilon0 = 8.85e-12;//electric permittivity of vaccum; unit:[C/(V*m)]
+    epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)]
+    epsilonR = 78.4;//default dielectric constant of water
+    epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity 
+    analysis_interval = 1000; 
+    chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3]
+    WriteLog = false;
+    TestPeriodic = false;
+    TestPeriodicTime = 1.0;//unit: [sec]
+    TestPeriodicTimeConv = 0.01; //unit [sec/lt]
+    TestPeriodicSaveInterval = 0.1; //unit [sec]
+
+	// LB-Poisson Model parameters
+	if (electric_db->keyExists( "timestepMax" )){
+		timestepMax = electric_db->getScalar<int>( "timestepMax" );
+	}
+	if (electric_db->keyExists( "analysis_interval" )){
+		analysis_interval = electric_db->getScalar<int>( "analysis_interval" );
+	}
+	if (electric_db->keyExists( "tolerance" )){
+		tolerance = electric_db->getScalar<double>( "tolerance" );
+	}
+	if (electric_db->keyExists( "epsilonR" )){
+		epsilonR = electric_db->getScalar<double>( "epsilonR" );
+	}
+	if (electric_db->keyExists( "DummyChargeDen" )){
+		chargeDen_dummy = electric_db->getScalar<double>( "DummyChargeDen" );
+	}
+	if (electric_db->keyExists( "WriteLog" )){
+		WriteLog = electric_db->getScalar<bool>( "WriteLog" );
+	}
+	if (electric_db->keyExists( "TestPeriodic" )){
+		TestPeriodic = electric_db->getScalar<bool>( "TestPeriodic" );
+	}
+	if (electric_db->keyExists( "TestPeriodicTime" )){
+		TestPeriodicTime = electric_db->getScalar<double>( "TestPeriodicTime" );
+	}
+	if (electric_db->keyExists( "TestPeriodicTimeConv" )){
+		TestPeriodicTimeConv = electric_db->getScalar<double>( "TestPeriodicTimeConv" );
+	}
+	if (electric_db->keyExists( "TestPeriodicSaveInterval" )){
+		TestPeriodicSaveInterval = electric_db->getScalar<double>( "TestPeriodicSaveInterval" );
+	}
+
+    // Read solid boundary condition specific to Poisson equation
+    BoundaryConditionSolid = 1;
+	if (electric_db->keyExists( "BC_Solid" )){
+		BoundaryConditionSolid = electric_db->getScalar<int>( "BC_Solid" );
+	}
+    // Read boundary condition for electric potential
+    // BC = 0: normal periodic BC
+    // BC = 1: fixed electric potential
+    // BC = 2: sine/cosine periodic electric potential (need extra input parameters)
+    BoundaryConditionInlet = 0;
+    BoundaryConditionOutlet = 0;
+	if (electric_db->keyExists( "BC_Inlet" )){
+		BoundaryConditionInlet = electric_db->getScalar<int>( "BC_Inlet" );
+	}
+	if (electric_db->keyExists( "BC_Outlet" )){
+		BoundaryConditionOutlet = electric_db->getScalar<int>( "BC_Outlet" );
+	}
+
+	// Read domain parameters
+	if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu
+		h = domain_db->getScalar<double>( "voxel_length" );
+	}
+
+
+    //Re-calcualte model parameters if user updates input
+    epsilon0_LB = epsilon0*(h*1.0e-6);//unit:[C/(V*lu)]
+    epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity 
+
+	if (rank==0) printf("***********************************************************************************\n");
+	if (rank==0) printf("LB-Poisson Solver: steady-state MaxTimeStep = %i; steady-state tolerance = %.3g \n", timestepMax,tolerance);
+	if (rank==0) printf("                   LB relaxation tau = %.5g \n", tau);
+	if (rank==0) printf("***********************************************************************************\n");
+
+    switch (BoundaryConditionSolid){
+        case 1:
+          if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned\n");  
+          break;
+        case 2:
+          if (rank==0) printf("LB-Poisson Solver: solid boundary: Neumann-type surfacen charge density is assigned\n");  
+          break;
+        default:
+          if (rank==0) printf("LB-Poisson Solver: solid boundary: Dirichlet-type surfacen potential is assigned\n");  
+          break;
+    }
+}
+void ScaLBL_Poisson::SetDomain(){
+	Dm  = std::shared_ptr<Domain>(new Domain(domain_db,comm));      // full domain for analysis
+	Mask  = std::shared_ptr<Domain>(new Domain(domain_db,comm));    // mask domain removes immobile phases
+
+	// domain parameters
+	Nx = Dm->Nx;
+	Ny = Dm->Ny;
+	Nz = Dm->Nz;
+	Lx = Dm->Lx;
+	Ly = Dm->Ly;
+	Lz = Dm->Lz;
+	
+	N = Nx*Ny*Nz;
+	Distance.resize(Nx,Ny,Nz);
+	Psi_host.resize(Nx,Ny,Nz);
+
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
+	//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
+	MPI_Barrier(comm);
+    if (BoundaryConditionInlet==0 && BoundaryConditionOutlet==0){
+        Dm->BoundaryCondition   = 0;
+        Mask->BoundaryCondition = 0;
+    }
+    else if (BoundaryConditionInlet>0 && BoundaryConditionOutlet>0){
+        Dm->BoundaryCondition   = 1;
+        Mask->BoundaryCondition = 1;
+    }
+    else {//i.e. non-periodic and periodic BCs are mixed
+        ERROR("Error: check the type of inlet and outlet boundary condition! Mixed periodic and non-periodic BCs are found!\n");
+    }
+	Dm->CommInit();
+	MPI_Barrier(comm);
+	
+	rank = Dm->rank();	
+	nprocx = Dm->nprocx();
+	nprocy = Dm->nprocy();
+	nprocz = Dm->nprocz();
+}
+
+void ScaLBL_Poisson::ReadInput(){
+    
+    sprintf(LocalRankString,"%05d",Dm->rank());
+    sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
+    sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
+
+    
+    if (domain_db->keyExists( "Filename" )){
+    	auto Filename = domain_db->getScalar<std::string>( "Filename" );
+    	Mask->Decomp(Filename);
+    }
+    else if (domain_db->keyExists( "GridFile" )){
+    	// Read the local domain data
+    	auto input_id = readMicroCT( *domain_db, comm );
+    	// Fill the halo (assuming GCW of 1)
+    	array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
+    	ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
+    	ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
+    	fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
+    	Array<signed char> id_view;
+    	id_view.viewRaw( size1, Mask->id );
+    	fill.copy( input_id, id_view );
+    	fill.fill( id_view );
+    }
+    else{
+    	Mask->ReadIDs();
+    }
+
+    // Generate the signed distance map
+	// Initialize the domain and communication
+	Array<char> id_solid(Nx,Ny,Nz);
+	// Solve for the position of the solid phase
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				// Initialize the solid phase
+				if (Mask->id[n] > 0)	id_solid(i,j,k) = 1;
+				else	     	    id_solid(i,j,k) = 0;
+			}
+		}
+	}
+	// Initialize the signed distance function
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				// Initialize distance to +/- 1
+				Distance(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
+			}
+		}
+	}
+//	MeanFilter(Averages->SDs);
+	if (rank==0) printf("LB-Poisson Solver: Initialized solid phase & converting to Signed Distance function \n");
+	CalcDist(Distance,id_solid,*Dm);
+    if (rank == 0) cout << "    Domain set." << endl;
+}
+
+void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid)
+{
+	size_t NLABELS=0;
+	signed char VALUE=0;
+	double AFFINITY=0.f;
+
+	auto LabelList = electric_db->getVector<int>( "SolidLabels" );
+	auto AffinityList = electric_db->getVector<double>( "SolidValues" );
+
+	NLABELS=LabelList.size();
+	if (NLABELS != AffinityList.size()){
+		ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n");
+	}
+
+	double label_count[NLABELS];
+	double label_count_global[NLABELS];
+	// Assign the labels
+
+	for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
+
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				VALUE=Mask->id[n];
+                AFFINITY=0.f;
+				// Assign the affinity from the paired list
+				for (unsigned int idx=0; idx < NLABELS; idx++){
+					if (VALUE == LabelList[idx]){
+						AFFINITY=AffinityList[idx];
+                        //NOTE need to convert the user input phys unit to LB unit
+                        if (BoundaryConditionSolid==2){
+                            //for BCS=1, i.e. Dirichlet-type, no need for unit conversion
+                            AFFINITY = AFFINITY*(h*h*1.0e-12)/epsilon_LB; 
+                        }
+						label_count[idx] += 1.0;
+						idx = NLABELS;
+						//Mask->id[n] = 0; // set mask to zero since this is an immobile component
+					}
+				}
+				poisson_solid[n] = AFFINITY;
+			}
+		}
+	}
+
+	for (size_t idx=0; idx<NLABELS; idx++)
+		label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
+
+	if (rank==0){
+		printf("LB-Poisson Solver: number of Poisson solid labels: %lu \n",NLABELS);
+		for (unsigned int idx=0; idx<NLABELS; idx++){
+			VALUE=LabelList[idx];
+			AFFINITY=AffinityList[idx];
+			double volume_fraction  = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+            switch (BoundaryConditionSolid){
+                case 1:
+			        printf("   label=%d, surface potential=%.3g [V], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+                    break;
+                case 2: 
+			        printf("   label=%d, surface charge density=%.3g [C/m^2], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+                    break;
+                default:
+			        printf("   label=%d, surface potential=%.3g [V], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+                    break;
+            }
+		}
+	}
+}
+
+void ScaLBL_Poisson::Create(){
+	/*
+	 *  This function creates the variables needed to run a LBM 
+	 */
+	int rank=Mask->rank();
+	//.........................................................
+	// Initialize communication structures in averaging domain
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
+	Mask->CommInit();
+	Np=Mask->PoreCount();
+	//...........................................................................
+	if (rank==0)    printf ("LB-Poisson Solver: Create ScaLBL_Communicator \n");
+	// Create a communicator for the device (will use optimized layout)
+	// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
+	ScaLBL_Comm  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+	ScaLBL_Comm_Regular  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+
+	int Npad=(Np/16 + 2)*16;
+	if (rank==0)    printf ("LB-Poisson Solver: Set up memory efficient layout \n");
+	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
+	auto neighborList= new int[18*Npad];
+	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
+	MPI_Barrier(comm);
+	//...........................................................................
+	//                MAIN  VARIABLES ALLOCATED HERE
+	//...........................................................................
+	// LBM variables
+	if (rank==0)    printf ("LB-Poisson Solver: Allocating distributions \n");
+	//......................device distributions.................................
+	int dist_mem_size = Np*sizeof(double);
+	int neighborSize=18*(Np*sizeof(int));
+	//...........................................................................
+	ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
+	ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
+	//ScaLBL_AllocateDeviceMemory((void **) &dvcID, sizeof(signed char)*Nx*Ny*Nz);
+	ScaLBL_AllocateDeviceMemory((void **) &fq, 7*dist_mem_size);  
+	ScaLBL_AllocateDeviceMemory((void **) &Psi, sizeof(double)*Nx*Ny*Nz);
+	ScaLBL_AllocateDeviceMemory((void **) &ElectricField, 3*sizeof(double)*Np);
+	//...........................................................................
+	
+	// Update GPU data structures
+	if (rank==0)    printf ("LB-Poisson Solver: Setting up device map and neighbor list \n");
+	fflush(stdout);
+	int *TmpMap;
+	TmpMap=new int[Np];
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				int idx=Map(i,j,k);
+				if (!(idx < 0))
+					TmpMap[idx] = k*Nx*Ny+j*Nx+i;
+			}
+		}
+	}
+	// check that TmpMap is valid
+	for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
+		auto n = TmpMap[idx];
+		if (n > Nx*Ny*Nz){
+			printf("Bad value! idx=%i \n", n);
+			TmpMap[idx] = Nx*Ny*Nz-1;
+		}
+	}
+	for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
+		auto n = TmpMap[idx];
+		if ( n > Nx*Ny*Nz ){
+			printf("Bad value! idx=%i \n",n);
+			TmpMap[idx] = Nx*Ny*Nz-1;
+		}
+	}
+	ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
+	ScaLBL_DeviceBarrier();
+	delete [] TmpMap;
+	// copy the neighbor list 
+	ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
+	ScaLBL_DeviceBarrier();
+	MPI_Barrier(comm);
+	delete [] neighborList;
+    // copy node ID
+	//ScaLBL_CopyToDevice(dvcID, Mask->id, sizeof(signed char)*Nx*Ny*Nz);
+	//ScaLBL_DeviceBarrier();
+	
+    //Initialize solid boundary for electric potential
+    ScaLBL_Comm->SetupBounceBackList(Map, Mask->id, Np);
+	MPI_Barrier(comm);
+}        
+
+void ScaLBL_Poisson::Potential_Init(double *psi_init){
+
+    //set up default boundary input parameters
+    Vin0 = Vout0 = 1.0; //unit: [V]
+    freqIn = freqOut = 50.0; //unit: [Hz]
+    t0_In = t0_Out = 0.0; //unit: [sec]
+    Vin_Type = Vout_Type = 1; //1->sin; 2->cos
+    Vin  = 1.0; //Boundary-z (inlet)  electric potential
+    Vout = 1.0; //Boundary-Z (outlet) electric potential
+
+    if (BoundaryConditionInlet>0){
+        switch (BoundaryConditionInlet){
+            case 1:
+                if (electric_db->keyExists( "Vin" )){
+                    Vin = electric_db->getScalar<double>( "Vin" );
+                }
+                if (rank==0) printf("LB-Poisson Solver: inlet boundary; fixed electric potential Vin = %.3g [V]\n",Vin);
+                break;
+            case 2:
+                if (electric_db->keyExists( "Vin0" )){//voltage amplitude; unit: Volt
+                    Vin0 = electric_db->getScalar<double>( "Vin0" );
+                }
+                if (electric_db->keyExists( "freqIn" )){//unit: Hz
+                    freqIn = electric_db->getScalar<double>( "freqIn" );
+                }
+                if (electric_db->keyExists( "t0_In" )){//timestep shift, unit: lt
+                    t0_In = electric_db->getScalar<double>( "t0_In" );
+                }
+                if (electric_db->keyExists( "Vin_Type" )){
+                    //type=1 -> sine
+                    //tyep=2 -> cosine
+                    Vin_Type = electric_db->getScalar<int>( "Vin_Type" );
+                    if (Vin_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vin_Type is currently not supported!  \n");
+                }
+                if (rank==0){
+                    if (Vin_Type==1){
+                        printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vin0,freqIn,t0_In);
+                        printf("                                   V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
+                    }
+                    else if (Vin_Type==2){
+                        printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V] \n",Vin0,freqIn,t0_In);
+                        printf("                                   V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
+                    } 
+                } 
+                break;
+        }
+    }
+    if (BoundaryConditionOutlet>0){
+        switch (BoundaryConditionOutlet){
+            case 1:
+                if (electric_db->keyExists( "Vout" )){
+                    Vout = electric_db->getScalar<double>( "Vout" );
+                }
+                if (rank==0) printf("LB-Poisson Solver: outlet boundary; fixed electric potential Vout = %.3g [V] \n",Vout);
+                break;
+            case 2:
+                if (electric_db->keyExists( "Vout0" )){//voltage amplitude; unit: Volt
+                    Vout0 = electric_db->getScalar<double>( "Vout0" );
+                }
+                if (electric_db->keyExists( "freqOut" )){//unit: Hz
+                    freqOut = electric_db->getScalar<double>( "freqOut" );
+                }
+                if (electric_db->keyExists( "t0_Out" )){//timestep shift, unit: lt
+                    t0_Out = electric_db->getScalar<double>( "t0_Out" );
+                }
+                if (electric_db->keyExists( "Vout_Type" )){
+                    //type=1 -> sine
+                    //tyep=2 -> cosine
+                    Vout_Type = electric_db->getScalar<int>( "Vout_Type" );
+                    if (Vout_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vout_Type is currently not supported!  \n");
+                }
+                if (rank==0){
+                    if (Vout_Type==1){
+                        printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
+                        printf("                                    V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
+                    }
+                    else if (Vout_Type==2){
+                        printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
+                        printf("                                    V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
+                    } 
+                } 
+                break;
+        }
+    }
+    //By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis
+    if (BoundaryConditionInlet==2)  Vin  = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,0);
+    if (BoundaryConditionOutlet==2) Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,0);
+    double slope = (Vout-Vin)/(Nz-2);
+    double psi_linearized;
+	for (int k=0;k<Nz;k++){
+        if (k==0 || k==1){
+            psi_linearized = Vin;
+        }
+        else if (k==Nz-1 || k==Nz-2){
+            psi_linearized = Vout;
+        }
+        else{
+            psi_linearized = slope*(k-1)+Vin;
+        }
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				if (Mask->id[n]>0){
+                    psi_init[n] = psi_linearized;
+                }
+            }
+        }
+    }
+}
+
+double ScaLBL_Poisson::getBoundaryVoltagefromPeriodicBC(double V0, double freq, double t0, int V_type, int time_step){
+    return V0*(V_type==1)*sin(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv))+V0*(V_type==2)*cos(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv));
+}
+
+void ScaLBL_Poisson::Initialize(double time_conv_from_Study){
+	/*
+	 * This function initializes model
+     * "time_conv_from_Study" is the phys to LB time conversion factor, unit=[sec/lt]
+     * which is used for periodic voltage input for inlet and outlet boundaries
+	 */
+    if (rank==0)    printf ("LB-Poisson Solver: initializing D3Q7 distributions\n");
+    //NOTE the initialization involves two steps:
+    //1. assign solid boundary value (surface potential or surface change density)
+    //2. Initialize electric potential for pore nodes
+    double *psi_host;
+    psi_host = new double [Nx*Ny*Nz];
+    time_conv = time_conv_from_Study;
+    AssignSolidBoundary(psi_host);//step1
+    Potential_Init(psi_host);//step2
+	ScaLBL_CopyToDevice(Psi, psi_host, Nx*Ny*Nz*sizeof(double));
+	ScaLBL_DeviceBarrier();
+    ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+    ScaLBL_D3Q7_Poisson_Init(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);
+    delete [] psi_host;
+    
+    //extra treatment for halo layer
+    //if (BoundaryCondition==1){
+	//	if (Dm->kproc()==0){
+	//		ScaLBL_SetSlice_z(Psi,Vin,Nx,Ny,Nz,0);
+	//	}
+	//	if (Dm->kproc() == nprocz-1){
+	//		ScaLBL_SetSlice_z(Psi,Vout,Nx,Ny,Nz,Nz-1);
+	//	}
+    //}
+}
+
+void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
+    
+	//.......create and start timer............
+	//double starttime,stoptime,cputime;
+	//ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	//starttime = MPI_Wtime();
+
+	timestep=0;
+	double error = 1.0;
+	double psi_avg_previous = 0.0;
+	while (timestep < timestepMax && error > tolerance) {
+		//************************************************************************/
+		// *************ODD TIMESTEP*************//
+        timestep++;
+        
+        SolveElectricPotentialAAodd(timestep_from_Study);//update electric potential
+        SolvePoissonAAodd(ChargeDensity);//perform collision
+		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+		// *************EVEN TIMESTEP*************//
+		timestep++;
+		SolveElectricPotentialAAeven(timestep_from_Study);//update electric potential
+        SolvePoissonAAeven(ChargeDensity);//perform collision
+		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		//************************************************************************/
+
+        // Check convergence of steady-state solution
+        if (timestep%analysis_interval==0){
+        
+			//ScaLBL_Comm->RegularLayout(Map,Psi,Psi_host);
+            ScaLBL_CopyToHost(Psi_host.data(),Psi,sizeof(double)*Nx*Ny*Nz);
+			double count_loc=0;
+			double count;
+            double psi_avg;
+            double psi_loc=0.f;
+
+			for (int k=1; k<Nz-1; k++){
+				for (int j=1; j<Ny-1; j++){
+					for (int i=1; i<Nx-1; i++){
+						if (Distance(i,j,k) > 0){
+							psi_loc += Psi_host(i,j,k);
+							count_loc+=1.0;
+						}
+					}
+				}
+			}
+			MPI_Allreduce(&psi_loc,&psi_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			
+			psi_avg /= count;
+            double psi_avg_mag=psi_avg;
+		    if (psi_avg==0.0) psi_avg_mag=1.0;
+            error = fabs(psi_avg-psi_avg_previous)/fabs(psi_avg_mag);
+			psi_avg_previous = psi_avg;
+        }
+	}
+    if(WriteLog==true){
+        getConvergenceLog(timestep,error);
+    }
+
+	//************************************************************************/
+	//stoptime = MPI_Wtime();
+	////if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n");
+	////if (rank==0) printf("---------------------------------------------------------------------------\n");
+	//// Compute the walltime per timestep
+	//cputime = (stoptime - starttime)/timestep;
+	//// Performance obtained from each node
+	//double MLUPS = double(Np)/cputime/1000000;
+
+	//if (rank==0) printf("******************* LB-Poisson Solver Statistics ********************\n");
+	//if (rank==0) printf("CPU time = %f \n", cputime);
+	//if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
+	//MLUPS *= nprocs;
+	//if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
+	//if (rank==0) printf("*********************************************************************\n");
+
+}
+
+void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
+    if (rank==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("PoissonSolver_Convergence.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("PoissonSolver_Convergence.csv","a+");
+		if (WriteHeader)
+		{
+			fprintf(TIMELOG,"Timestep Error\n");				
+            fprintf(TIMELOG,"%i %.5g\n",timestep,error); 
+            fflush(TIMELOG);
+		}
+        else {
+            fprintf(TIMELOG,"%i %.5g\n",timestep,error); 
+            fflush(TIMELOG);
+        }
+    }
+}
+
+void ScaLBL_Poisson::SolveElectricPotentialAAodd(int timestep_from_Study){
+	ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL
+	ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
+    ScaLBL_DeviceBarrier();
+	// Set boundary conditions
+	if (BoundaryConditionInlet > 0){
+        switch (BoundaryConditionInlet){
+            case 1:
+                ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq,  Vin, timestep);
+                break;
+            case 2:
+                Vin  = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
+                ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq,  Vin, timestep);
+                break;
+        }
+	}
+	if (BoundaryConditionOutlet > 0){
+        switch (BoundaryConditionOutlet){
+            case 1:
+		        ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
+                break;
+            case 2:
+                Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
+		        ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
+                break;
+        }
+	}
+    //-------------------------//
+	ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);
+}
+
+void ScaLBL_Poisson::SolveElectricPotentialAAeven(int timestep_from_Study){
+	ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL
+	ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
+    ScaLBL_DeviceBarrier();
+	// Set boundary conditions
+	if (BoundaryConditionInlet > 0){
+        switch (BoundaryConditionInlet){
+            case 1:
+                ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq,  Vin, timestep);
+                break;
+            case 2:
+                Vin  = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
+                ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq,  Vin, timestep);
+                break;
+        }
+	}
+	if (BoundaryConditionOutlet > 0){
+        switch (BoundaryConditionOutlet){
+            case 1:
+		        ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
+                break;
+            case 2:
+                Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
+		        ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
+                break;
+        }
+	}
+    //-------------------------//
+	ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);
+}
+
+void ScaLBL_Poisson::SolvePoissonAAodd(double *ChargeDensity){
+	ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	ScaLBL_D3Q7_AAodd_Poisson(NeighborList, dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, 0, ScaLBL_Comm->LastExterior(), Np);
+    if (BoundaryConditionSolid==1){
+	    ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi);
+    }
+    else if (BoundaryConditionSolid==2){
+	    ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi);
+    }
+}
+
+void ScaLBL_Poisson::SolvePoissonAAeven(double *ChargeDensity){
+	ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	ScaLBL_D3Q7_AAeven_Poisson(dvcMap, fq, ChargeDensity, Psi, ElectricField, tau, epsilon_LB, 0, ScaLBL_Comm->LastExterior(), Np);
+    if (BoundaryConditionSolid==1){
+	    ScaLBL_Comm->SolidDirichletD3Q7(fq, Psi);
+    }
+    else if (BoundaryConditionSolid==2){
+	    ScaLBL_Comm->SolidNeumannD3Q7(fq, Psi);
+    }
+}
+
+void ScaLBL_Poisson::DummyChargeDensity(){
+    double *ChargeDensity_host;
+    ChargeDensity_host = new double[Np];
+
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				int idx=Map(i,j,k);
+				if (!(idx < 0))
+                    ChargeDensity_host[idx] = chargeDen_dummy*(h*h*h*1.0e-18);
+            }
+        }
+    }
+	ScaLBL_AllocateDeviceMemory((void **) &ChargeDensityDummy, sizeof(double)*Np);
+	ScaLBL_CopyToDevice(ChargeDensityDummy, ChargeDensity_host, sizeof(double)*Np);
+	ScaLBL_DeviceBarrier();
+	delete [] ChargeDensity_host;
+}
+
+void ScaLBL_Poisson::getElectricPotential_debug(int timestep){
+    //This function write out decomposed data
+    DoubleArray PhaseField(Nx,Ny,Nz);
+	//ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField);
+    ScaLBL_CopyToHost(PhaseField.data(),Psi,sizeof(double)*Nx*Ny*Nz);
+    //ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+    FILE *OUTFILE;
+    sprintf(LocalRankFilename,"Electric_Potential_Time_%i.%05i.raw",timestep,rank);
+    OUTFILE = fopen(LocalRankFilename,"wb");
+    fwrite(PhaseField.data(),8,N,OUTFILE);
+    fclose(OUTFILE);
+}
+
+void ScaLBL_Poisson::getElectricPotential(DoubleArray &ReturnValues){
+    //This function wirte out the data in a normal layout (by aggregating all decomposed domains)
+	//ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField);
+    ScaLBL_CopyToHost(ReturnValues.data(),Psi,sizeof(double)*Nx*Ny*Nz);
+}
+
+void ScaLBL_Poisson::getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z){
+
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],Values_x);
+        ElectricField_LB_to_Phys(Values_x);
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],Values_y);
+        ElectricField_LB_to_Phys(Values_y);
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],Values_z);
+        ElectricField_LB_to_Phys(Values_z);
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+}
+
+void ScaLBL_Poisson::getElectricField_debug(int timestep){
+
+        //ScaLBL_D3Q7_Poisson_getElectricField(fq,ElectricField,tau,Np);
+        //ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+        DoubleArray PhaseField(Nx,Ny,Nz);
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[0*Np],PhaseField);
+        ElectricField_LB_to_Phys(PhaseField);
+        FILE *EX;
+        sprintf(LocalRankFilename,"ElectricField_X_Time_%i.%05i.raw",timestep,rank);
+        EX = fopen(LocalRankFilename,"wb");
+        fwrite(PhaseField.data(),8,N,EX);
+        fclose(EX);
+
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[1*Np],PhaseField);
+        ElectricField_LB_to_Phys(PhaseField);
+        FILE *EY;
+        sprintf(LocalRankFilename,"ElectricField_Y_Time_%i.%05i.raw",timestep,rank);
+        EY = fopen(LocalRankFilename,"wb");
+        fwrite(PhaseField.data(),8,N,EY);
+        fclose(EY);
+
+	    ScaLBL_Comm->RegularLayout(Map,&ElectricField[2*Np],PhaseField);
+        ElectricField_LB_to_Phys(PhaseField);
+        FILE *EZ;
+        sprintf(LocalRankFilename,"ElectricField_Z_Time_%i.%05i.raw",timestep,rank);
+        EZ = fopen(LocalRankFilename,"wb");
+        fwrite(PhaseField.data(),8,N,EZ);
+        fclose(EZ);
+}
+
+void ScaLBL_Poisson::ElectricField_LB_to_Phys(DoubleArray &Efield_reg){
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+                int idx=Map(i,j,k);
+				if (!(idx < 0)){
+                    Efield_reg(i,j,k) = Efield_reg(i,j,k)/(h*1.0e-6); 
+                }
+            }
+        }
+    }
+}
+
+//void ScaLBL_Poisson::SolveElectricField(){
+//    ScaLBL_Comm_Regular->SendHalo(Psi);
+//    ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid,
+//                                      Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+//    ScaLBL_Comm_Regular->RecvHalo(Psi);
+//    ScaLBL_DeviceBarrier();
+//    if (BoundaryCondition == 1){
+//        ScaLBL_Comm->Poisson_D3Q7_BC_z(dvcMap,Psi,Vin);
+//        ScaLBL_Comm->Poisson_D3Q7_BC_Z(dvcMap,Psi,Vout);
+//    }
+//    ScaLBL_D3Q7_Poisson_ElectricField(NeighborList, dvcMap, dvcID, Psi, ElectricField, BoundaryConditionSolid, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
+//
+//}
+
+//void ScaLBL_Poisson::getElectricPotential(){
+//
+//        DoubleArray PhaseField(Nx,Ny,Nz);
+//	    ScaLBL_Comm->RegularLayout(Map,Psi,PhaseField);
+//        //ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+//        FILE *OUTFILE;
+//        sprintf(LocalRankFilename,"Electric_Potential.%05i.raw",rank);
+//        OUTFILE = fopen(LocalRankFilename,"wb");
+//        fwrite(PhaseField.data(),8,N,OUTFILE);
+//        fclose(OUTFILE);
+//}
+
+//old version where Psi is of size Np
+//void ScaLBL_Poisson::AssignSolidBoundary(double *poisson_solid)
+//{
+//	size_t NLABELS=0;
+//	signed char VALUE=0;
+//	double AFFINITY=0.f;
+//
+//	auto LabelList = electric_db->getVector<int>( "SolidLabels" );
+//	auto AffinityList = electric_db->getVector<double>( "SolidValues" );
+//
+//	NLABELS=LabelList.size();
+//	if (NLABELS != AffinityList.size()){
+//		ERROR("Error: LB-Poisson Solver: SolidLabels and SolidValues must be the same length! \n");
+//	}
+//
+//	double label_count[NLABELS];
+//	double label_count_global[NLABELS];
+//	// Assign the labels
+//
+//	for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
+//
+//	for (int k=0;k<Nz;k++){
+//		for (int j=0;j<Ny;j++){
+//			for (int i=0;i<Nx;i++){
+//				int n = k*Nx*Ny+j*Nx+i;
+//				VALUE=Mask->id[n];
+//                AFFINITY=0.f;
+//				// Assign the affinity from the paired list
+//				for (unsigned int idx=0; idx < NLABELS; idx++){
+//				      //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]);
+//					if (VALUE == LabelList[idx]){
+//						AFFINITY=AffinityList[idx];
+//                        //NOTE need to convert the user input phys unit to LB unit
+//                        if (BoundaryConditionSolid==2){
+//                            //for BCS=1, i.e. Dirichlet-type, no need for unit conversion
+//                            //TODO maybe there is a factor of gamm missing here ?
+//                            AFFINITY = AFFINITY*(h*h*1.0e-12)/epsilon_LB; 
+//                        }
+//						label_count[idx] += 1.0;
+//						idx = NLABELS;
+//						//Mask->id[n] = 0; // set mask to zero since this is an immobile component
+//					}
+//				}
+//				poisson_solid[n] = AFFINITY;
+//			}
+//		}
+//	}
+//
+//	for (size_t idx=0; idx<NLABELS; idx++)
+//		label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
+//
+//	if (rank==0){
+//		printf("LB-Poisson Solver: number of Poisson solid labels: %lu \n",NLABELS);
+//		for (unsigned int idx=0; idx<NLABELS; idx++){
+//			VALUE=LabelList[idx];
+//			AFFINITY=AffinityList[idx];
+//			double volume_fraction  = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+//            switch (BoundaryConditionSolid){
+//                case 1:
+//			        printf("   label=%d, surface potential=%.3g [V], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+//                    break;
+//                case 2: 
+//			        printf("   label=%d, surface charge density=%.3g [C/m^2], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+//                    break;
+//                default:
+//			        printf("   label=%d, surface potential=%.3g [V], volume fraction=%.2g\n",VALUE,AFFINITY,volume_fraction); 
+//                    break;
+//            }
+//		}
+//	}
+//}
+
+// old version where Psi is of size Np
+//void ScaLBL_Poisson::Potential_Init(double *psi_init){
+//
+//    if (BoundaryCondition==1){
+//	    if (electric_db->keyExists( "Vin" )){
+//	    	Vin = electric_db->getScalar<double>( "Vin" );
+//	    }
+//	    if (electric_db->keyExists( "Vout" )){
+//	    	Vout = electric_db->getScalar<double>( "Vout" );
+//	    }
+//    }
+//    //By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis
+//    double slope = (Vout-Vin)/(Nz-2);
+//    double psi_linearized;
+//	for (int k=0;k<Nz;k++){
+//        if (k==0 || k==1){
+//            psi_linearized = Vin;
+//        }
+//        else if (k==Nz-1 || k==Nz-2){
+//            psi_linearized = Vout;
+//        }
+//        else{
+//            psi_linearized = slope*(k-1)+Vin;
+//        }
+//		for (int j=0;j<Ny;j++){
+//			for (int i=0;i<Nx;i++){
+//				int idx = Map(i,j,k);
+//				if (!(idx < 0)){
+//                    psi_init[idx] = psi_linearized;
+//                }
+//            }
+//        }
+//    }
+//}

models/PoissonSolver.h

@@ -0,0 +1,113 @@
+/*
+ * Multi-relaxation time LBM Model
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <cmath>
+
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "analysis/Minkowski.h"
+#include "ProfilerApp.h"
+
+#define _USE_MATH_DEFINES
+#ifndef ScaLBL_POISSON_INC
+#define ScaLBL_POISSON_INC
+
+class ScaLBL_Poisson{
+public:
+	ScaLBL_Poisson(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_Poisson();	
+	
+	// functions in they should be run
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+	void SetDomain();
+	void ReadInput();
+	void Create();
+	void Initialize(double time_conv_from_Study);
+	void Run(double *ChargeDensity,int timestep_from_Study);
+    void getElectricPotential(DoubleArray &ReturnValues);
+    void getElectricPotential_debug(int timestep);
+    void getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z);
+    void getElectricField_debug(int timestep);
+    void DummyChargeDensity();//for debugging
+
+	//bool Restart,pBC;
+	int timestep,timestepMax;
+    int analysis_interval;
+	int BoundaryConditionInlet;
+	int BoundaryConditionOutlet;
+    int BoundaryConditionSolid;
+	double tau;
+	double tolerance;
+    double k2_inv;
+    double epsilon0,epsilon0_LB,epsilonR,epsilon_LB;
+    double Vin, Vout;
+    double chargeDen_dummy;//for debugging
+    bool WriteLog;
+    double Vin0,freqIn,t0_In,Vin_Type;
+    double Vout0,freqOut,t0_Out,Vout_Type;
+    bool   TestPeriodic;
+    double TestPeriodicTime;//unit: [sec]
+    double TestPeriodicTimeConv; //unit [sec/lt]
+    double TestPeriodicSaveInterval; //unit [sec]
+	
+	int Nx,Ny,Nz,N,Np;
+	int rank,nprocx,nprocy,nprocz,nprocs;
+	double Lx,Ly,Lz;
+    double h;//image resolution
+    double time_conv;//phys to LB time converting factor; unit=[sec/lt]
+
+	std::shared_ptr<Domain> Dm;   // this domain is for analysis
+	std::shared_ptr<Domain> Mask; // this domain is for lbm
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular;
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> domain_db;
+    std::shared_ptr<Database> electric_db;
+
+    IntArray Map;
+    DoubleArray Distance;
+    DoubleArray Psi_host;
+    int *NeighborList;
+    int *dvcMap;
+    //signed char *dvcID;
+    double *fq;
+    double *Psi; 
+    double *ElectricField;
+    double *ChargeDensityDummy;// for debugging
+
+private:
+	MPI_Comm comm;
+	
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+    char OutputFilename[200];
+	FILE *TIMELOG;
+   
+    //int rank,nprocs;
+    void LoadParams(std::shared_ptr<Database> db0);    	
+    void AssignSolidBoundary(double *poisson_solid);
+    void Potential_Init(double *psi_init);
+    void ElectricField_LB_to_Phys(DoubleArray &Efield_reg);
+    void SolveElectricPotentialAAodd(int timestep_from_Study);
+    void SolveElectricPotentialAAeven(int timestep_from_Study);
+    //void SolveElectricField();
+    void SolvePoissonAAodd(double *ChargeDensity);
+    void SolvePoissonAAeven(double *ChargeDensity);
+    void getConvergenceLog(int timestep,double error);
+    double getBoundaryVoltagefromPeriodicBC(double V0,double freq,double t0,int V_type,int time_step);
+    
+};
+#endif

models/StokesModel.cpp

@@ -0,0 +1,815 @@
+/*
+ * Multi-relaxation time LBM Model
+ */
+#include "models/StokesModel.h"
+#include "analysis/distance.h"
+#include "common/ReadMicroCT.h"
+
+ScaLBL_StokesModel::ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM):
+rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
+Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),mu(0),h(0),nu_phys(0),rho_phys(0),rho0(0),den_scale(0),time_conv(0),tolerance(0),
+Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
+{
+
+}
+ScaLBL_StokesModel::~ScaLBL_StokesModel(){
+
+}
+
+void ScaLBL_StokesModel::ReadParams(string filename,int num_iter){
+	// read the input database 
+	db = std::make_shared<Database>( filename );
+	domain_db = db->getDatabase( "Domain" );
+	stokes_db = db->getDatabase( "Stokes" );
+	
+    //------ Load number of iteration from multiphysics controller ------//
+	timestepMax = num_iter;
+    //-------------------------------------------------------------------//
+
+	//---------------------- Default model parameters --------------------------//		
+    rho_phys = 1000.0; //by default use water density; unit [kg/m^3]
+    nu_phys = 1.004e-6;//by default use water kinematic viscosity at 20C; unit [m^2/sec]    
+    h = 1.0;//image resolution;[um]
+	tau = 1.0;
+    mu = (tau-0.5)/3.0;//LB kinematic viscosity;unit [lu^2/lt]
+    time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt]
+    rho0 = 1.0;//LB density
+    den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density
+	tolerance = 1.0e-8;
+	Fx = Fy = 0.0;
+	Fz = 1.0e-5;
+    //--------------------------------------------------------------------------//
+
+	// Read domain parameters
+	if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu
+		h = domain_db->getScalar<double>( "voxel_length" );
+	}
+
+	// Single-fluid Navier-Stokes Model parameters
+	//if (stokes_db->keyExists( "timestepMax" )){
+	//	timestepMax = stokes_db->getScalar<int>( "timestepMax" );
+	//}
+    BoundaryCondition = 0;
+	if (stokes_db->keyExists( "BC" )){
+		BoundaryCondition = stokes_db->getScalar<int>( "BC" );
+	}
+	if (stokes_db->keyExists( "tolerance" )){
+		tolerance = stokes_db->getScalar<double>( "tolerance" );
+	}
+	if (stokes_db->keyExists( "tau" )){
+		tau = stokes_db->getScalar<double>( "tau" );
+	}
+	if (stokes_db->keyExists( "rho0" )){
+		rho0 = stokes_db->getScalar<double>( "rho0" );
+	}
+	if (stokes_db->keyExists( "nu_phys" )){
+		nu_phys = stokes_db->getScalar<double>( "nu_phys" );
+	}
+	if (stokes_db->keyExists( "rho_phys" )){
+		rho_phys = stokes_db->getScalar<double>( "rho_phys" );
+	}
+	if (stokes_db->keyExists( "F" )){
+		Fx = stokes_db->getVector<double>( "F" )[0];
+		Fy = stokes_db->getVector<double>( "F" )[1];
+		Fz = stokes_db->getVector<double>( "F" )[2];
+	}
+	if (stokes_db->keyExists( "Restart" )){
+		Restart = stokes_db->getScalar<bool>( "Restart" );
+	}
+	if (stokes_db->keyExists( "din" )){
+		din = stokes_db->getScalar<double>( "din" );
+	}
+	if (stokes_db->keyExists( "dout" )){
+		dout = stokes_db->getScalar<double>( "dout" );
+	}
+	if (stokes_db->keyExists( "flux" )){
+		flux = stokes_db->getScalar<double>( "flux" );
+	}	
+
+    // Re-calculate model parameters due to parameter read
+	mu=(tau-0.5)/3.0;
+    time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt]
+    den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density
+
+}
+
+void ScaLBL_StokesModel::ReadParams(string filename){
+    //NOTE the max time step is left unspecified
+
+	// read the input database 
+	db = std::make_shared<Database>( filename );
+	domain_db = db->getDatabase( "Domain" );
+	stokes_db = db->getDatabase( "Stokes" );
+	
+
+	//---------------------- Default model parameters --------------------------//		
+    rho_phys = 1000.0; //by default use water density; unit [kg/m^3]
+    nu_phys = 1.004e-6;//by default use water kinematic viscosity at 20C; unit [m^2/sec]    
+    h = 1.0;//image resolution;[um]
+	tau = 1.0;
+    mu = (tau-0.5)/3.0;//LB kinematic viscosity;unit [lu^2/lt]
+    time_conv = h*h*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt]
+    rho0 = 1.0;//LB density
+    den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density
+	tolerance = 1.0e-8;
+	Fx = Fy = 0.0;
+	Fz = 1.0e-5;
+    //--------------------------------------------------------------------------//
+
+	// Read domain parameters
+	if (domain_db->keyExists( "voxel_length" )){//default unit: um/lu
+		h = domain_db->getScalar<double>( "voxel_length" );
+	}
+
+	// Single-fluid Navier-Stokes Model parameters
+	//if (stokes_db->keyExists( "timestepMax" )){
+	//	timestepMax = stokes_db->getScalar<int>( "timestepMax" );
+	//}
+    BoundaryCondition = 0;
+	if (stokes_db->keyExists( "BC" )){
+		BoundaryCondition = stokes_db->getScalar<int>( "BC" );
+	}
+	if (stokes_db->keyExists( "tolerance" )){
+		tolerance = stokes_db->getScalar<double>( "tolerance" );
+	}
+	if (stokes_db->keyExists( "tau" )){
+		tau = stokes_db->getScalar<double>( "tau" );
+	}
+	if (stokes_db->keyExists( "rho0" )){
+		rho0 = stokes_db->getScalar<double>( "rho0" );
+	}
+	if (stokes_db->keyExists( "nu_phys" )){
+		nu_phys = stokes_db->getScalar<double>( "nu_phys" );
+	}
+	if (stokes_db->keyExists( "rho_phys" )){
+		rho_phys = stokes_db->getScalar<double>( "rho_phys" );
+	}
+	if (stokes_db->keyExists( "F" )){
+		Fx = stokes_db->getVector<double>( "F" )[0];
+		Fy = stokes_db->getVector<double>( "F" )[1];
+		Fz = stokes_db->getVector<double>( "F" )[2];
+	}
+	if (stokes_db->keyExists( "Restart" )){
+		Restart = stokes_db->getScalar<bool>( "Restart" );
+	}
+	if (stokes_db->keyExists( "din" )){
+		din = stokes_db->getScalar<double>( "din" );
+	}
+	if (stokes_db->keyExists( "dout" )){
+		dout = stokes_db->getScalar<double>( "dout" );
+	}
+	if (stokes_db->keyExists( "flux" )){
+		flux = stokes_db->getScalar<double>( "flux" );
+	}	
+
+    // Re-calculate model parameters due to parameter read
+	mu=(tau-0.5)/3.0;
+    time_conv = (h*h*1.0e-12)*mu/nu_phys;//time conversion factor from physical to LB unit; [sec/lt]
+    den_scale = rho_phys/rho0*(h*h*h*1.0e-18);//scale factor for density
+
+}
+
+void ScaLBL_StokesModel::SetDomain(){
+	Dm  = std::shared_ptr<Domain>(new Domain(domain_db,comm));      // full domain for analysis
+	Mask  = std::shared_ptr<Domain>(new Domain(domain_db,comm));    // mask domain removes immobile phases
+
+	// domain parameters
+	Nx = Dm->Nx;
+	Ny = Dm->Ny;
+	Nz = Dm->Nz;
+	Lx = Dm->Lx;
+	Ly = Dm->Ly;
+	Lz = Dm->Lz;
+	
+	N = Nx*Ny*Nz;
+	Distance.resize(Nx,Ny,Nz);
+	Velocity_x.resize(Nx,Ny,Nz);
+	Velocity_y.resize(Nx,Ny,Nz);
+	Velocity_z.resize(Nx,Ny,Nz);
+	
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
+	//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
+	MPI_Barrier(comm);
+	Dm->BoundaryCondition = BoundaryCondition;
+	Mask->BoundaryCondition = BoundaryCondition;
+	Dm->CommInit();
+	MPI_Barrier(comm);
+	
+	rank = Dm->rank();	
+	nprocx = Dm->nprocx();
+	nprocy = Dm->nprocy();
+	nprocz = Dm->nprocz();
+}
+
+void ScaLBL_StokesModel::ReadInput(){
+    
+    sprintf(LocalRankString,"%05d",Dm->rank());
+    sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
+    sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
+
+    
+    if (domain_db->keyExists( "Filename" )){
+    	auto Filename = domain_db->getScalar<std::string>( "Filename" );
+    	Mask->Decomp(Filename);
+    }
+    else if (domain_db->keyExists( "GridFile" )){
+    	// Read the local domain data
+    	auto input_id = readMicroCT( *domain_db, comm );
+    	// Fill the halo (assuming GCW of 1)
+    	array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
+    	ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
+    	ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
+    	fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
+    	Array<signed char> id_view;
+    	id_view.viewRaw( size1, Mask->id );
+    	fill.copy( input_id, id_view );
+    	fill.fill( id_view );
+    }
+    else{
+    	Mask->ReadIDs();
+    }
+
+    // Generate the signed distance map
+	// Initialize the domain and communication
+	Array<char> id_solid(Nx,Ny,Nz);
+	// Solve for the position of the solid phase
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				// Initialize the solid phase
+				if (Mask->id[n] > 0)	id_solid(i,j,k) = 1;
+				else	     	    id_solid(i,j,k) = 0;
+			}
+		}
+	}
+	// Initialize the signed distance function
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				// Initialize distance to +/- 1
+				Distance(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
+			}
+		}
+	}
+//	MeanFilter(Averages->SDs);
+	if (rank==0) printf("LB Single-Fluid Solver: initialized solid phase & converting to Signed Distance function \n");
+	CalcDist(Distance,id_solid,*Dm);
+    if (rank == 0) cout << "    Domain set." << endl;
+}
+
+void ScaLBL_StokesModel::Create(){
+	/*
+	 *  This function creates the variables needed to run a LBM 
+	 */
+	int rank=Mask->rank();
+	//.........................................................
+	// Initialize communication structures in averaging domain
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
+	Mask->CommInit();
+	Np=Mask->PoreCount();
+	//...........................................................................
+	if (rank==0)    printf ("LB Single-Fluid Solver: Create ScaLBL_Communicator \n");
+	// Create a communicator for the device (will use optimized layout)
+	// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
+	ScaLBL_Comm  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+
+	int Npad=(Np/16 + 2)*16;
+	if (rank==0)    printf ("LB Single-Fluid Solver: Set up memory efficient layout \n");
+	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
+	auto neighborList= new int[18*Npad];
+	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
+	MPI_Barrier(comm);
+	//...........................................................................
+	//                MAIN  VARIABLES ALLOCATED HERE
+	//...........................................................................
+	// LBM variables
+	if (rank==0)    printf ("LB Single-Fluid Solver: Allocating distributions \n");
+	//......................device distributions.................................
+	int dist_mem_size = Np*sizeof(double);
+	int neighborSize=18*(Np*sizeof(int));
+	//...........................................................................
+	ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
+	ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);  
+	ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
+	ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
+	//...........................................................................
+	// Update GPU data structures
+	if (rank==0)    printf ("LB Single-Fluid Solver: Setting up device map and neighbor list \n");
+	// copy the neighbor list 
+	ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
+	MPI_Barrier(comm);
+	
+}        
+
+void ScaLBL_StokesModel::Initialize(){
+	/*
+	 * This function initializes model
+	 */
+    if (rank==0) printf("LB Single-Fluid Solver: Initializing distributions \n");
+	if (rank==0) printf("****************************************************************\n");
+    ScaLBL_D3Q19_Init(fq, Np);
+
+	if (rank==0) printf("*****************************************************\n");
+	if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: \n");
+	if (rank==0) printf("      Time conversion factor: %.5g [sec/lt]\n", time_conv);
+	if (rank==0) printf("      Internal iteration: %i [lt]\n", timestepMax);
+	if (rank==0) printf("*****************************************************\n");
+}
+
+void ScaLBL_StokesModel::Run_Lite(double *ChargeDensity, double *ElectricField){
+	double rlx_setA=1.0/tau;
+	double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
+    timestep = 0;
+	while (timestep < timestepMax) {
+        //************************************************************************/
+        timestep++;
+        ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
+        ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv,
+                                     ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+        ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+        // Set boundary conditions
+        if (BoundaryCondition == 3){
+            ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+            ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+        }
+        else if (BoundaryCondition == 4){
+            din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+            ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+        }
+        else if (BoundaryCondition == 5){
+            ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+            ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+        }
+        ScaLBL_D3Q19_AAodd_StokesMRT(NeighborList, fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, 
+                                     0, ScaLBL_Comm->LastExterior(), Np);
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+        timestep++;
+        ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
+        ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv, 
+                                      ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+        ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+        // Set boundary conditions
+        if (BoundaryCondition == 3){
+            ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+            ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+        }
+        else if (BoundaryCondition == 4){
+            din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+            ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+        }
+        else if (BoundaryCondition == 5){
+            ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+            ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+        }
+        ScaLBL_D3Q19_AAeven_StokesMRT(fq, Velocity, ChargeDensity, ElectricField, rlx_setA, rlx_setB, Fx, Fy, Fz,rho0,den_scale,h,time_conv,
+                                      0, ScaLBL_Comm->LastExterior(), Np);
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+        //************************************************************************/
+    }
+}
+
+void ScaLBL_StokesModel::getVelocity(DoubleArray &Vel_x, DoubleArray &Vel_y, DoubleArray &Vel_z){
+    //get velocity in physical unit [m/sec]
+	ScaLBL_D3Q19_Momentum(fq, Velocity, Np);
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Vel_x);
+    Velocity_LB_to_Phys(Vel_x);
+    ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Vel_y);
+    Velocity_LB_to_Phys(Vel_y);
+    ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Vel_z);
+    Velocity_LB_to_Phys(Vel_z);
+    ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+}
+
+void ScaLBL_StokesModel::getVelocity_debug(int timestep){
+    //get velocity in physical unit [m/sec]
+	ScaLBL_D3Q19_Momentum(fq, Velocity, Np);
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+    DoubleArray PhaseField(Nx,Ny,Nz);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField);
+    Velocity_LB_to_Phys(PhaseField);
+	FILE *VELX_FILE;
+	sprintf(LocalRankFilename,"Velocity_X_Time_%i.%05i.raw",timestep,rank);
+	VELX_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELX_FILE);
+	fclose(VELX_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField);
+    Velocity_LB_to_Phys(PhaseField);
+	FILE *VELY_FILE;
+	sprintf(LocalRankFilename,"Velocity_Y_Time_%i.%05i.raw",timestep,rank);
+	VELY_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELY_FILE);
+	fclose(VELY_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField);
+    Velocity_LB_to_Phys(PhaseField);
+	FILE *VELZ_FILE;
+	sprintf(LocalRankFilename,"Velocity_Z_Time_%i.%05i.raw",timestep,rank);
+	VELZ_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELZ_FILE);
+	fclose(VELZ_FILE);
+
+}
+
+void ScaLBL_StokesModel::Velocity_LB_to_Phys(DoubleArray &Vel_reg){
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+                int idx=Map(i,j,k);
+				if (!(idx < 0)){
+                    Vel_reg(i,j,k) = Vel_reg(i,j,k)*(h*1.0e-6)/time_conv; 
+                }
+            }
+        }
+    }
+}
+
+vector<double> ScaLBL_StokesModel::computeElectricForceAvg(double *ChargeDensity, double *ElectricField){
+
+    double *Ex_host;
+    double *Ey_host;
+    double *Ez_host;
+    Ex_host = new double[Np];
+    Ey_host = new double[Np];
+    Ez_host = new double[Np];
+    
+    double *rhoE_host;
+    rhoE_host = new double[Np];
+
+	ScaLBL_CopyToHost(Ex_host,&ElectricField[0*Np],Np*sizeof(double));
+	ScaLBL_CopyToHost(Ey_host,&ElectricField[1*Np],Np*sizeof(double));
+	ScaLBL_CopyToHost(Ez_host,&ElectricField[2*Np],Np*sizeof(double));
+	ScaLBL_CopyToHost(rhoE_host,ChargeDensity,Np*sizeof(double));
+
+	double count_loc=0;
+	double count;
+    double Fx_avg,Fy_avg,Fz_avg;//average electric field induced force
+    double Fx_loc,Fy_loc,Fz_loc;
+    Fx_loc = Fy_loc = Fz_loc = 0.0;
+
+    for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
+        Fx_loc += rhoE_host[idx]*Ex_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fy_loc += rhoE_host[idx]*Ey_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz_loc += rhoE_host[idx]*Ez_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        count_loc+=1.0;
+    }
+    for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
+        Fx_loc += rhoE_host[idx]*Ex_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fy_loc += rhoE_host[idx]*Ey_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz_loc += rhoE_host[idx]*Ez_host[idx]*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        count_loc+=1.0;
+    }
+
+	MPI_Allreduce(&Fx_loc,&Fx_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&Fy_loc,&Fy_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&Fz_loc,&Fz_avg,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	
+	Fx_avg /= count;
+	Fy_avg /= count;
+	Fz_avg /= count;
+
+    vector<double>F_avg{Fx_avg,Fy_avg,Fz_avg};
+
+    delete [] Ex_host;
+    delete [] Ey_host;
+    delete [] Ez_host;
+    delete [] rhoE_host;
+
+    return F_avg;
+}
+
+double ScaLBL_StokesModel::CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField){
+    
+    //-----------------------------------------------------
+	ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z);
+	
+	double count_loc=0;
+	double count;
+	double vax,vay,vaz;
+	double vax_loc,vay_loc,vaz_loc;
+	vax_loc = vay_loc = vaz_loc = 0.f;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				if (Distance(i,j,k) > 0){
+					vax_loc += Velocity_x(i,j,k);
+					vay_loc += Velocity_y(i,j,k);
+					vaz_loc += Velocity_z(i,j,k);
+					count_loc+=1.0;
+				}
+			}
+		}
+	}
+	MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	
+	vax /= count;
+	vay /= count;
+	vaz /= count;
+    
+    vector<double> Eforce;
+    Eforce = computeElectricForceAvg(ChargeDensity,ElectricField);
+    double TFx = Fx+Eforce[0];//TF: total body force
+    double TFy = Fy+Eforce[1];
+    double TFz = Fz+Eforce[2];
+	double force_mag = sqrt(TFx*TFx+TFy*TFy+TFz*TFz);
+	double dir_x = TFx/force_mag;
+	double dir_y = TFy/force_mag;
+	double dir_z = TFz/force_mag;
+	if (force_mag == 0.0){
+		// default to z direction
+		dir_x = 0.0;
+		dir_y = 0.0;
+		dir_z = 1.0;
+		force_mag = 1.0;
+	}
+	double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z);
+	double error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate);
+	flow_rate_previous = flow_rate;
+    //----------------------------------------------------
+
+    //for debugging 
+    if (rank==0){
+        printf("StokesModel: error: %.5g\n",error);
+    }
+    return error;
+}
+
+void ScaLBL_StokesModel::Run(){
+	double rlx_setA=1.0/tau;
+	double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
+	
+	Minkowski Morphology(Mask);
+
+	if (rank==0){
+		bool WriteHeader=false;
+		FILE *log_file = fopen("Permeability.csv","r");
+		if (log_file != NULL)
+			fclose(log_file);
+		else
+			WriteHeader=true;
+
+		if (WriteHeader){
+			log_file = fopen("Permeability.csv","a+");
+			fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n");
+			fclose(log_file);
+		}
+	}
+
+	//.......create and start timer............
+	double starttime,stoptime,cputime;
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	starttime = MPI_Wtime();
+	if (rank==0) printf("****************************************************************\n");
+	if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: timestepMax = %i\n", timestepMax);
+	if (rank==0) printf("****************************************************************\n");
+	timestep=0;
+	double error = 1.0;
+	double flow_rate_previous = 0.0;
+	while (timestep < timestepMax && error > tolerance) {
+		//************************************************************************/
+		timestep++;
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
+		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
+		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		// Set boundary conditions
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
+		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
+		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		timestep++;
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
+		ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
+		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		// Set boundary conditions
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
+		ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
+		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		//************************************************************************/
+		
+		if (timestep%1000==0){
+			ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
+			ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x);
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y);
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z);
+			
+			double count_loc=0;
+			double count;
+			double vax,vay,vaz;
+			double vax_loc,vay_loc,vaz_loc;
+			vax_loc = vay_loc = vaz_loc = 0.f;
+			for (int k=1; k<Nz-1; k++){
+				for (int j=1; j<Ny-1; j++){
+					for (int i=1; i<Nx-1; i++){
+						if (Distance(i,j,k) > 0){
+							vax_loc += Velocity_x(i,j,k);
+							vay_loc += Velocity_y(i,j,k);
+							vaz_loc += Velocity_z(i,j,k);
+							count_loc+=1.0;
+						}
+					}
+				}
+			}
+			MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+			
+			vax /= count;
+			vay /= count;
+			vaz /= count;
+			
+			double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+			double dir_x = Fx/force_mag;
+			double dir_y = Fy/force_mag;
+			double dir_z = Fz/force_mag;
+			if (force_mag == 0.0){
+				// default to z direction
+				dir_x = 0.0;
+				dir_y = 0.0;
+				dir_z = 1.0;
+				force_mag = 1.0;
+			}
+			double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z);
+			
+			error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate);
+			flow_rate_previous = flow_rate;
+			
+			//if (rank==0) printf("Computing Minkowski functionals \n");
+			Morphology.ComputeScalar(Distance,0.f);
+			//Morphology.PrintAll();
+			double mu = (tau-0.5)/3.f;
+			double Vs = Morphology.V();
+			double As = Morphology.A();
+			double Hs = Morphology.H();
+			double Xs = Morphology.X();
+			Vs=sumReduce( Dm->Comm, Vs);
+			As=sumReduce( Dm->Comm, As);
+			Hs=sumReduce( Dm->Comm, Hs);
+			Xs=sumReduce( Dm->Comm, Xs);
+			double h = Dm->voxel_length;
+			double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag;
+			if (rank==0) {
+				printf("     %f\n",absperm);
+				FILE * log_file = fopen("Permeability.csv","a");
+				fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, 
+						h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm);
+				fclose(log_file);
+			}
+		}
+	}
+	//************************************************************************/
+	stoptime = MPI_Wtime();
+	if (rank==0) printf("-------------------------------------------------------------------\n");
+	// Compute the walltime per timestep
+	cputime = (stoptime - starttime)/timestep;
+	// Performance obtained from each node
+	double MLUPS = double(Np)/cputime/1000000;
+
+	if (rank==0) printf("********************************************************\n");
+	if (rank==0) printf("CPU time = %f \n", cputime);
+	if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
+	MLUPS *= nprocs;
+	if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
+	if (rank==0) printf("********************************************************\n");
+
+}
+
+void ScaLBL_StokesModel::VelocityField(){
+
+/*	Minkowski Morphology(Mask);
+	int SIZE=Np*sizeof(double);
+	ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE);
+
+	memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double));
+	Morphology.Initialize();
+	Morphology.UpdateMeshValues();
+	Morphology.ComputeLocal();
+	Morphology.Reduce();
+	
+	double count_loc=0;
+	double count;
+	double vax,vay,vaz;
+	double vax_loc,vay_loc,vaz_loc;
+	vax_loc = vay_loc = vaz_loc = 0.f;
+	for (int n=0; n<ScaLBL_Comm->LastExterior(); n++){
+		vax_loc += VELOCITY[n];
+		vay_loc += VELOCITY[Np+n];
+		vaz_loc += VELOCITY[2*Np+n];
+		count_loc+=1.0;
+	}
+	
+	for (int n=ScaLBL_Comm->FirstInterior(); n<ScaLBL_Comm->LastInterior(); n++){
+		vax_loc += VELOCITY[n];
+		vay_loc += VELOCITY[Np+n];
+		vaz_loc += VELOCITY[2*Np+n];
+		count_loc+=1.0;
+	}
+	MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	
+	vax /= count;
+	vay /= count;
+	vaz /= count;
+	
+	double mu = (tau-0.5)/3.f;
+	if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n");
+	if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, 
+						Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz);
+						*/
+	
+	std::vector<IO::MeshDataStruct> visData;
+	fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
+
+	auto VxVar = std::make_shared<IO::Variable>();
+	auto VyVar = std::make_shared<IO::Variable>();
+	auto VzVar = std::make_shared<IO::Variable>();
+	auto SignDistVar = std::make_shared<IO::Variable>();
+
+	IO::initialize("","silo","false");
+	// Create the MeshDataStruct	
+	visData.resize(1);
+	visData[0].meshName = "domain";
+	visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+	SignDistVar->name = "SignDist";
+	SignDistVar->type = IO::VariableType::VolumeVariable;
+	SignDistVar->dim = 1;
+	SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(SignDistVar);
+	
+	VxVar->name = "Velocity_x";
+	VxVar->type = IO::VariableType::VolumeVariable;
+	VxVar->dim = 1;
+	VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VxVar);
+	VyVar->name = "Velocity_y";
+	VyVar->type = IO::VariableType::VolumeVariable;
+	VyVar->dim = 1;
+	VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VyVar);
+	VzVar->name = "Velocity_z";
+	VzVar->type = IO::VariableType::VolumeVariable;
+	VzVar->dim = 1;
+	VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VzVar);
+	
+	Array<double>& SignData  = visData[0].vars[0]->data;
+	Array<double>& VelxData = visData[0].vars[1]->data;
+	Array<double>& VelyData = visData[0].vars[2]->data;
+	Array<double>& VelzData = visData[0].vars[3]->data;
+	
+    ASSERT(visData[0].vars[0]->name=="SignDist");
+    ASSERT(visData[0].vars[1]->name=="Velocity_x");
+    ASSERT(visData[0].vars[2]->name=="Velocity_y");
+    ASSERT(visData[0].vars[3]->name=="Velocity_z");
+	
+    fillData.copy(Distance,SignData);
+    fillData.copy(Velocity_x,VelxData);
+    fillData.copy(Velocity_y,VelyData);
+    fillData.copy(Velocity_z,VelzData);
+	
+    IO::writeData( timestep, visData, Dm->Comm );
+
+}

models/StokesModel.h

@@ -0,0 +1,92 @@
+/*
+ * Multi-relaxation time LBM Model
+ */
+#ifndef ScaLBL_StokesModel_INC
+#define ScaLBL_StokesModel_INC
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "analysis/Minkowski.h"
+#include "ProfilerApp.h"
+
+class ScaLBL_StokesModel{
+public:
+	ScaLBL_StokesModel(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_StokesModel();	
+	
+	// functions in they should be run
+	void ReadParams(string filename,int num_iter);
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+	void SetDomain();
+	void ReadInput();
+	void Create();
+	void Initialize();
+	void Run();
+	void Run_Lite(double *ChargeDensity, double *ElectricField);
+	void VelocityField();
+    void getVelocity(DoubleArray &Velx, DoubleArray &Vel_y, DoubleArray &Vel_z);
+    void getVelocity_debug(int timestep);
+    double CalVelocityConvergence(double& flow_rate_previous,double *ChargeDensity, double *ElectricField);
+	
+	bool Restart,pBC;
+	int timestep,timestepMax;
+	int BoundaryCondition;
+	double tau,mu;
+    double rho0;
+	double Fx,Fy,Fz,flux;
+	double din,dout;
+	double tolerance;
+    double nu_phys;
+    double rho_phys;
+    double time_conv;
+    double h;//image resolution
+    double den_scale;//scale factor for density
+	
+	int Nx,Ny,Nz,N,Np;
+	int rank,nprocx,nprocy,nprocz,nprocs;
+	double Lx,Ly,Lz;
+
+	std::shared_ptr<Domain> Dm;   // this domain is for analysis
+	std::shared_ptr<Domain> Mask; // this domain is for lbm
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> domain_db;
+    std::shared_ptr<Database> stokes_db;
+
+    IntArray Map;
+    DoubleArray Distance;
+    int *NeighborList;
+    double *fq;
+    double *Velocity;
+    double *Pressure;
+    
+    //Minkowski Morphology;
+    DoubleArray Velocity_x;
+    DoubleArray Velocity_y;
+    DoubleArray Velocity_z;
+private:
+	MPI_Comm comm;
+	
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+    char OutputFilename[200];
+   
+    //int rank,nprocs;
+    void LoadParams(std::shared_ptr<Database> db0);    	
+    void Velocity_LB_to_Phys(DoubleArray &Vel_reg);
+    vector<double> computeElectricForceAvg(double *ChargeDensity, double *ElectricField);
+};
+#endif

tests/CMakeLists.txt

@@ -4,6 +4,7 @@
 ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator )
+ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator )
 #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
 #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
@@ -48,6 +49,10 @@ ADD_LBPM_TEST( TestTorusEvolve )
 ADD_LBPM_TEST( TestTopo3D )
 ADD_LBPM_TEST( TestFluxBC )
 ADD_LBPM_TEST( TestMap )
+ADD_LBPM_TEST( TestPoissonSolver )
+ADD_LBPM_TEST( TestIonModel )
+ADD_LBPM_TEST( TestNernstPlanck )
+ADD_LBPM_TEST( TestPNP_Stokes )
 #ADD_LBPM_TEST( TestMRT )
 #ADD_LBPM_TEST( TestColorGrad )
 #ADD_LBPM_TEST( TestColorGradDFH )

tests/TestIonModel.cpp

@@ -0,0 +1,96 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <math.h>
+
+#include "models/IonModel.h"
+#include "models/MultiPhysController.h"
+#include "common/Utilities.h"
+
+using namespace std;
+
+//***************************************************************************
+// Test lattice-Boltzmann Ion Model coupled with Poisson equation
+//***************************************************************************
+
+int main(int argc, char **argv)
+{
+    // Initialize MPI and error handlers
+    Utilities::startup( argc, argv );
+
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        MPI_Comm comm;
+        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
+        int rank = comm_rank(comm);
+        int nprocs = comm_size(comm);
+
+        if (rank == 0){
+            printf("**************************************\n");
+            printf("Running Test for Ion Transport \n");
+            printf("**************************************\n");
+        }
+        // Initialize compute device
+        ScaLBL_SetDevice(rank);
+        ScaLBL_DeviceBarrier();
+        MPI_Barrier(comm);
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_IonModel IonModel(rank,nprocs,comm);
+        ScaLBL_Multiphys_Controller Study(rank,nprocs,comm);//multiphysics controller coordinating multi-model coupling
+
+        // Load controller information
+        Study.ReadParams(filename);
+
+        // Initialize LB-Ion model
+        IonModel.ReadParams(filename,Study.num_iter_Ion);
+        IonModel.SetDomain();    
+        IonModel.ReadInput();    
+        IonModel.Create();       
+        IonModel.Initialize();   
+        IonModel.DummyFluidVelocity();
+        IonModel.DummyElectricField();
+
+
+        int timestep=0;
+        double error = 1.0;
+        vector<double>ci_avg_previous{0.0,0.0};//assuming 1:1 solution
+        while (timestep < Study.timestepMax && error > Study.tolerance){
+            
+            timestep++;
+            IonModel.Run(IonModel.FluidVelocityDummy,IonModel.ElectricFieldDummy); //solve for ion transport and electric potential
+            timestep++;//AA operations
+
+            if (timestep%Study.analysis_interval==0){
+                error = IonModel.CalIonDenConvergence(ci_avg_previous);
+            }
+        }
+        IonModel.getIonConcentration_debug(timestep);
+
+        if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
+        if (rank==0) printf("*************************************************************\n");
+
+        PROFILE_STOP("Main");
+        PROFILE_SAVE("TestIonModel",1);
+        // ****************************************************
+        
+        MPI_Barrier(comm);
+        MPI_Comm_free(&comm);
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+}
+
+

tests/TestNernstPlanck.cpp

@@ -0,0 +1,108 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <math.h>
+
+#include "models/IonModel.h"
+#include "models/PoissonSolver.h"
+#include "models/MultiPhysController.h"
+#include "common/Utilities.h"
+
+using namespace std;
+
+//***************************************************************************
+// Test lattice-Boltzmann Ion Model coupled with Poisson equation
+//***************************************************************************
+
+int main(int argc, char **argv)
+{
+    // Initialize MPI
+    Utilities::startup( argc, argv );
+
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        MPI_Comm comm;
+        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
+        int rank = comm_rank(comm);
+        int nprocs = comm_size(comm);
+
+        if (rank == 0){
+            printf("********************************************************\n");
+            printf("Running Test for LB-Poisson-Ion Coupling \n");
+            printf("********************************************************\n");
+        }
+
+        // Initialize compute device
+        ScaLBL_SetDevice(rank);
+        ScaLBL_DeviceBarrier();
+        MPI_Barrier(comm);
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_IonModel IonModel(rank,nprocs,comm);
+        ScaLBL_Poisson PoissonSolver(rank,nprocs,comm); 
+        ScaLBL_Multiphys_Controller Study(rank,nprocs,comm);//multiphysics controller coordinating multi-model coupling
+
+        // Load controller information
+        Study.ReadParams(filename);
+
+        // Initialize LB-Ion model
+        IonModel.ReadParams(filename,Study.num_iter_Ion);
+        IonModel.SetDomain();    
+        IonModel.ReadInput();    
+        IonModel.Create();       
+        IonModel.Initialize();   
+        IonModel.DummyFluidVelocity();
+
+        // Initialize LB-Poisson model
+        PoissonSolver.ReadParams(filename);
+        PoissonSolver.SetDomain();    
+        PoissonSolver.ReadInput();    
+        PoissonSolver.Create();       
+        PoissonSolver.Initialize(0);   
+
+        int timestep=0;
+        double error = 1.0;
+        vector<double>ci_avg_previous{0.0,0.0};//assuming 1:1 solution
+        while (timestep < Study.timestepMax && error > Study.tolerance){
+            
+            timestep++;
+            PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
+            IonModel.Run(IonModel.FluidVelocityDummy,PoissonSolver.ElectricField); //solve for ion transport and electric potential
+            
+            timestep++;//AA operations
+
+            if (timestep%Study.analysis_interval==0){
+                error = IonModel.CalIonDenConvergence(ci_avg_previous);
+            }
+        }
+
+        PoissonSolver.getElectricPotential_debug(timestep);
+        PoissonSolver.getElectricField_debug(timestep);
+        IonModel.getIonConcentration_debug(timestep);
+
+        if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
+        if (rank==0) printf("*************************************************************\n");
+
+        PROFILE_STOP("Main");
+        PROFILE_SAVE("TestNernstPlanck",1);
+        // ****************************************************
+        //
+        MPI_Barrier(comm);
+        MPI_Comm_free(&comm);
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+}
+
+

tests/TestPNP_Stokes.cpp

@@ -0,0 +1,130 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <math.h>
+
+#include "models/IonModel.h"
+#include "models/StokesModel.h"
+#include "models/PoissonSolver.h"
+#include "models/MultiPhysController.h"
+#include "common/Utilities.h"
+
+using namespace std;
+
+//***************************************************************************
+// Test lattice-Boltzmann Ion Model coupled with Poisson equation
+//***************************************************************************
+
+int main(int argc, char **argv)
+{
+    // Initialize MPI and error handlers
+    Utilities::startup( argc, argv );
+
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        MPI_Comm comm;
+        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
+        int rank = comm_rank(comm);
+        int nprocs = comm_size(comm);
+
+        if (rank == 0){
+            printf("********************************************************\n");
+            printf("Running Test for LB-Poisson-Ion Coupling \n");
+            printf("********************************************************\n");
+        }
+        // Initialize compute device
+        ScaLBL_SetDevice(rank);
+        ScaLBL_DeviceBarrier();
+        MPI_Barrier(comm);
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_StokesModel StokesModel(rank,nprocs,comm);
+        ScaLBL_IonModel IonModel(rank,nprocs,comm);
+        ScaLBL_Poisson PoissonSolver(rank,nprocs,comm); 
+        ScaLBL_Multiphys_Controller Study(rank,nprocs,comm);//multiphysics controller coordinating multi-model coupling
+
+        // Load controller information
+        Study.ReadParams(filename);
+
+        // Load user input database files for Navier-Stokes and Ion solvers
+        StokesModel.ReadParams(filename);
+        IonModel.ReadParams(filename);
+
+        // Setup other model specific structures
+        StokesModel.SetDomain();    
+        StokesModel.ReadInput();    
+        StokesModel.Create();       // creating the model will create data structure to match the pore structure and allocate variables
+
+        IonModel.SetDomain();    
+        IonModel.ReadInput();    
+        IonModel.Create();       
+
+        // Get internal iteration number
+        StokesModel.timestepMax = Study.getStokesNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv);
+        StokesModel.Initialize();   // initializing the model will set initial conditions for variables
+
+        IonModel.timestepMax = Study.getIonNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv);
+        IonModel.Initialize();   
+
+        // Initialize LB-Poisson model
+        PoissonSolver.ReadParams(filename);
+        PoissonSolver.SetDomain();    
+        PoissonSolver.ReadInput();    
+        PoissonSolver.Create();       
+        PoissonSolver.Initialize(0);   
+
+
+        int timestep=0;
+        double error = 1.0;
+        double error_ion    = 1.0;
+        double error_stokes = 1.0;
+        vector<double>ci_avg_previous{0.0,0.0};//assuming 1:1 solution
+        double vel_avg_previous = 0.0;
+        while (timestep < Study.timestepMax && error > Study.tolerance){
+            
+            timestep++;
+            PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
+            StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity
+            IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential
+            
+            timestep++;//AA operations
+
+            if (timestep%Study.analysis_interval==0){
+                error_ion = IonModel.CalIonDenConvergence(ci_avg_previous);
+                error_stokes = StokesModel.CalVelocityConvergence(vel_avg_previous,IonModel.ChargeDensity,PoissonSolver.ElectricField);
+                error = max(error_ion,error_stokes);
+            }
+        }
+
+        PoissonSolver.getElectricPotential_debug(timestep);
+        PoissonSolver.getElectricField_debug(timestep);
+        IonModel.getIonConcentration_debug(timestep);
+        StokesModel.getVelocity_debug(timestep);
+
+        if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
+        if (rank==0) printf("*************************************************************\n");
+
+        PROFILE_STOP("Main");
+        PROFILE_SAVE("TestPNP_Stokes",1);
+        // ****************************************************
+        
+        MPI_Barrier(comm);
+        MPI_Comm_free(&comm);
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+}
+
+

tests/TestPoissonSolver.cpp

@@ -0,0 +1,103 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <math.h>
+
+#include "models/PoissonSolver.h"
+#include "common/Utilities.h"
+
+using namespace std;
+
+//********************************************************
+// Test lattice-Boltzmann solver of Poisson equation
+//********************************************************
+
+int main(int argc, char **argv)
+{
+    // Initialize MPI
+    Utilities::startup( argc, argv );
+    
+    {// Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        MPI_Comm comm;
+        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
+        int rank = comm_rank(comm);
+        int nprocs = comm_size(comm);
+
+        if (rank == 0){
+            printf("********************************************************\n");
+            printf("Running Test for LB-Poisson Solver \n");
+            printf("********************************************************\n");
+        }
+        // Initialize compute device
+        ScaLBL_SetDevice(rank);
+        ScaLBL_DeviceBarrier();
+        MPI_Barrier(comm);
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_Poisson PoissonSolver(rank,nprocs,comm); 
+
+        // Initialize LB-Poisson model
+        PoissonSolver.ReadParams(filename);
+        PoissonSolver.SetDomain();    
+        PoissonSolver.ReadInput();    
+        PoissonSolver.Create();       
+        if (PoissonSolver.TestPeriodic==true){
+            PoissonSolver.Initialize(PoissonSolver.TestPeriodicTimeConv);   
+        }
+        else {
+            PoissonSolver.Initialize(0);   
+        }
+
+        //Initialize dummy charge density for test
+        PoissonSolver.DummyChargeDensity();   
+
+        if (PoissonSolver.TestPeriodic==true){
+            if (rank==0) printf("Testing periodic voltage input is enabled. Total test time is %.3g[s], saving data every %.3g[s]; user-specified time resolution is %.3g[s/lt]\n",
+                                PoissonSolver.TestPeriodicTime,PoissonSolver.TestPeriodicSaveInterval,PoissonSolver.TestPeriodicTimeConv);
+            int timestep = 0;
+            int timeMax = int(PoissonSolver.TestPeriodicTime/PoissonSolver.TestPeriodicTimeConv);
+            int timeSave = int(PoissonSolver.TestPeriodicSaveInterval/PoissonSolver.TestPeriodicTimeConv);
+            while (timestep<timeMax){
+                timestep++;
+                PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,timestep);
+                if (timestep%timeSave==0){
+                    if (rank==0) printf("   Time = %.3g[s]; saving electric potential and field\n",timestep*PoissonSolver.TestPeriodicTimeConv);
+                    PoissonSolver.getElectricPotential_debug(timestep);
+                    PoissonSolver.getElectricField_debug(timestep);
+                }
+            }
+        }
+        else {
+            PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,1);
+            PoissonSolver.getElectricPotential_debug(1);
+            PoissonSolver.getElectricField_debug(1);
+        }
+
+        if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
+        if (rank==0) printf("*************************************************************\n");
+
+        PROFILE_STOP("Main");
+        PROFILE_SAVE("TestPoissonSolver",1);
+        // ****************************************************
+        
+        MPI_Barrier(comm);
+        MPI_Comm_free(&comm);
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+}
+
+

tests/lbpm_electrokinetic_SingleFluid_simulator.cpp

@@ -0,0 +1,135 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+#include <math.h>
+
+#include "models/IonModel.h"
+#include "models/StokesModel.h"
+#include "models/PoissonSolver.h"
+#include "models/MultiPhysController.h"
+#include "common/Utilities.h"
+#include "analysis/ElectroChemistry.h"
+
+using namespace std;
+
+//***************************************************************************
+// Test lattice-Boltzmann Ion Model coupled with Poisson equation
+//***************************************************************************
+
+int main(int argc, char **argv)
+{
+    // Initialize MPI and error handlers
+    Utilities::startup( argc, argv );
+
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        MPI_Comm comm;
+        MPI_Comm_dup(MPI_COMM_WORLD,&comm);
+        int rank = comm_rank(comm);
+        int nprocs = comm_size(comm);
+
+        if (rank == 0){
+            printf("********************************************************\n");
+            printf("Running LBPM electrokinetic single-fluid solver \n");
+            printf("********************************************************\n");
+        }
+        // Initialize compute device
+        ScaLBL_SetDevice(rank);
+        ScaLBL_DeviceBarrier();
+        MPI_Barrier(comm);
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_StokesModel StokesModel(rank,nprocs,comm);
+        ScaLBL_IonModel IonModel(rank,nprocs,comm);
+        ScaLBL_Poisson PoissonSolver(rank,nprocs,comm); 
+        ScaLBL_Multiphys_Controller Study(rank,nprocs,comm);//multiphysics controller coordinating multi-model coupling
+        
+        // Load controller information
+        Study.ReadParams(filename);
+
+        // Load user input database files for Navier-Stokes and Ion solvers
+        StokesModel.ReadParams(filename);
+        IonModel.ReadParams(filename);
+
+        // Setup other model specific structures
+        StokesModel.SetDomain();    
+        StokesModel.ReadInput();    
+        StokesModel.Create();       // creating the model will create data structure to match the pore structure and allocate variables
+
+        IonModel.SetDomain();    
+        IonModel.ReadInput();    
+        IonModel.Create();      
+        
+        // Create analysis object
+        ElectroChemistryAnalyzer Analysis(IonModel.Dm);
+
+        // Get internal iteration number
+        StokesModel.timestepMax = Study.getStokesNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv);
+        StokesModel.Initialize();   // initializing the model will set initial conditions for variables
+
+        IonModel.timestepMax = Study.getIonNumIter_PNP_coupling(StokesModel.time_conv,IonModel.time_conv);
+        IonModel.Initialize();   
+        // Get maximal time converting factor based on Sotkes and Ion solvers
+        Study.getTimeConvMax_PNP_coupling(StokesModel.time_conv,IonModel.time_conv);
+
+        // Initialize LB-Poisson model
+        PoissonSolver.ReadParams(filename);
+        PoissonSolver.SetDomain();    
+        PoissonSolver.ReadInput();    
+        PoissonSolver.Create();       
+        PoissonSolver.Initialize(Study.time_conv_max);   
+
+
+        int timestep=0;
+        while (timestep < Study.timestepMax){
+            
+            timestep++;
+            PoissonSolver.Run(IonModel.ChargeDensity,timestep);//solve Poisson equtaion to get steady-state electrical potental
+            StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity
+            IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential
+            
+            timestep++;//AA operations
+
+            if (timestep%Study.analysis_interval==0){
+            	Analysis.Basic(IonModel,PoissonSolver,StokesModel,timestep);
+            }
+            if (timestep%Study.visualization_interval==0){
+            	Analysis.WriteVis(IonModel,PoissonSolver,StokesModel,Study.db,timestep);
+            	/*  PoissonSolver.getElectricPotential(timestep);
+                PoissonSolver.getElectricField(timestep);
+                IonModel.getIonConcentration(timestep);
+                StokesModel.getVelocity(timestep);
+            	 */
+            }
+        }
+
+        if (rank==0) printf("Save simulation raw data at maximum timestep\n");
+    	Analysis.WriteVis(IonModel,PoissonSolver,StokesModel,Study.db,timestep);
+
+        if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
+        if (rank==0) printf("*************************************************************\n");
+
+        PROFILE_STOP("Main");
+        PROFILE_SAVE("lbpm_electrokinetic_SingleFluid_simulator",1);
+        // ****************************************************
+        
+        MPI_Barrier(comm);
+        MPI_Comm_free(&comm);
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+  Utilities::shutdown();
+}
+
+