Merge branch 'master' of github.com:JamesEMcClure/LBPM-WIA

This reverts commit 2e47e9c306.

IO/netcdf.cpp

@@ -204,6 +204,7 @@ std::vector<size_t> getAttDim( int fid, const std::string& att )
 {
     std::vector<size_t> dim(1,0);
     int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
+    CHECK_NC_ERR( err );
     return dim;
 }
 std::vector<std::string> getVarNames( int fid )

README.md

@@ -23,7 +23,7 @@ Configure, build & install procedure
 
 * edit configure script from sample_scripts directory and configure (e.g.)
 
-   `/path/to/LBPM-WIA/sample_scripts/configure_desktop`
+   `/path/to/LBPM/sample_scripts/configure_desktop`
 
 * compile and install
 

StackTrace/StackTrace.cpp

@@ -7,6 +7,7 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cerrno>
 #include <csignal>
 #include <cstring>
 #include <iostream>
@@ -348,8 +349,11 @@ static inline int exec3( const char *cmd, FUNCTION &fun )
         if ( buffer[0] != 0 )
             fun( buffer );
     }
-    auto status = pclose( pipe );
-    int code    = WEXITSTATUS( status );
+    int code = pclose( pipe );
+    if ( errno == ECHILD ) {
+        errno = 0;
+        code  = 0;
+    }
     std::this_thread::yield(); // Allow any signals to process
     resetSignal( SIGCHLD );    // Clear child exited
     return code;
@@ -1741,7 +1745,7 @@ std::vector<int> StackTrace::defaultSignalsToCatch()
  *  Set the signal handlers                                                  *
  ****************************************************************************/
 static std::function<void( const StackTrace::abort_error &err )> abort_fun;
-static StackTrace::abort_error rethrow()
+StackTrace::abort_error rethrow()
 {
     StackTrace::abort_error error;
 #ifdef USE_LINUX
@@ -1775,14 +1779,14 @@ static StackTrace::abort_error rethrow()
     }
     return error;
 }
-static void term_func_abort( int sig )
+void StackTrace::terminateFunctionSignal( int sig )
 {
     StackTrace::abort_error err;
     err.type      = StackTrace::terminateType::signal;
     err.signal    = sig;
     err.bytes     = StackTrace::Utilities::getMemoryUsage();
     err.stack     = StackTrace::backtrace();
-    err.stackType = StackTrace::printStackType::global;
+    err.stackType = StackTrace::getDefaultStackType();
     abort_fun( err );
 }
 static bool signals_set[256] = { false };
@@ -1829,7 +1833,7 @@ void StackTrace::setErrorHandler( std::function<void( const StackTrace::abort_er
 {
     abort_fun = abort;
     std::set_terminate( term_func );
-    setSignals( defaultSignalsToCatch(), &term_func_abort );
+    setSignals( defaultSignalsToCatch(), &terminateFunctionSignal );
     std::set_unexpected( term_func );
 }
 void StackTrace::clearErrorHandler()
@@ -2215,7 +2219,7 @@ void StackTrace::cleanupStackTrace( multi_stack_info &stack )
             // Remove callstack (and all children) for threads that are just contributing
             bool test = function.find( "_callstack_signal_handler" ) != npos ||
                         function.find( "getGlobalCallStacks" ) != npos ||
-                        function.find( "(" ) == npos;
+                        function.find( "backtrace" ) != npos || function.find( "(" ) == npos;
             if ( test ) {
                 it = stack.children.erase( it );
                 continue;
@@ -2515,3 +2519,11 @@ const char *StackTrace::abort_error::what() const noexcept
             d_msg.erase( i, 1 );
     return d_msg.c_str();
 }
+
+
+/****************************************************************************
+ * Get/Set default stack type                                                *
+ ****************************************************************************/
+static StackTrace::printStackType abort_stackType = StackTrace::printStackType::global;
+void StackTrace::setDefaultStackType( StackTrace::printStackType type ) { abort_stackType = type; }
+StackTrace::printStackType StackTrace::getDefaultStackType() { return abort_stackType; }

StackTrace/StackTrace.h

@@ -261,6 +261,10 @@ void clearSignals();
 void raiseSignal( int signal );
 
 
+//! Default function to abort after catching a signal
+void terminateFunctionSignal( int signal );
+
+
 //! Return a list of all signals that can be caught
 std::vector<int> allSignalsToCatch();
 
@@ -304,6 +308,13 @@ multi_stack_info generateFromString( const std::vector<std::string> &str );
 multi_stack_info generateFromString( const std::string &str );
 
 
+//! Set default stack type
+void setDefaultStackType( StackTrace::printStackType );
+
+//! Get default stack type
+StackTrace::printStackType getDefaultStackType();
+
+
 } // namespace StackTrace
 
 

StackTrace/Utilities.cpp

@@ -8,8 +8,10 @@
 #include <cstring>
 #include <fstream>
 #include <iostream>
+#include <mutex>
 #include <sstream>
 #include <stdexcept>
+#include <typeinfo>
 
 #ifdef USE_MPI
 #include "mpi.h"
@@ -19,6 +21,10 @@
 #include "MemoryApp.h"
 #endif
 
+#ifdef USE_GCOV
+extern "C" void __gcov_flush( void );
+#endif
+
 
 #define perr std::cerr
 
@@ -65,6 +71,12 @@
 // clang-format on
 
 
+#ifdef __GNUC__
+#define USE_ABI
+#include <cxxabi.h>
+#endif
+
+
 namespace StackTrace {
 
 
@@ -96,13 +108,12 @@ inline size_t findfirst( const std::vector<TYPE> &X, TYPE Y )
 /****************************************************************************
  *  Function to terminate the program                                        *
  ****************************************************************************/
-static bool abort_throwException      = false;
-static printStackType abort_stackType = printStackType::global;
-static int force_exit                 = 0;
+static bool abort_throwException = false;
+static int force_exit            = 0;
 void Utilities::setAbortBehavior( bool throwException, int stackType )
 {
     abort_throwException = throwException;
-    abort_stackType      = static_cast<printStackType>( stackType );
+    StackTrace::setDefaultStackType( static_cast<printStackType>( stackType ) );
 }
 void Utilities::abort( const std::string &message, const std::string &filename, const int line )
 {
@@ -112,16 +123,28 @@ void Utilities::abort( const std::string &message, const std::string &filename,
     err.type      = terminateType::abort;
     err.line      = line;
     err.bytes     = Utilities::getMemoryUsage();
-    err.stackType = abort_stackType;
+    err.stackType = StackTrace::getDefaultStackType();
     err.stack     = StackTrace::backtrace();
     throw err;
 }
-static void terminate( const StackTrace::abort_error &err )
+static std::mutex terminate_mutex;
+static inline void callAbort()
 {
+#ifdef USE_GCOV
+    __gcov_flush();
+#endif
+    terminate_mutex.unlock();
+    std::abort();
+}
+void Utilities::terminate( const StackTrace::abort_error &err )
+{
+    // Lock mutex to ensure multiple threads do not try to abort simultaneously
+    terminate_mutex.lock();
+    // Clear the error handlers
     clearErrorHandler();
     // Print the message and abort
     if ( force_exit > 1 ) {
-        std::abort();
+        callAbort();
     } else if ( !abort_throwException ) {
         // Use MPI_abort (will terminate all processes)
         force_exit = 2;
@@ -135,10 +158,11 @@ static void terminate( const StackTrace::abort_error &err )
             MPI_Abort( MPI_COMM_WORLD, -1 );
         }
 #endif
-        std::abort();
+        callAbort();
     } else {
         perr << err.what();
-        std::abort();
+        perr.flush();
+        callAbort();
     }
 }
 
@@ -149,7 +173,7 @@ static void terminate( const StackTrace::abort_error &err )
 static void setTerminateErrorHandler()
 {
     // Set the terminate routine for runtime errors
-    StackTrace::setErrorHandler( terminate );
+    StackTrace::setErrorHandler( Utilities::terminate );
 }
 void Utilities::setErrorHandlers()
 {
@@ -293,4 +317,18 @@ std::string Utilities::exec( const string_view &cmd, int &exit_code )
 }
 
 
+/****************************************************************************
+ *  Get the type name                                                        *
+ ****************************************************************************/
+std::string Utilities::getTypeName( const std::type_info &id )
+{
+    std::string name = id.name();
+#if defined( USE_ABI )
+    int status;
+    name = abi::__cxa_demangle( name.c_str(), 0, 0, &status );
+#endif
+    return name;
+}
+
+
 } // namespace StackTrace

StackTrace/Utilities.h

@@ -4,6 +4,7 @@
 #include <stdexcept>
 #include <string>
 #include <thread>
+#include <typeinfo>
 
 #include "StackTrace/StackTrace.h"
 #include "StackTrace/string_view.h"
@@ -28,9 +29,14 @@ void abort( const std::string &message, const std::string &filename, const int l
 void setAbortBehavior( bool throwException, int stackType = 2 );
 
 
+//! Function to terminate the application
+void terminate( const StackTrace::abort_error &err );
+
+
 //! Function to set the error handlers
 void setErrorHandlers();
 
+
 //! Function to clear the error handlers
 void clearErrorHandlers();
 
@@ -92,6 +98,18 @@ void cause_segfault();
 std::string exec( const StackTrace::string_view &cmd, int &exit_code );
 
 
+//! Return the hopefully demangled name of the given type
+std::string getTypeName( const std::type_info &id );
+
+
+//! Return the hopefully demangled name of the given type
+template<class TYPE>
+inline std::string getTypeName()
+{
+    return getTypeName( typeid( TYPE ) );
+}
+
+
 } // namespace Utilities
 } // namespace StackTrace
 

StackTrace/string_view.h

@@ -119,7 +119,7 @@ public:
         int result = 0;
         for ( int i = 0; i < N && result == 0; i++ )
             if ( d_data[i] != other[i] )
-                result = d_data[i] < other[i] ? -i : i;
+                result = d_data[i] < other[i] ? -( i + 1 ) : ( i + 1 );
         if ( result == 0 )
             result = size() == other.size() ? 0 : size() < other.size() ? -1 : 1;
         return result;

analysis/GreyPhase.cpp

@@ -0,0 +1,259 @@
+#include "analysis/GreyPhase.h"
+
+// Constructor
+GreyPhaseAnalysis::GreyPhaseAnalysis(std::shared_ptr <Domain> dm):
+	Dm(dm)
+{
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	// Global arrays
+    SDs.resize(Nx,Ny,Nz);           SDs.fill(0);
+	Porosity.resize(Nx,Ny,Nz);      Porosity.fill(0);
+	//PhaseID.resize(Nx,Ny,Nz);       PhaseID.fill(0);
+	Rho_n.resize(Nx,Ny,Nz);       	Rho_n.fill(0);
+	Rho_w.resize(Nx,Ny,Nz);       	Rho_w.fill(0);
+	Pressure.resize(Nx,Ny,Nz);      Pressure.fill(0);
+	//Phi.resize(Nx,Ny,Nz);         	Phi.fill(0);
+	//DelPhi.resize(Nx,Ny,Nz);        DelPhi.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	//.........................................
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("timelog.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("timelog.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");				
+		}
+	}
+}
+
+
+// Destructor
+GreyPhaseAnalysis::~GreyPhaseAnalysis()
+{
+
+}
+
+void GreyPhaseAnalysis::Write(int timestep)
+{
+
+}
+
+void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B, double GreyPorosity)
+{
+	Fx = force_x;
+	Fy = force_y;
+	Fz = force_z;
+	rho_n = rhoA;
+	rho_w = rhoB;
+	nu_n = (tauA-0.5)/3.f;
+	nu_w = (tauB-0.5)/3.f;
+	gamma_wn = 6.0*alpha;
+	beta = B;
+    grey_porosity = GreyPorosity;
+}
+
+void GreyPhaseAnalysis::Basic(){
+	int i,j,k,n,imin,jmin,kmin,kmax;
+
+	// If external boundary conditions are set, do not average over the inlet
+	kmin=1; kmax=Nz-1;
+	imin=jmin=1;
+	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
+	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
+
+	Water_local.reset();
+	Oil_local.reset();
+	double count_w = 0.0;
+	double count_n = 0.0;
+	for (k=kmin; k<kmax; k++){
+		for (j=jmin; j<Ny-1; j++){
+			for (i=imin; i<Nx-1; i++){
+				n = k*Nx*Ny + j*Nx + i;
+				// Compute volume averages
+				if ( Dm->id[n] > 0 ){
+					// compute density
+					double nA = Rho_n(n);
+					double nB = Rho_w(n);				
+					double phi = (nA-nB)/(nA+nB);					
+					double porosity = Porosity(n);
+					Water_local.M += rho_w*nB*porosity;
+					Water_local.Px += porosity*rho_w*nB*Vel_x(n);
+					Water_local.Py += porosity*rho_w*nB*Vel_y(n);
+					Water_local.Pz += porosity*rho_w*nB*Vel_z(n);
+					Oil_local.M += rho_n*nA*porosity;
+					Oil_local.Px += porosity*rho_n*nA*Vel_x(n);
+					Oil_local.Py += porosity*rho_n*nA*Vel_y(n);
+					Oil_local.Pz += porosity*rho_n*nA*Vel_z(n);
+
+					if ( phi > 0.99 ){
+						Oil_local.p += Pressure(n);
+					    //Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB);
+						count_n += 1.0;
+					}
+					else if ( phi < -0.99 ){
+						Water_local.p += Pressure(n);
+					    //Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB);
+						count_w += 1.0;
+					}
+				}
+			}
+		}
+	}
+	Oil.M=sumReduce( Dm->Comm, Oil_local.M);
+	Oil.Px=sumReduce( Dm->Comm, Oil_local.Px);
+	Oil.Py=sumReduce( Dm->Comm, Oil_local.Py);
+	Oil.Pz=sumReduce( Dm->Comm, Oil_local.Pz);
+
+	Water.M=sumReduce( Dm->Comm, Water_local.M);
+	Water.Px=sumReduce( Dm->Comm, Water_local.Px);
+	Water.Py=sumReduce( Dm->Comm, Water_local.Py);
+	Water.Pz=sumReduce( Dm->Comm, Water_local.Pz);
+
+
+	//Oil.p /= Oil.M;	
+	//Water.p /= Water.M;
+	count_w=sumReduce( Dm->Comm, count_w);
+	count_n=sumReduce( Dm->Comm, count_n);
+	if (count_w > 0.0)
+		Water.p=sumReduce( Dm->Comm, Water_local.p) / count_w;
+	else 
+		Water.p = 0.0;
+	if (count_n > 0.0)
+		Oil.p=sumReduce( Dm->Comm, Oil_local.p) / count_n;
+	else 
+		Oil.p = 0.0;
+
+	// check for NaN
+	bool err=false;
+	if (Water.M != Water.M) err=true;
+	if (Water.p != Water.p) err=true;
+	if (Water.Px != Water.Px) err=true;
+	if (Water.Py != Water.Py) err=true;
+	if (Water.Pz != Water.Pz) err=true;
+
+	if (Oil.M != Oil.M) err=true;
+	if (Oil.p != Oil.p) err=true;
+	if (Oil.Px != Oil.Px) err=true;
+	if (Oil.Py != Oil.Py) err=true;
+	if (Oil.Pz != Oil.Pz) err=true;
+	
+	if (Dm->rank() == 0){
+		double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+		double dir_x = 0.0;
+		double dir_y = 0.0;
+		double dir_z = 0.0;
+		if (force_mag > 0.0){
+			dir_x = Fx/force_mag;
+			dir_y = Fy/force_mag;
+			dir_z = Fz/force_mag;
+		}
+		else {
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+		}
+		if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){
+			// compute the pressure drop
+			double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0;
+			double length = ((Nz-2)*Dm->nprocz());
+			force_mag -= pressure_drop/length;
+		}
+		if (force_mag == 0.0){
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+			force_mag = 1.0;
+		}
+		saturation=Water.M/(Water.M + Oil.M); // assume constant density
+		water_flow_rate=grey_porosity*saturation*(Water.Px*dir_x + Water.Py*dir_y + Water.Pz*dir_z)/Water.M;
+		oil_flow_rate  =grey_porosity*(1.0-saturation)*(Oil.Px*dir_x + Oil.Py*dir_y + Oil.Pz*dir_z)/Oil.M;
+
+		double h = Dm->voxel_length;		
+        //TODO check if need greyporosity or domain porosity ? - compare to analytical solution
+		double krn = h*h*nu_n*oil_flow_rate / force_mag ;
+		double krw = h*h*nu_w*water_flow_rate / force_mag;
+		//printf("   water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*oil_flow_rate, Water.p, Oil.p); 
+		fflush(TIMELOG); 
+	}
+
+	if (err==true){
+		// exception if simulation produceds NaN
+		printf("GreyPhaseAnalysis.cpp: NaN encountered, may need to check simulation parameters \n");
+	}	
+	ASSERT(err==false);
+}
+/*
+inline void InterfaceTransportMeasures( double beta, double rA, double rB, double nA, double nB, 
+		double nx, double ny, double nz, double ux, double uy, double uz, interface &I){
+	
+	double A1,A2,A3,A4,A5,A6;
+	double B1,B2,B3,B4,B5,B6;
+	double nAB,delta;
+	// Instantiate mass transport distributions
+	// Stationary value - distribution 0
+	nAB = 1.0/(nA+nB);
+	//...............................................
+	// q = 0,2,4
+	// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nx;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
+	B1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
+	A2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
+	B2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+
+	//...............................................
+	// Cq = {0,1,0}
+	delta = beta*nA*nB*nAB*0.1111111111111111*ny;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A3 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
+	B3 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
+	A4 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
+	B4 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+
+	//...............................................
+	// q = 4
+	// Cq = {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nz;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A5 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
+	B5 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
+	A6 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
+	B6 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+	
+	double unx = (A1-A2);
+	double uny = (A3-A4);
+	double unz = (A5-A6);
+	double uwx = (B1-B2);
+	double uwy = (B3-B4);
+	double uwz = (B5-B6);
+	
+	I.Mn += rA*nA;
+	I.Mw += rB*nB;
+	I.Pnx += rA*nA*unx;
+	I.Pny += rA*nA*uny;
+	I.Pnz += rA*nA*unz;
+	I.Pwx += rB*nB*uwx;
+	I.Pwy += rB*nB*uwy;
+	I.Pwz += rB*nB*uwz;
+	I.Kn += rA*nA*(unx*unx + uny*uny + unz*unz);
+	I.Kw += rB*nB*(uwx*uwx + uwy*uwy + uwz*uwz);
+
+}
+*/

analysis/GreyPhase.h

@@ -0,0 +1,71 @@
+/*
+ * Sub-phase averaging tools
+ */
+
+#ifndef GreyPhase_INC
+#define GreyPhase_INC
+
+#include <vector>
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+
+class GreyPhase{
+ public:
+  double p;
+  double M,Px,Py,Pz;
+  void reset(){
+    p=M=Px=Py=Pz=0.0;
+  }
+
+ private:
+};
+
+class GreyPhaseAnalysis{
+public:
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+    double grey_porosity;
+	// outputs
+	double saturation,water_flow_rate, oil_flow_rate;
+
+	//simulation outputs (averaged values)
+	GreyPhase Water, Oil;
+	GreyPhase Water_local, Oil_local;
+	//...........................................................................
+    int Nx,Ny,Nz;
+	//IntArray PhaseID;		// Phase ID array 
+	DoubleArray SDs;		// contains porosity map 
+	DoubleArray Porosity;		// contains porosity map 
+	DoubleArray Rho_n;	// density field 
+	DoubleArray Rho_w;	// density field 
+	//DoubleArray Phi;		// phase indicator field
+	//DoubleArray DelPhi;		// Magnitude of Gradient of the phase indicator field
+	DoubleArray Pressure; 	// pressure field
+	DoubleArray Vel_x;		// velocity field
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+
+	GreyPhaseAnalysis(std::shared_ptr <Domain> Dm);
+	~GreyPhaseAnalysis();
+	
+	void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta, double GreyPorosity);
+	void Basic();
+	void Write(int time);
+
+private:
+	FILE *TIMELOG;
+};
+
+#endif
+

analysis/Minkowski.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -106,7 +107,21 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
 					Xi -= 0.5;
 				}
 				// Euler characteristic -- each vertex shared by four cubes
-				Xi += 0.25*double(object.VertexCount);
+				//Xi += 0.25*double(object.VertexCount);
+				// check if vertices are at corners
+				for (int idx=0; idx<object.VertexCount; idx++){
+				  /*auto P1 = object.vertex.coords(idx);
+					if ( remainder(P1.x,1.0)==0.0 && remainder(P1.y,1.0)==0.0  && remainder(P1.z,1.0)==0.0 ){
+					  Xi += 0.125;
+					}
+					else
+				  */
+					Xi += 0.25;
+				}
+				/*double nside_extern = double(npts);
+				double nside_intern = double(npts)-3.0;
+				EulerChar=0.0;
+				if (npts > 0)	EulerChar = (0.25*nvert - nside_intern - 0.5*nside_extern + nface); */
 			}
 		}
 	}
@@ -143,7 +158,7 @@ void Minkowski::MeasureObject(){
 	 *    0 - labels the object
 	 *    1 - labels the rest of the 
 	 */
-
+        //DoubleArray smooth_distance(Nx,Ny,Nz);
 	for (int k=0; k<Nz; k++){
 		for (int j=0; j<Ny; j++){
 			for (int i=0; i<Nx; i++){
@@ -152,6 +167,44 @@ void Minkowski::MeasureObject(){
 		}
 	}	
 	CalcDist(distance,id,*Dm);
+	//Mean3D(distance,smooth_distance);
+	//Eikonal(distance, id, *Dm, 20, {true, true, true});
+	ComputeScalar(distance,0.0);
+}
+
+
+void Minkowski::MeasureObject(double factor, const DoubleArray &Phi){
+	/*
+	 *  compute the distance to an object 
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 *    0 - labels the object
+	 *    1 - labels the rest of the 
+	 */
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				distance(i,j,k) =2.0*double(id(i,j,k))-1.0;
+			}
+		}
+	}	
+	CalcDist(distance,id,*Dm);
+	
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				double value = Phi(i,j,k);
+				double dist_value = distance(i,j,k);
+				if (dist_value < 2.5 &&  dist_value > -2.5) {
+					double new_distance = factor*log((1.0+value)/(1.0-value));
+					if (dist_value*new_distance < 0.0 )
+						new_distance = (-1.0)*new_distance;
+					distance(i,j,k) = new_distance;					
+				}
+			}
+		}
+	}
+	
 	ComputeScalar(distance,0.0);
 
 }
@@ -201,6 +254,50 @@ int Minkowski::MeasureConnectedPathway(){
 	return n_connected_components; 
 }
 
+int Minkowski::MeasureConnectedPathway(double factor, const DoubleArray &Phi){
+	/*
+	 * compute the connected pathway for object with LABEL in id field
+	 * compute the labels for connected components
+	 * compute the distance to the connected pathway
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 */
+	
+	char LABEL = 0;
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if (id(i,j,k) == LABEL){
+					distance(i,j,k) = 1.0;
+				}
+				else
+					distance(i,j,k) = -1.0;
+			}
+		}
+	}
+	
+	// Extract only the connected part of NWP
+	double vF=0.0; 
+	n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
+//	int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
+	MPI_Barrier(Dm->Comm);
+	
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if ( label(i,j,k) == 0){
+					id(i,j,k) = 0;
+				}
+				else{
+					id(i,j,k) = 1;
+				}
+			}
+		}
+	}
+	MeasureObject(factor,Phi);
+	return n_connected_components; 
+}
+
 
 void Minkowski::PrintAll()
 {

analysis/Minkowski.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -25,6 +26,7 @@
 #include "common/Communication.h"
 #include "analysis/analysis.h"
 #include "analysis/distance.h"
+#include "analysis/filters.h"
 
 #include "common/Utilities.h"
 #include "common/MPI_Helpers.h"
@@ -77,7 +79,9 @@ public:
 	Minkowski(std::shared_ptr <Domain> Dm);
 	~Minkowski();
 	void MeasureObject();
+	void MeasureObject(double factor, const DoubleArray &Phi);
 	int MeasureConnectedPathway();
+	int MeasureConnectedPathway(double factor, const DoubleArray &Phi);
 	void ComputeScalar(const DoubleArray& Field, const double isovalue);
 
 	void PrintAll();

analysis/PointList.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/SubPhase.cpp

@@ -93,7 +93,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
 		{
 			// If timelog is empty, write a short header to list the averages
 			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
-			fprintf(TIMELOG,"sw krw krn qw qn pw pn\n");				
+			fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");				
 		}
 	}
 }
@@ -109,35 +109,35 @@ SubPhase::~SubPhase()
 void SubPhase::Write(int timestep)
 {
 	if (Dm->rank()==0){
-		fprintf(SUBPHASE,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn); 
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",gwc.p, gwd.p, gnc.p, gnd.p);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",gwc.Py, gwd.Py, giwn.Pwy, gnc.Py, gnd.Py, giwn.Pny);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",gwc.Pz, gwd.Pz, giwn.Pwz, gnc.Pz, gnd.Pz, giwn.Pnz);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",gwc.K, gwd.K, giwn.Kw, gnc.K, gnd.K, giwn.Kn);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",gwc.V, gwc.A, gwc.H, gwc.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %i ",gwd.V, gwd.A, gwd.H, gwd.X, gwd.Nc);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",gnc.V, gnc.A, gnc.H, gnc.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %i ",gnd.V, gnd.A, gnd.H, gnd.X, gnd.Nc);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",giwn.V, giwn.A, giwn.H, giwn.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %i\n",giwnc.V, giwnc.A, giwnc.H, giwnc.X, giwnc.Nc);
+		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn); 
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Py, gwd.Py, giwn.Pwy, gnc.Py, gnd.Py, giwn.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Pz, gwd.Pz, giwn.Pwz, gnc.Pz, gnd.Pz, giwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.K, gwd.K, giwn.Kw, gnc.K, gnd.K, giwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.V, gwc.A, gwc.H, gwc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",gwd.V, gwd.A, gwd.H, gwd.X, gwd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gnc.V, gnc.A, gnc.H, gnc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",gnd.V, gnd.A, gnd.H, gnd.X, gnd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",giwn.V, giwn.A, giwn.H, giwn.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i\n",giwnc.V, giwnc.A, giwnc.H, giwnc.X, giwnc.Nc);
 		fflush(SUBPHASE);
 	}
 	else{
-		fprintf(SUBPHASE,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",wc.p, wd.p, nc.p, nd.p);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",wc.Py, wd.Py, iwn.Pwy, nc.Py, nd.Py, iwn.Pny);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",wc.Pz, wd.Pz, iwn.Pwz, nc.Pz, nd.Pz, iwn.Pnz);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %.5g %.5g ",wc.K, wd.K, iwn.Kw, nc.K, nd.K, iwn.Kn);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",wc.V, wc.A, wc.H, wc.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %i ",wd.V, wd.A, wd.H, wd.X, wd.Nc);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",nc.V, nc.A, nc.H, nc.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g %i ",nd.V, nd.A, nd.H, nd.X, nd.Nc);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g ",iwn.V, iwn.A, iwn.H, iwn.X);
-		fprintf(SUBPHASE,"%.5g %.5g %.5g %.5g\n",iwnc.V, iwnc.A, iwnc.H, iwnc.X);
+		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Py, wd.Py, iwn.Pwy, nc.Py, nd.Py, iwn.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Pz, wd.Pz, iwn.Pwz, nc.Pz, nd.Pz, iwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.K, wd.K, iwn.Kw, nc.K, nd.K, iwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.V, wc.A, wc.H, wc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",wd.V, wd.A, wd.H, wd.X, wd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",nc.V, nc.A, nc.H, nc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",nd.V, nd.A, nd.H, nd.X, nd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",iwn.V, iwn.A, iwn.H, iwn.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g\n",iwnc.V, iwnc.A, iwnc.H, iwnc.X);
 	}
 
 }
@@ -160,19 +160,15 @@ void SubPhase::Basic(){
 
 	// If external boundary conditions are set, do not average over the inlet
 	kmin=1; kmax=Nz-1;
-	if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
-	if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
-
 	imin=jmin=1;
-	// If inlet/outlet layers exist use these as default
+	/*// If inlet/outlet layers exist use these as default
 	if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
 	if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
 	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
 	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
-	
+	*/
 	nb.reset(); wb.reset();
 
-	double nA,nB;
 	double count_w = 0.0;
 	double count_n = 0.0;
 	
@@ -270,9 +266,26 @@ void SubPhase::Basic(){
 	
 	if (Dm->rank() == 0){
 		double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
-		double dir_x = Fx/force_mag;
-		double dir_y = Fy/force_mag;
-		double dir_z = Fz/force_mag;
+		double dir_x = 0.0;
+		double dir_y = 0.0;
+		double dir_z = 0.0;
+		if (force_mag > 0.0){
+			dir_x = Fx/force_mag;
+			dir_y = Fy/force_mag;
+			dir_z = Fz/force_mag;
+		}
+		else {
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+		}
+		if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){
+			// compute the pressure drop
+			double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0;
+			double length = ((Nz-2)*Dm->nprocz());
+			force_mag -= pressure_drop/length;
+		}
 		if (force_mag == 0.0){
 			// default to z direction
 			dir_x = 0.0;
@@ -283,14 +296,14 @@ void SubPhase::Basic(){
 		double saturation=gwb.V/(gwb.V + gnb.V);
 		double water_flow_rate=gwb.V*(gwb.Px*dir_x + gwb.Py*dir_y + gwb.Pz*dir_z)/gwb.M / Dm->Volume;
 		double not_water_flow_rate=gnb.V*(gnb.Px*dir_x + gnb.Py*dir_y + gnb.Pz*dir_z)/gnb.M/ Dm->Volume;
-		double total_flow_rate = water_flow_rate + not_water_flow_rate;
-		double fractional_flow= water_flow_rate / total_flow_rate;
+		//double total_flow_rate = water_flow_rate + not_water_flow_rate;
+		//double fractional_flow = water_flow_rate / total_flow_rate;
 
 		double h = Dm->voxel_length;		
 		double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
 		double krw = h*h*nu_w*water_flow_rate / force_mag;
 		//printf("   water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
-		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*h*h*water_flow_rate,h*h*h*not_water_flow_rate, gwb.p, gnb.p); 
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p); 
 		fflush(TIMELOG);
 	}
 	if (err==true){
@@ -363,16 +376,17 @@ void SubPhase::Full(){
 
 	// If external boundary conditions are set, do not average over the inlet
 	kmin=1; kmax=Nz-1;
-	if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
-	if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
-
+	/*if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == 0) kmin=4;
+	if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
+	*/
 	imin=jmin=1;
-	// If inlet layers exist use these as default
+	/*// If inlet layers exist use these as default
+	 * NOTE -- excluding inlet / outlet will screw up topological averages!!!
 	if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
 	if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
 	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
 	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
-		
+	*/
 	nd.reset();	nc.reset(); wd.reset();	wc.reset();	iwn.reset();	iwnc.reset();
 
  	Dm->CommunicateMeshHalo(Phi);
@@ -413,13 +427,13 @@ void SubPhase::Full(){
 		}
 	}
 	// measure the whole object
-	morph_n->MeasureObject();
+	morph_n->MeasureObject();//0.5/beta,Phi);
 	nd.V = morph_n->V(); 
 	nd.A = morph_n->A(); 
 	nd.H = morph_n->H(); 
 	nd.X = morph_n->X(); 
 	// measure only the connected part
-	nd.Nc = morph_n->MeasureConnectedPathway();
+	nd.Nc = morph_n->MeasureConnectedPathway();//0.5/beta,Phi);
 	nc.V = morph_n->V(); 
 	nc.A = morph_n->A(); 
 	nc.H = morph_n->H(); 
@@ -461,13 +475,13 @@ void SubPhase::Full(){
 			}
 		}
 	}	
-	morph_w->MeasureObject();
+	morph_w->MeasureObject();//-0.5/beta,Phi);
 	wd.V = morph_w->V(); 
 	wd.A = morph_w->A(); 
 	wd.H = morph_w->H(); 
 	wd.X = morph_w->X(); 
 	// measure only the connected part
-	wd.Nc = morph_w->MeasureConnectedPathway();
+	wd.Nc = morph_w->MeasureConnectedPathway();//-0.5/beta,Phi);
 	wc.V = morph_w->V(); 
 	wc.A = morph_w->A(); 
 	wc.H = morph_w->H(); 
@@ -683,7 +697,8 @@ void SubPhase::Full(){
 }
 
 
-void SubPhase::AggregateLabels(char *FILENAME){
+void SubPhase::AggregateLabels( const std::string& filename )
+{
 	
 	int nx = Dm->Nx;
 	int ny = Dm->Ny;
@@ -707,7 +722,7 @@ void SubPhase::AggregateLabels(char *FILENAME){
 	}
 	MPI_Barrier(Dm->Comm);
 
-	Dm->AggregateLabels(FILENAME);
+	Dm->AggregateLabels( filename );
 
 }
 

analysis/SubPhase.h

@@ -101,7 +101,7 @@ public:
 	void Basic();
 	void Full();
 	void Write(int time);
-    void AggregateLabels(char *FILENAME);
+    void AggregateLabels( const std::string& filename );
 
 private:
 	FILE *TIMELOG;

analysis/TwoPhase.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -234,6 +236,7 @@ TwoPhase::~TwoPhase()
 
 void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData)
 {
+    NULL_USE( Beta );
   /*double factor,temp,value;
 		factor=0.5/Beta;
 	// Initialize to -1,1 (segmentation)
@@ -657,8 +660,8 @@ void TwoPhase::ComputeLocal()
 
 void TwoPhase::AssignComponentLabels()
 {
-	int LabelNWP=1;
-	int LabelWP=2;
+	//int LabelNWP=1;
+	//int LabelWP=2;
 	// NOTE: labeling the wetting phase components is tricky! One sandstone media had over 800,000 components
 	// NumberComponents_WP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelWP,Label_WP);
 	// treat all wetting phase is connected
@@ -1202,6 +1205,8 @@ void TwoPhase::Reduce()
 
 void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT)
 {
+    NULL_USE( viscosity );
+    NULL_USE( IFT );
 	awn_global *= D;
 	ans_global *= D;
 	ans_global *= D;

analysis/TwoPhase.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/analysis.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/analysis.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/dcel.cpp

@@ -1,7 +1,5 @@
 #include "analysis/dcel.h"
 
-
-
 DECL::DECL(){
 }
 
@@ -15,6 +13,25 @@ int DECL::Face(int index){
 	return FaceData[index];
 }
 
+void DECL::Write(){
+	int e1,e2,e3;
+	FILE *TRIANGLES;
+	TRIANGLES = fopen("triangles.stl","w");
+	fprintf(TRIANGLES,"solid \n");
+	for (int idx=0; idx<TriangleCount; idx++){
+		e1 = Face(idx); 
+		e2 = halfedge.next(e1);
+		e3 = halfedge.next(e2);
+		auto P1 = vertex.coords(halfedge.v1(e1));
+		auto P2 = vertex.coords(halfedge.v1(e2));
+		auto P3 = vertex.coords(halfedge.v1(e3));
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P1.x,P1.y,P1.z);
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P2.x,P2.y,P2.z);
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P3.x,P3.y,P3.z);
+	}
+	fclose(TRIANGLES);
+}
+
 void DECL::LocalIsosurface(const DoubleArray& A, double value, const int i, const int j, const int k){
 	Point P,Q;
 	Point PlaceHolder;
@@ -350,241 +367,43 @@ double DECL::EdgeAngle(int edge)
 	return angle;
 }
 
-void Isosurface(DoubleArray &A, const double &v)
+void iso_surface(const Array<double>&Field, const double isovalue)
 {
-	Point P,Q;
-	Point PlaceHolder;
-	Point C0,C1,C2,C3,C4,C5,C6,C7;
-
-	int TriangleCount;
-	int VertexCount;
-	int CubeIndex;
-
-	Point VertexList[12];
-	Point NewVertexList[12];
-	int LocalRemap[12];
-
-	Point cellvertices[20];
-    std::array<std::array<int,3>,20> Triangles;
-    Triangles.fill( { 0 } );
-
-	// Values from array 'A' at the cube corners
-	double CubeValues[8];
-
-	int Nx = A.size(0);
-	int Ny = A.size(1);
-	int Nz = A.size(2);
-
-	// Points corresponding to cube corners
-	C0.x = 0.0; C0.y = 0.0; C0.z = 0.0;
-	C1.x = 1.0; C1.y = 0.0; C1.z = 0.0;
-	C2.x = 1.0; C2.y = 1.0; C2.z = 0.0;
-	C3.x = 0.0; C3.y = 1.0; C3.z = 0.0;
-	C4.x = 0.0; C4.y = 0.0; C4.z = 1.0;
-	C5.x = 1.0; C5.y = 0.0; C5.z = 1.0;
-	C6.x = 1.0; C6.y = 1.0; C6.z = 1.0;
-	C7.x = 0.0; C7.y = 1.0; C7.z = 1.0;							
-
-	std::vector<std::array<int,6>> HalfEdge;
-	for (int k=1; k<Nz-1; k++){		
-		for (int j=1; j<Ny-1; j++){
-			for (int i=1; i<Nx-1; i++){
-				// Set the corner values for this cube
-				CubeValues[0] = A(i,j,k);
-				CubeValues[1] = A(i+1,j,k);
-				CubeValues[2] = A(i+1,j+1,k);
-				CubeValues[3] = A(i,j+1,k);
-				CubeValues[4] = A(i,j,k+1);
-				CubeValues[5] = A(i+1,j,k+1);
-				CubeValues[6] = A(i+1,j+1,k+1);
-				CubeValues[7] = A(i,j+1,k+1);
-
-				//Determine the index into the edge table which
-				//tells us which vertices are inside of the surface
-				CubeIndex = 0;
-				if (CubeValues[0] < 0.0f) CubeIndex |= 1;
-				if (CubeValues[1] < 0.0f) CubeIndex |= 2;
-				if (CubeValues[2] < 0.0f) CubeIndex |= 4;
-				if (CubeValues[3] < 0.0f) CubeIndex |= 8;
-				if (CubeValues[4] < 0.0f) CubeIndex |= 16;
-				if (CubeValues[5] < 0.0f) CubeIndex |= 32;
-				if (CubeValues[6] < 0.0f) CubeIndex |= 64;
-				if (CubeValues[7] < 0.0f) CubeIndex |= 128;
-
-				//Find the vertices where the surface intersects the cube
-				if (edgeTable[CubeIndex] & 1){
-					P = VertexInterp(C0,C1,CubeValues[0],CubeValues[1]);
-					VertexList[0] = P;
-					Q = C0;
-				}
-				if (edgeTable[CubeIndex] & 2){
-					P = VertexInterp(C1,C2,CubeValues[1],CubeValues[2]);
-					VertexList[1] = P;
-					Q = C1;
-				}
-				if (edgeTable[CubeIndex] & 4){
-					P = VertexInterp(C2,C3,CubeValues[2],CubeValues[3]);
-					VertexList[2] =	P;
-					Q = C2;
-				}
-				if (edgeTable[CubeIndex] & 8){
-					P = VertexInterp(C3,C0,CubeValues[3],CubeValues[0]);
-					VertexList[3] =	P;
-					Q = C3;
-				}
-				if (edgeTable[CubeIndex] & 16){
-					P = VertexInterp(C4,C5,CubeValues[4],CubeValues[5]);
-					VertexList[4] =	P;
-					Q = C4;
-				}
-				if (edgeTable[CubeIndex] & 32){
-					P = VertexInterp(C5,C6,CubeValues[5],CubeValues[6]);
-					VertexList[5] = P;
-					Q = C5;
-				}
-				if (edgeTable[CubeIndex] & 64){ 
-					P = VertexInterp(C6,C7,CubeValues[6],CubeValues[7]);
-					VertexList[6] = P;
-					Q = C6;
-				}
-				if (edgeTable[CubeIndex] & 128){
-					P = VertexInterp(C7,C4,CubeValues[7],CubeValues[4]);
-					VertexList[7] =	P;
-					Q = C7;
-				}
-				if (edgeTable[CubeIndex] & 256){
-					P = VertexInterp(C0,C4,CubeValues[0],CubeValues[4]);
-					VertexList[8] =	P;
-					Q = C0;
-				}
-				if (edgeTable[CubeIndex] & 512){
-					P = VertexInterp(C1,C5,CubeValues[1],CubeValues[5]);
-					VertexList[9] =	P;
-					Q = C1;
-				}
-				if (edgeTable[CubeIndex] & 1024){
-					P = VertexInterp(C2,C6,CubeValues[2],CubeValues[6]);
-					VertexList[10] = P;
-					Q = C2;
-				}
-				if (edgeTable[CubeIndex] & 2048){
-					P = VertexInterp(C3,C7,CubeValues[3],CubeValues[7]);
-					VertexList[11] = P;
-					Q = C3;
-				}
-
-				VertexCount=0;
-				for (int idx=0;idx<12;idx++)
-					LocalRemap[idx] = -1;
-
-				for (int idx=0;triTable[CubeIndex][idx]!=-1;idx++)
-				{
-					if(LocalRemap[triTable[CubeIndex][idx]] == -1)
-					{
-						NewVertexList[VertexCount] = VertexList[triTable[CubeIndex][idx]];
-						LocalRemap[triTable[CubeIndex][idx]] = VertexCount;
-						VertexCount++;
-					}
-				}
-
-				for (int idx=0;idx<VertexCount;idx++) {
-					P = NewVertexList[idx];
-					//P.x  += i;
-					//P.y  += j;
-					//P.z  += k;
-					cellvertices[idx] = P;
-				}
-
-				TriangleCount = 0;
-				for (int idx=0;triTable[CubeIndex][idx]!=-1;idx+=3) {
-					Triangles[TriangleCount][0] = LocalRemap[triTable[CubeIndex][idx+0]];
-					Triangles[TriangleCount][1] = LocalRemap[triTable[CubeIndex][idx+1]];
-					Triangles[TriangleCount][2] = LocalRemap[triTable[CubeIndex][idx+2]];
-					TriangleCount++;
-				}
-				int nTris = TriangleCount;
-
-				// Now add the local values to the DECL data structure
-				HalfEdge.resize(nTris*3);
-				int idx_edge=0;
-				for (int idx=0; idx<TriangleCount; idx++){
-					int V1 = Triangles[idx][0];
-					int V2 = Triangles[idx][1];
-					int V3 = Triangles[idx][2];
-					// first edge: V1->V2 
-					HalfEdge[idx_edge][0] = V1;  // first vertex
-					HalfEdge[idx_edge][1] = V2;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge+2;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge+1;  // next edge
-					idx_edge++;
-					// second edge: V2->V3
-					HalfEdge[idx_edge][0] = V2;  // first vertex
-					HalfEdge[idx_edge][1] = V3;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge-1;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge+1;  // next edge
-					idx_edge++;
-					// third edge: V3->V1
-					HalfEdge[idx_edge][0] = V3;  // first vertex
-					HalfEdge[idx_edge][1] = V1;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge-1;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge-2;  // next edge
-					idx_edge++;
-				}
-				int EdgeCount=idx_edge;
-				for (int idx=0; idx<EdgeCount; idx++){
-					int V1=HalfEdge[idx][0];
-					int V2=HalfEdge[idx][1];
-					// Find all the twins within the cube
-					for (int jdx=0; idx<EdgeCount; jdx++){
-						if (HalfEdge[jdx][1] == V1 && HalfEdge[jdx][0] == V2){
-							// this is the pair
-							HalfEdge[idx][3] = jdx;
-							HalfEdge[jdx][3] = idx;
-						}
-						if (HalfEdge[jdx][1] == V2 && HalfEdge[jdx][0] == V1 && !(idx==jdx)){
-							std::printf("WARNING: half edges with identical orientation! \n");
-						}
-					}	
-					// Use "ghost" twins if edge is on a cube face
-					P = cellvertices[V1];
-					Q = cellvertices[V2];
-					if (P.x == 0.0 && Q.x == 0.0) HalfEdge[idx_edge][3] = -1;  // ghost twin for x=0 face
-					if (P.x == 1.0 && Q.x == 1.0) HalfEdge[idx_edge][3] = -2;  // ghost twin for x=1 face
-					if (P.y == 0.0 && Q.y == 0.0) HalfEdge[idx_edge][3] = -3;  // ghost twin for y=0 face
-					if (P.y == 1.0 && Q.y == 1.0) HalfEdge[idx_edge][3] = -4;  // ghost twin for y=1 face
-					if (P.z == 0.0 && Q.z == 0.0) HalfEdge[idx_edge][3] = -5;  // ghost twin for z=0 face
-					if (P.z == 1.0 && Q.z == 1.0) HalfEdge[idx_edge][3] = -6;  // ghost twin for z=1 face
-				}
-				// Find all the angles
-				for (int idx=0; idx<EdgeCount; idx++){
-					int V1=HalfEdge[idx][0];
-					int V2=HalfEdge[idx][1];
-					int T1= HalfEdge[idx_edge][2];
-					int twin=HalfEdge[idx_edge][3];
-					if (twin == -1){
-						
-					}
-				}
-
-				// Map vertices to global coordinates
-				for (int idx=0;idx<VertexCount;idx++) {
-					P = cellvertices[idx];
-					P.x  += i;
-					P.y  += j;
-					P.z  += k;
-					cellvertices[idx] = P;
-				}
-			}
-		}
-	}
+  DECL object;
+  int e1,e2,e3;
+  FILE *TRIANGLES;
+  TRIANGLES = fopen("isosurface.stl","w");
+  fprintf(TRIANGLES,"solid isosurface\n");
+  int Nx = Field.size(0);
+  int Ny = Field.size(1);
+  int Nz = Field.size(2);
+  for (int k=1; k<Nz-1; k++){
+    for (int j=1; j<Ny-1; j++){
+      for (int i=1; i<Nx-1; i++){
+        object.LocalIsosurface(Field,isovalue,i,j,k);
+        for (int idx=0; idx<object.TriangleCount; idx++){
+          e1 = object.Face(idx);
+          e2 = object.halfedge.next(e1);
+          e3 = object.halfedge.next(e2);
+          auto P1 = object.vertex.coords(object.halfedge.v1(e1));
+          auto P2 = object.vertex.coords(object.halfedge.v1(e2));
+          auto P3 = object.vertex.coords(object.halfedge.v1(e3));
+	  auto Normal  = object.TriNormal(e1);
+	  // P1.x += 1.0*i; P1.y += 1.0*j; P1.z +=1.0*k;
+          //P2.x += 1.0*i; P2.y += 1.0*j; P2.z +=1.0*k;
+          //P3.x += 1.0*i; P3.y += 1.0*j; P3.z +=1.0*k;
+          fprintf(TRIANGLES,"facet normal %f %f %f\n",Normal.x,Normal.y,Normal.z);
+	  fprintf(TRIANGLES,"   outer loop\n");
+	  fprintf(TRIANGLES,"      vertex %f %f %f\n",P1.x,P1.y,P1.z);
+          fprintf(TRIANGLES,"      vertex %f %f %f\n",P2.x,P2.y,P2.z);
+          fprintf(TRIANGLES,"      vertex %f %f %f\n",P3.x,P3.y,P3.z);
+	  fprintf(TRIANGLES,"   endloop\n");
+	  fprintf(TRIANGLES,"endfacet\n");
 
+        }
+      }
+    }
+  }
+  fprintf(TRIANGLES,"endsolid isosurface\n");
+  fclose(TRIANGLES);
 }
-
-
-

analysis/dcel.h

@@ -1,3 +1,6 @@
+#ifndef DCEL_INC
+#define DCEL_INC
+
 #include <vector>
 #include "analysis/pmmc.h"
 
@@ -67,6 +70,7 @@ public:
 	Vertex vertex;
 	Halfedge halfedge;
 	void LocalIsosurface(const DoubleArray& A, double value, int i, int j, int k);
+	void Write();
 	int Face(int index);
 	
 	double origin(int edge);
@@ -78,3 +82,7 @@ public:
 private:
 	std::vector<int> FaceData;
 };
+
+void iso_surface(const Array<double>&Field, const double isovalue);
+
+#endif

analysis/distance.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -197,6 +198,148 @@ void CalcVecDist( Array<Vec> &d, const Array<int> &ID0, const Domain &Dm,
     }
 }
 
+double Eikonal(DoubleArray &Distance, const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic){
+
+  /*
+   * This routine converts the data in the Distance array to a signed distance
+   * by solving the equation df/dt = sign(1-|grad f|), where Distance provides
+   * the values of f on the mesh associated with domain Dm
+   * It has been tested with segmented data initialized to values [-1,1]
+   * and will converge toward the signed distance to the surface bounding the associated phases
+   *
+   * Reference:
+   * Min C (2010) On reinitializing level set functions, Journal of Computational Physics229
+   */
+
+  int i,j,k;
+  double dt=0.1;
+  double Dx,Dy,Dz;
+  double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
+  double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
+  double sign,norm;
+  double LocalVar,GlobalVar,LocalMax,GlobalMax;
+
+  int xdim,ydim,zdim;
+  xdim=Dm.Nx-2;
+  ydim=Dm.Ny-2;
+  zdim=Dm.Nz-2;
+  //fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
+  fillHalo<double> fillData(  Dm.Comm, Dm.rank_info, {xdim, ydim, zdim}, {1,1,1}, 50, 1, {true,true,true}, periodic );
+
+  // Arrays to store the second derivatives
+  DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
+  DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
+  DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
+
+  int count = 0;
+  while (count < timesteps){
+
+    // Communicate the halo of values
+    fillData.fill(Distance);
+
+    // Compute second order derivatives
+    for (k=1;k<Dm.Nz-1;k++){
+      for (j=1;j<Dm.Ny-1;j++){
+	for (i=1;i<Dm.Nx-1;i++){
+	  Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
+	  Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
+	  Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
+	}
+      }
+    }
+    fillData.fill(Dxx);
+    fillData.fill(Dyy);
+    fillData.fill(Dzz);
+
+    LocalMax=LocalVar=0.0;
+    // Execute the next timestep
+    for (k=1;k<Dm.Nz-1;k++){
+      for (j=1;j<Dm.Ny-1;j++){
+	for (i=1;i<Dm.Nx-1;i++){
+
+	  int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
+
+	  sign = -1;
+	  if (ID(i,j,k) == 1) sign = 1;
+
+	  // local second derivative terms
+	  Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
+	  Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
+	  Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
+	  Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
+	  Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
+	  Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
+
+	  /* //............Compute upwind derivatives ...................
+                    Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
+                    Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
+                    Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
+                    Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+                    Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+                    Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+	  */
+	  Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
+	  Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
+	  Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
+
+	  Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+	  Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+	  Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+
+	  // Compute upwind derivatives for Godunov Hamiltonian
+	  if (sign < 0.0){
+	    if (Dxp + Dxm > 0.f)  Dx = Dxp*Dxp;
+	    else Dx = Dxm*Dxm;
+
+	    if (Dyp + Dym > 0.f)  Dy = Dyp*Dyp;
+	    else Dy = Dym*Dym;
+
+	    if (Dzp + Dzm > 0.f)  Dz = Dzp*Dzp;
+	    else Dz = Dzm*Dzm;
+	  }
+	  else{
+
+	    if (Dxp + Dxm < 0.f)  Dx = Dxp*Dxp;
+	    else Dx = Dxm*Dxm;
+
+	    if (Dyp + Dym < 0.f)  Dy = Dyp*Dyp;
+	    else Dy = Dym*Dym;
+
+	    if (Dzp + Dzm < 0.f)  Dz = Dzp*Dzp;
+	    else Dz = Dzm*Dzm;
+	  }
+
+	  //Dx = max(Dxp*Dxp,Dxm*Dxm);
+	  //Dy = max(Dyp*Dyp,Dym*Dym);
+	  //Dz = max(Dzp*Dzp,Dzm*Dzm);
+
+	  norm=sqrt(Dx + Dy + Dz);
+	  if (norm > 1.0) norm=1.0;
+
+	  Distance(i,j,k) += dt*sign*(1.0 - norm);
+	  LocalVar +=  dt*sign*(1.0 - norm);
+
+	  if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
+	    LocalMax = fabs(dt*sign*(1.0 - norm));
+	}
+      }
+    }
+
+    MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
+    MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
+    GlobalVar /= Dm.Volume;
+    count++;
+
+    if (count%50 == 0 && Dm.rank()==0 )
+      printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
+
+    if (fabs(GlobalMax) < 1e-5){
+      if (Dm.rank()==0) printf("Exiting with max tolerance of 1e-5 \n");
+      count=timesteps;
+    }
+  }
+  return GlobalVar;
+}
 
 // Explicit instantiations
 template void CalcDist<float>( Array<float>&, const Array<char>&, const Domain&, const std::array<bool,3>&, const std::array<double,3>& );

analysis/distance.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -31,6 +32,16 @@ struct Vec {
 };
 inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; }
 
+inline double minmod(double &a, double &b){
+
+  double value;
+
+  value = a;
+  if ( a*b < 0.0)    value=0.0;
+  else if (fabs(a) > fabs(b)) value = b;
+
+  return value;
+}
 
 /*!
  * @brief  Calculate the distance using a simple method
@@ -55,4 +66,16 @@ void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
 void CalcVecDist( Array<Vec> &Distance, const Array<int> &ID, const Domain &Dm,
     const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );
 
+
+/*!
+ * @brief  Calculate the distance based on solution of Eikonal equation
+ * @details  This routine calculates the signed distance to the nearest domain surface.
+ * @param[out] Distance     Distance function
+ * @param[in] ID            Domain id
+ * @param[in] Dm            Domain information
+ * @param[in] timesteps      number of timesteps to run for Eikonal solver
+ * @param[in] periodic      Directions that are periodic 
+ */
+double Eikonal(DoubleArray &Distance,  const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic);
+
 #endif

analysis/filters.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -17,6 +18,33 @@
 #include "math.h"
 #include "ProfilerApp.h"
 
+void Mean3D( const Array<double> &Input, Array<double> &Output )
+{
+	PROFILE_START("Mean3D");
+	// Perform a 3D Mean filter on Input array
+	int i,j,k;
+
+	int Nx = int(Input.size(0));
+	int Ny = int(Input.size(1));
+	int Nz = int(Input.size(2));
+
+	for (k=1; k<Nz-1; k++){
+		for (j=1; j<Ny-1; j++){
+			for (i=1; i<Nx-1; i++){
+			  double MeanValue = Input(i,j,k);
+			  // next neighbors
+			  MeanValue += Input(i+1,j,k)+Input(i,j+1,k)+Input(i,j,k+1)+Input(i-1,j,k)+Input(i,j-1,k)+Input(i,j,k-1);
+			  MeanValue += Input(i+1,j+1,k)+Input(i-1,j+1,k)+Input(i+1,j-1,k)+Input(i-1,j-1,k);
+			  MeanValue += Input(i+1,j,k+1)+Input(i-1,j,k+1)+Input(i+1,j,k-1)+Input(i-1,j,k-1);
+			  MeanValue += Input(i,j+1,k+1)+Input(i,j-1,k+1)+Input(i,j+1,k-1)+Input(i,j-1,k-1);
+			  MeanValue += Input(i+1,j+1,k+1)+Input(i-1,j+1,k+1)+Input(i+1,j-1,k+1)+Input(i-1,j-1,k+1);
+			  MeanValue += Input(i+1,j+1,k-1)+Input(i-1,j+1,k-1)+Input(i+1,j-1,k-1)+Input(i-1,j-1,k-1);
+			  Output(i,j,k) = MeanValue/27.0;
+			}
+		}
+	}
+	PROFILE_STOP("Mean3D");
+}
 
 void Med3D( const Array<float> &Input, Array<float> &Output )
 {

analysis/filters.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -19,6 +20,13 @@
 
 #include "common/Array.h"
 
+/*!
+ * @brief  Filter image
+ * @details  This routine performs a mean filter
+ * @param[in] Input     Input image
+ * @param[out] Output   Output image
+ */
+void Mean3D( const Array<double> &Input, Array<double> &Output );
 
 /*!
  * @brief  Filter image
@@ -28,7 +36,6 @@
  */
 void Med3D( const Array<float> &Input, Array<float> &Output );
 
-
 /*!
  * @brief  Filter image
  * @details  This routine performs a non-linear local means filter

analysis/histogram.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/imfilter.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/imfilter.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/morphology.cpp

@@ -34,9 +34,6 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	int nx = Dm->Nx;
 	int ny = Dm->Ny;
 	int nz = Dm->Nz;
-	int iproc = Dm->iproc();
-	int jproc = Dm->jproc();
-	int kproc = Dm->kproc();
 	int nprocx = Dm->nprocx();
 	int nprocy = Dm->nprocy();
 	int nprocz = Dm->nprocz();
@@ -122,7 +119,6 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	int sendtag,recvtag;
 	sendtag = recvtag = 7;
 
-	int x,y,z;
 	int ii,jj,kk;
 	int Nx = nx;
 	int Ny = ny;
@@ -135,7 +131,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 
 	// Increase the critical radius until the target saturation is met
 	double deltaR=0.05; // amount to change the radius in voxel units
-	double Rcrit_old;
+	double Rcrit_old=0.0;
 
 	double GlobalNumber = 1.f;
 	int imin,jmin,kmin,imax,jmax,kmax;
@@ -336,9 +332,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	int nx = Dm->Nx;
 	int ny = Dm->Ny;
 	int nz = Dm->Nz;
-	int iproc = Dm->iproc();
-	int jproc = Dm->jproc();
-	int kproc = Dm->kproc();
 	int nprocx = Dm->nprocx();
 	int nprocy = Dm->nprocy();
 	int nprocz = Dm->nprocz();
@@ -427,7 +420,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	int sendtag,recvtag;
 	sendtag = recvtag = 7;
 
-	int x,y,z;
 	int ii,jj,kk;
 	int Nx = nx;
 	int Ny = ny;
@@ -693,19 +685,13 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	return final_void_fraction;
 }
 
-double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth){
-	
+double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth, double WallFactor)
+{
 	int Nx = Dm->Nx;
 	int Ny = Dm->Ny;
 	int Nz = Dm->Nz;
-	int iproc = Dm->iproc();
-	int jproc = Dm->jproc();
-	int kproc = Dm->kproc();
-	int nprocx = Dm->nprocx();
-	int nprocy = Dm->nprocy();
-	int nprocz = Dm->nprocz();
 	int rank = Dm->rank();
-	
+		
 	double count=0.0;
 	for (int k=1; k<Nz-1; k++){
 		for (int j=1; j<Ny-1; j++){
@@ -736,8 +722,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
 			for (int j=1; j<Ny-1; j++){
 				for (int i=1; i<Nx-1; i++){
 					double walldist=BoundaryDist(i,j,k);
-					double wallweight = 1.0 / (1+exp(-5.f*(walldist-1.f))); 
-					//wallweight = 1.0;
+					double wallweight = WallFactor/ (1+exp(-5.f*(walldist-1.f))); 
 					if (fabs(wallweight*morph_delta) > MAX_DISPLACEMENT) MAX_DISPLACEMENT= fabs(wallweight*morph_delta);
 					
 					if (Dist(i,j,k) - wallweight*morph_delta < 0.0){
@@ -783,7 +768,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
 		for (int j=1; j<Ny-1; j++){
 			for (int i=1; i<Nx-1; i++){
 				double walldist=BoundaryDist(i,j,k);
-				double wallweight = 1.0 / (1+exp(-5.f*(walldist-1.f))); 
+				double wallweight = WallFactor / (1+exp(-5.f*(walldist-1.f))); 
 				//wallweight = 1.0;
 				Dist(i,j,k) -= wallweight*morph_delta;
 				if (Dist(i,j,k) < 0.0)	count+=1.0;

analysis/morphology.h

@@ -5,4 +5,4 @@
 
 double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction, signed char ErodeLabel, signed char ReplaceLabel);
 double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction);
-double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetVol);
+double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetVol, double WallFactor);

analysis/pmmc.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/runAnalysis.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -782,6 +783,8 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
         double *Pressure, double *Velocity, double *fq, double *Den)
 {
     int N = d_N[0]*d_N[1]*d_N[2];
+    NULL_USE( N );
+    NULL_USE( Phi );
     
 	auto db = input_db->getDatabase( "Analysis" );
     //int timestep = db->getWithDefault<int>( "timestep", 0 );
@@ -920,12 +923,12 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
     // Spawn a thread to write the restart file
     //    if ( matches(type,AnalysisType::CreateRestart) ) {
     if (timestep%d_restart_interval==0){
-
+    	auto Restart_db = input_db->cloneDatabase();
+	//	Restart_db->putScalar<bool>( "Restart", true );
     	if (d_rank==0) {
-    		input_db->putScalar<bool>( "Restart", true );
-    		std::ofstream OutStream("Restart.db");
-    		input_db->print(OutStream, "");
-    		OutStream.close();
+	  //	std::ofstream OutStream("Restart.db");
+	  //	Restart_db->print(OutStream, "");
+	  //	OutStream.close();
     	}
     	// Write the restart file (using a seperate thread)
         auto work = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np);
@@ -952,8 +955,6 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
  ******************************************************************/
 void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
 {
-    int N = d_N[0]*d_N[1]*d_N[2];
-
     // Check which analysis steps we need to perform
 	auto color_db =  input_db->getDatabase( "Color" );
 	auto vis_db =  input_db->getDatabase( "Visualization" );
@@ -969,7 +970,7 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
         finish();
     }
 
-    PROFILE_START("run");
+    PROFILE_START("basic");
 
     // Copy the appropriate variables to the host (so we can spawn new threads)
     ScaLBL_DeviceBarrier();
@@ -998,7 +999,6 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
     }
     PROFILE_STOP("Copy data to host");
 
-    PROFILE_START("run",1);
     // Spawn threads to do the analysis work
     //if (timestep%d_restart_interval==0){
     // if ( matches(type,AnalysisType::ComputeAverages) ) {
@@ -1025,21 +1025,21 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
     	cfq = std::shared_ptr<double>(new double[19*d_Np],DeleteArray<double>);
     	ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double));
     	ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double));
-
+    	// clone the input database to avoid modifying shared data
+    	auto Restart_db = input_db->cloneDatabase();
+    	auto tmp_color_db =  Restart_db->getDatabase( "Color" );
+    	tmp_color_db->putScalar<int>("timestep",timestep);    		
+    	tmp_color_db->putScalar<bool>( "Restart", true );
+    	Restart_db->putDatabase("Color", tmp_color_db);
     	if (d_rank==0) {
-    		color_db->putScalar<int>("timestep",timestep);    		
-    		color_db->putScalar<bool>( "Restart", true );
-    		input_db->putDatabase("Color", color_db);
     		std::ofstream OutStream("Restart.db");
-    		input_db->print(OutStream, "");
+    		Restart_db->print(OutStream, "");
     		OutStream.close();
-  
     	}
     	// Write the restart file (using a seperate thread)
     	auto work1 = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np);
     	work1->add_dependency(d_wait_restart);
     	d_wait_restart = d_tpool.add_work(work1);
-
     }
     
     if (timestep%d_visualization_interval==0){
@@ -1051,12 +1051,11 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
         d_wait_vis = d_tpool.add_work(work);
     }
 
-    PROFILE_STOP("run");
+    PROFILE_STOP("basic");
 }
 
 void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
 {
-    int N = d_N[0]*d_N[1]*d_N[2];
 	auto color_db =  input_db->getDatabase( "Color" );
 	auto vis_db =  input_db->getDatabase( "Visualization" );
     //int timestep = color_db->getWithDefault<int>( "timestep", 0 );
@@ -1083,7 +1082,6 @@ void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db,
     d_wait_vis = d_tpool.add_work(work2);
 
     //Averages.WriteVis = false;
-   // }
     
     PROFILE_STOP("write vis");
 }

analysis/runAnalysis.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

analysis/uCT.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -172,6 +173,7 @@ void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
     //	int depth = 5;
     //	float sigsq=0.1;
 	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+    NULL_USE( nlm_count );
 	fillFloat.fill(NonLocalMean);
 }
 
@@ -216,6 +218,7 @@ void refine( const Array<float>& Dist_coarse,
 //	int depth = 3;
 //	float sigsq = 0.1;
 	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+    NULL_USE( nlm_count );
 	fillFloat.fill(NonLocalMean);
     segment( NonLocalMean, ID, 0.001 );
     for (size_t i=0; i<ID.length(); i++) {

analysis/uCT.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cmake/libraries.cmake

@@ -77,7 +77,7 @@ MACRO( CONFIGURE_MPI )
                 ENDIF ()
             ELSE ()
                 # Search for the MPI executable in the current directory
-                FIND_PROGRAM ( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
+                FIND_PROGRAM( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
                 IF ( NOT MPIEXEC )
                     MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
                 ENDIF()
@@ -305,4 +305,9 @@ MACRO ( CONFIGURE_LBPM )
         SET( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )
         SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}" "${LBPM_INSTALL_DIR}/lib" )
     ENDIF()
+    # Suppress some common warnings
+    IF ( USING_GCC )
+        SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder -Wno-unused-parameter")
+        SET( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --compiler-options -Wno-reorder,-Wno-unused-parameter")
+    ENDIF()
 ENDMACRO ()

cmake/macros.cmake

@@ -848,7 +848,7 @@ FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
     ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
     CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
     IF ( USE_MPI_FOR_SERIAL_TESTS )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
     ELSE()
         ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@@ -877,7 +877,7 @@ FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
     ELSEIF( ${PROCS} STREQUAL "1" )
         CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
         IF ( USE_MPI_FOR_SERIAL_TESTS )
-            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
             SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         ELSE()
             ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@@ -886,7 +886,7 @@ FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
         CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -909,7 +909,7 @@ FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
     ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
         MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
     ELSE()
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -930,7 +930,7 @@ MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
         SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ELSEIF ( USE_MPI OR USE_EXT_MPI )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -966,7 +966,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
         ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
     ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
         CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
     ENDIF()
     SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )

common/Array.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Array.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -724,15 +726,15 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::repmat(
     for ( size_t i4 = 0, index = 0; i4 < N1[4]; i4++ ) {
         for ( size_t j4 = 0; j4 < Nr[4]; j4++ ) {
             for ( size_t i3 = 0; i3 < N1[3]; i3++ ) {
-                for ( size_t j4 = 0; j4 < Nr[3]; j4++ ) {
+                for ( size_t j3 = 0; j3 < Nr[3]; j3++ ) {
                     for ( size_t i2 = 0; i2 < N1[2]; i2++ ) {
-                        for ( size_t j4 = 0; j4 < Nr[2]; j4++ ) {
+                        for ( size_t j2 = 0; j2 < Nr[2]; j2++ ) {
                             for ( size_t i1 = 0; i1 < N1[1]; i1++ ) {
-                                for ( size_t j4 = 0; j4 < Nr[1]; j4++ ) {
+                                for ( size_t j1 = 0; j1 < Nr[1]; j1++ ) {
                                     for ( size_t i0 = 0; i0 < N1[0]; i0++ ) {
                                         size_t k = d_size.index( i0, i1, i2, i3, i4 );
                                         TYPE x   = d_data[k];
-                                        for ( size_t j4 = 0; j4 < Nr[0]; j4++, index++ )
+                                        for ( size_t j0 = 0; j0 < Nr[0]; j0++, index++ )
                                             y2[index] = x;
                                     }
                                 }

common/Communication.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -19,16 +20,28 @@
 /********************************************************
 *  Structure to store the rank info                     *
 ********************************************************/
-inline int getRankForBlock( int nprocx, int nprocy, int nprocz, int i, int j, int k )
+int RankInfoStruct::getRankForBlock( int i, int j, int k ) const
 {
-	int i2 = (i+nprocx)%nprocx;
-	int j2 = (j+nprocy)%nprocy;
-	int k2 = (k+nprocz)%nprocz;
-	return i2 + j2*nprocx + k2*nprocx*nprocy;
+	int i2 = (i+nx)%nx;
+	int j2 = (j+ny)%ny;
+	int k2 = (k+nz)%nz;
+	return i2 + j2*nx + k2*nx*ny;
 }
 RankInfoStruct::RankInfoStruct()
 {
-    memset(this,0,sizeof(RankInfoStruct));
+    nx = 0;
+    ny = 0;
+    nz = 0;
+    ix = -1;
+    jy = -1;
+    kz = -1;
+    for (int i=-1; i<=1; i++) {
+        for (int j=-1; j<=1; j++) {
+            for (int k=-1; k<=1; k++) {
+                rank[i+1][j+1][k+1] = -1;
+            }
+        }
+    }
 }
 RankInfoStruct::RankInfoStruct( int rank0, int nprocx, int nprocy, int nprocz )
 {
@@ -36,17 +49,30 @@ RankInfoStruct::RankInfoStruct( int rank0, int nprocx, int nprocy, int nprocz )
     nx = nprocx;
     ny = nprocy;
     nz = nprocz;
-	ix = rank0%nprocx;
-	jy = (rank0/nprocx)%nprocy;
-	kz = rank0/(nprocx*nprocy);
-    for (int i=-1; i<=1; i++) {
-        for (int j=-1; j<=1; j++) {
-            for (int k=-1; k<=1; k++) {
-                rank[i+1][j+1][k+1] = getRankForBlock(nprocx,nprocy,nprocz,ix+i,jy+j,kz+k);
+    if ( rank0 >= nprocx * nprocy * nprocz ) {
+	    ix = -1;
+	    jy = -1;
+	    kz = -1;
+        for (int i=-1; i<=1; i++) {
+            for (int j=-1; j<=1; j++) {
+                for (int k=-1; k<=1; k++) {
+                    rank[i+1][j+1][k+1] = -1;
+                }
             }
         }
+    } else {
+	    ix = rank0%nprocx;
+	    jy = (rank0/nprocx)%nprocy;
+	    kz = rank0/(nprocx*nprocy);
+        for (int i=-1; i<=1; i++) {
+            for (int j=-1; j<=1; j++) {
+                for (int k=-1; k<=1; k++) {
+                    rank[i+1][j+1][k+1] = getRankForBlock(ix+i,jy+j,kz+k);
+                }
+            }
+        }
+        ASSERT(rank[1][1][1]==rank0);
     }
-    ASSERT(rank[1][1][1]==rank0);
 }
 
 

common/Communication.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -46,9 +47,16 @@ struct RankInfoStruct {
     int rank[3][3][3];      //!<  The rank for the neighbor [i][j][k]
     RankInfoStruct();
     RankInfoStruct( int rank, int nprocx, int nprocy, int nprocz );
+    int getRankForBlock( int i, int j, int k ) const;
 };
 
 
+//! Redistribute domain data (dst may be smaller than the src)
+template<class TYPE>
+Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
+    const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm );
+
+
 /*!
  * @brief  Communicate halo
  * @details  Fill the halo cells in an array from the neighboring processes

common/Communication.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -38,7 +40,90 @@
 
 
 /********************************************************
-*  Structure to store the rank info                     *
+*  Redistribute data between two grids                  *
+********************************************************/
+template<class TYPE>
+Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
+    const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm )
+{
+#ifdef USE_MPI
+    // Get the src size
+    std::array<int,3> src_size;
+    int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) };
+    MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm );
+    if ( !src_data.empty() )
+        ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] );
+    // Check that dst_size matches on all ranks
+    MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm );
+    ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] );
+    // Function to get overlap range
+    auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) {
+        std::vector<size_t> index;
+        if ( i1[0] > j2[0] || i2[0] < j1[0] || i1[1] > j2[1] || i2[1] < j1[1] || i1[2] > j2[2] || i2[2] < j1[2] )
+            return index;
+        index.resize( 6 );
+        index[0] = std::max( j1[0] - i1[0], 0 );
+        index[1] = std::min( j2[0] - i1[0], i2[0] - i1[0] );
+        index[2] = std::max( j1[1] - i1[1], 0 );
+        index[3] = std::min( j2[1] - i1[1], i2[1] - i1[1] );
+        index[4] = std::max( j1[2] - i1[2], 0 );
+        index[5] = std::min( j2[2] - i1[2], i2[2] - i1[2] );
+        return index;
+    };
+    // Pack and send my data to the appropriate ranks (including myself)
+    std::vector<int> send_rank;
+    std::vector<Array<TYPE>> send_data;
+    if ( !src_data.empty() ) {
+        int i1[3] = { src_size[0] * src_rank.ix, src_size[1] * src_rank.jy, src_size[2] * src_rank.kz };
+        int i2[3] = { i1[0] + src_size[0] - 1, i1[1] + src_size[1] - 1, i1[2] + src_size[2] - 1 };
+        for ( int i=0; i<dst_rank.nx; i++ ) {
+            for ( int j=0; j<dst_rank.ny; j++ ) {
+                for ( int k=0; k<dst_rank.nz; k++ ) {
+                    int j1[3] = { i * dst_size[0], j * dst_size[1], k * dst_size[2] };
+                    int j2[3] = { j1[0] + dst_size[0] - 1, j1[1] + dst_size[1] - 1, j1[2] + dst_size[2] - 1 };
+                    auto index = calcOverlap( i1, i2, j1, j2 );
+                    if ( index.empty() )
+                        continue;
+                    send_rank.push_back( dst_rank.getRankForBlock(i,j,k) );
+                    send_data.push_back( src_data.subset( index ) );
+                }
+            }
+        }
+    }
+    std::vector<MPI_Request> send_request( send_rank.size() );
+    for (size_t i=0; i<send_rank.size(); i++)
+        MPI_Isend( send_data[i].data(), sizeof(TYPE)*send_data[i].length(), MPI_BYTE, send_rank[i], 5462, comm, &send_request[i]);
+    // Unpack data from the appropriate ranks (including myself)
+    Array<TYPE> dst_data( dst_size[0], dst_size[1], dst_size[2] );
+    int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz };
+    int i2[3] = { i1[0] + dst_size[0] - 1, i1[1] + dst_size[1] - 1, i1[2] + dst_size[2] - 1 };
+    for ( int i=0; i<src_rank.nx; i++ ) {
+        for ( int j=0; j<src_rank.ny; j++ ) {
+            for ( int k=0; k<src_rank.nz; k++ ) {
+                int j1[3] = { i * src_size[0], j * src_size[1], k * src_size[2] };
+                int j2[3] = { j1[0] + src_size[0] - 1, j1[1] + src_size[1] - 1, j1[2] + src_size[2] - 1 };
+                auto index = calcOverlap( i1, i2, j1, j2 );
+                if ( index.empty() )
+                    continue;
+                int rank  = src_rank.getRankForBlock(i,j,k);
+                Array<TYPE> data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 );
+                MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE );
+                dst_data.copySubset( index, data );
+            }
+        }
+    }
+    // Free data
+    MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE );
+    return dst_data;
+#else
+    return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 );
+#endif
+}
+
+
+
+/********************************************************
+*  Structure to fill halo cells                         *
 ********************************************************/
 template<class TYPE>
 fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,

common/Database.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Database.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Database.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Domain.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -73,6 +74,9 @@ Domain::Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
 	recvData_xY(NULL), recvData_yZ(NULL), recvData_Xz(NULL), recvData_XY(NULL), recvData_YZ(NULL), recvData_XZ(NULL),
 	id(NULL)
 {	
+    NULL_USE( rnk );
+    NULL_USE( npy );
+    NULL_USE( npz );
 	// set up the neighbor ranks
     int myrank;
     MPI_Comm_rank( Comm, &myrank );
@@ -177,6 +181,7 @@ Domain::~Domain()
 	delete [] recvData_yZ;  delete [] recvData_Yz;  delete [] recvData_YZ;
 	// Free id
 	delete [] id;
+ 
 	// Free the communicator
 	if ( Comm != MPI_COMM_WORLD && Comm != MPI_COMM_NULL ) {
 		MPI_Comm_free(&Comm);
@@ -244,7 +249,7 @@ void Domain::initialize( std::shared_ptr<Database> db )
 	if (rank_info.kz < nproc[2]-1) outlet_layers_z = 0;
     // Fill remaining variables
 	N = Nx*Ny*Nz;
-	Volume = nx*ny*nx*nproc[0]*nproc[1]*nproc[2]*1.0;
+	Volume = nx*ny*nz*nproc[0]*nproc[1]*nproc[2]*1.0;
 
 	if (myrank==0) printf("voxel length = %f micron \n", voxel_length);
 
@@ -256,7 +261,7 @@ void Domain::initialize( std::shared_ptr<Database> db )
 	INSIST(nprocs == nproc[0]*nproc[1]*nproc[2],"Fatal error in processor count!");
 }
 
-void Domain::Decomp(std::string Filename)
+void Domain::Decomp( const std::string& Filename )
 {
 	//.......................................................................
 	// Reading the domain information file
@@ -266,9 +271,9 @@ void Domain::Decomp(std::string Filename)
 	int nprocs, nprocx, nprocy, nprocz, nx, ny, nz;
 	int64_t global_Nx,global_Ny,global_Nz;
 	int64_t i,j,k,n;
-	int BC=0;
 	int64_t xStart,yStart,zStart;
 	int checkerSize;
+	bool USE_CHECKER = false;
 	//int inlet_layers_x, inlet_layers_y, inlet_layers_z;
 	//int outlet_layers_x, outlet_layers_y, outlet_layers_z;
 	xStart=yStart=zStart=0;
@@ -278,8 +283,8 @@ void Domain::Decomp(std::string Filename)
 	outlet_layers_x = 0;
 	outlet_layers_y = 0;
 	outlet_layers_z = 0;
-    inlet_layers_phase=1;
-    outlet_layers_phase=2;
+	inlet_layers_phase=1;
+	outlet_layers_phase=2;
 	checkerSize = 32;
 
 	// Read domain parameters
@@ -308,6 +313,7 @@ void Domain::Decomp(std::string Filename)
 	}
 	if (database->keyExists( "checkerSize" )){
 		checkerSize = database->getScalar<int>( "checkerSize" );
+		USE_CHECKER = true;
 	}
 	else {
 		checkerSize = SIZE[0];
@@ -344,10 +350,9 @@ void Domain::Decomp(std::string Filename)
 	char *SegData = NULL;
 	
 	if (RANK==0){
-
 		printf("Input media: %s\n",Filename.c_str());
 		printf("Relabeling %lu values\n",ReadValues.size());
-		for (int idx=0; idx<ReadValues.size(); idx++){
+		for (size_t idx=0; idx<ReadValues.size(); idx++){
 			int oldvalue=ReadValues[idx];
 			int newvalue=WriteValues[idx];
 			printf("oldvalue=%d, newvalue =%d \n",oldvalue,newvalue);
@@ -381,7 +386,7 @@ void Domain::Decomp(std::string Filename)
 			}
 		}
 		printf("Read segmented data from %s \n",Filename.c_str());
-		
+
 		// relabel the data
 		std::vector<long int> LabelCount(ReadValues.size(),0);
 		for (int k = 0; k<global_Nz; k++){
@@ -390,7 +395,7 @@ void Domain::Decomp(std::string Filename)
 					n = k*global_Nx*global_Ny+j*global_Nx+i;
 					//char locval = loc_id[n];
 					char locval = SegData[n];
-					for (int idx=0; idx<ReadValues.size(); idx++){
+					for (size_t idx=0; idx<ReadValues.size(); idx++){
 						signed char oldvalue=ReadValues[idx];
 						signed char newvalue=WriteValues[idx];
 						if (locval == oldvalue){
@@ -402,124 +407,154 @@ void Domain::Decomp(std::string Filename)
 				}
 			}
 		}
-		if (RANK==0){
-			for (int idx=0; idx<ReadValues.size(); idx++){
-				long int label=ReadValues[idx];
-				long int count=LabelCount[idx];
-				printf("Label=%d, Count=%d \n",label,count);
-			}
+		for (size_t idx=0; idx<ReadValues.size(); idx++){
+			long int label=ReadValues[idx];
+			long int count=LabelCount[idx];
+			printf("Label=%ld, Count=%ld \n",label,count);
 		}
-
-		if (inlet_layers_x > 0){
-			// use checkerboard pattern
-			printf("Checkerboard pattern at x inlet for %i layers \n",inlet_layers_x);
-			for (int k = 0; k<global_Nz; k++){
-				for (int j = 0; j<global_Ny; j++){
-					for (int i = xStart; i < xStart+inlet_layers_x; i++){
-						if ( (j/checkerSize + k/checkerSize)%2 == 0){
-							// void checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+		if (USE_CHECKER) {
+			if (inlet_layers_x > 0){
+				// use checkerboard pattern
+				printf("Checkerboard pattern at x inlet for %i layers \n",inlet_layers_x);
+				for (int k = 0; k<global_Nz; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = xStart; i < xStart+inlet_layers_x; i++){
+							if ( (j/checkerSize + k/checkerSize)%2 == 0){
+								// void checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+							}
 						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+					}
+				}
+			}
+
+			if (inlet_layers_y > 0){
+				printf("Checkerboard pattern at y inlet for %i layers \n",inlet_layers_y);
+				// use checkerboard pattern
+				for (int k = 0; k<global_Nz; k++){
+					for (int j = yStart; j < yStart+inlet_layers_y; j++){
+						for (int i = 0; i<global_Nx; i++){
+							if ( (i/checkerSize + k/checkerSize)%2 == 0){
+								// void checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+							}
+						}
+					}
+				}
+			}
+
+			if (inlet_layers_z > 0){
+				printf("Checkerboard pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase);
+				// use checkerboard pattern
+				for (int k = zStart; k < zStart+inlet_layers_z; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = 0; i<global_Nx; i++){
+							if ( (i/checkerSize+j/checkerSize)%2 == 0){
+								// void checkers
+								//SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = inlet_layers_phase;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+							}
+						}
+					}
+				}
+			}
+
+			if (outlet_layers_x > 0){
+				// use checkerboard pattern
+				printf("Checkerboard pattern at x outlet for %i layers \n",outlet_layers_x);
+				for (int k = 0; k<global_Nz; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = xStart + nx*nprocx - outlet_layers_x; i <  xStart + nx*nprocx; i++){
+							if ( (j/checkerSize + k/checkerSize)%2 == 0){
+								// void checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+							}
+						}
+					}
+				}
+			}
+
+			if (outlet_layers_y > 0){
+				printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y);
+				// use checkerboard pattern
+				for (int k = 0; k<global_Nz; k++){
+					for (int j = yStart + ny*nprocy - outlet_layers_y; j < yStart + ny*nprocy; j++){
+						for (int i = 0; i<global_Nx; i++){
+							if ( (i/checkerSize + k/checkerSize)%2 == 0){
+								// void checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+
+							}
+						}
+					}
+				}
+			}
+
+			if (outlet_layers_z > 0){
+				printf("Checkerboard pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase);
+				// use checkerboard pattern
+				for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = 0; i<global_Nx; i++){
+							if ( (i/checkerSize+j/checkerSize)%2 == 0){
+								// void checkers
+								//SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = outlet_layers_phase;
+							}
+							else{
+								// solid checkers
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+							}
 						}
 					}
 				}
 			}
 		}
-
-		if (inlet_layers_y > 0){
-			printf("Checkerboard pattern at y inlet for %i layers \n",inlet_layers_y);
-			// use checkerboard pattern
-			for (int k = 0; k<global_Nz; k++){
-				for (int j = yStart; i < yStart+inlet_layers_y; j++){
-					for (int i = 0; i<global_Nx; i++){
-						if ( (i/checkerSize + k/checkerSize)%2 == 0){
-							// void checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
-						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+		else {
+			if (inlet_layers_z > 0){
+				printf("Mixed reflection pattern at z inlet for %i layers, saturated with phase label=%i \n",inlet_layers_z,inlet_layers_phase);
+				for (int k = zStart; k < zStart+inlet_layers_z; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = 0; i<global_Nx; i++){
+							signed char local_id = SegData[k*global_Nx*global_Ny+j*global_Nx+i];
+							signed char reflection_id = SegData[(zStart + nz*nprocz - 1)*global_Nx*global_Ny+j*global_Nx+i];
+							if ( local_id < 1 && reflection_id > 0){
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id;
+							}
 						}
 					}
 				}
 			}
-		}
-
-		if (inlet_layers_z > 0){
-			printf("Checkerboard pattern at z inlet for %i layers \n",inlet_layers_z);
-			// use checkerboard pattern
-			for (int k = zStart; k < zStart+inlet_layers_z; k++){
-				for (int j = 0; j<global_Ny; j++){
-					for (int i = 0; i<global_Nx; i++){
-						if ( (i/checkerSize+j/checkerSize)%2 == 0){
-							// void checkers
-							//SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = inlet_layers_phase;
-						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
-						}
-					}
-				}
-			}
-		}
-
-		if (outlet_layers_x > 0){
-			// use checkerboard pattern
-			printf("Checkerboard pattern at x outlet for %i layers \n",outlet_layers_x);
-			for (int k = 0; k<global_Nz; k++){
-				for (int j = 0; j<global_Ny; j++){
-					for (int i = xStart + nx*nprocx - outlet_layers_x; i <  xStart + nx*nprocx; i++){
-						if ( (j/checkerSize + k/checkerSize)%2 == 0){
-							// void checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
-						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
-						}
-					}
-				}
-			}
-		}
-
-		if (outlet_layers_y > 0){
-			printf("Checkerboard pattern at y outlet for %i layers \n",outlet_layers_y);
-			// use checkerboard pattern
-			for (int k = 0; k<global_Nz; k++){
-				for (int j = yStart + ny*nprocy - outlet_layers_y; i < yStart + ny*nprocy; j++){
-					for (int i = 0; i<global_Nx; i++){
-						if ( (i/checkerSize + k/checkerSize)%2 == 0){
-							// void checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
-						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
-						}
-					}
-				}
-			}
-		}
-
-		if (outlet_layers_z > 0){
-			printf("Checkerboard pattern at z outlet for %i layers \n",outlet_layers_z);
-			// use checkerboard pattern
-			for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){
-				for (int j = 0; j<global_Ny; j++){
-					for (int i = 0; i<global_Nx; i++){
-						if ( (i/checkerSize+j/checkerSize)%2 == 0){
-							// void checkers
-							//SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 2;
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = outlet_layers_phase;
-						}
-						else{
-							// solid checkers
-							SegData[k*global_Nx*global_Ny+j*global_Nx+i] = 0;
+			if (outlet_layers_z > 0){
+				printf("Mixed reflection pattern at z outlet for %i layers, saturated with phase label=%i \n",outlet_layers_z,outlet_layers_phase);
+				for (int k = zStart + nz*nprocz - outlet_layers_z; k < zStart + nz*nprocz; k++){
+					for (int j = 0; j<global_Ny; j++){
+						for (int i = 0; i<global_Nx; i++){
+							signed char local_id = SegData[k*global_Nx*global_Ny+j*global_Nx+i];
+							signed char reflection_id = SegData[zStart*global_Nx*global_Ny+j*global_Nx+i];
+							if ( local_id < 1 && reflection_id > 0){
+								SegData[k*global_Nx*global_Ny+j*global_Nx+i] = reflection_id;
+							}
 						}
 					}
 				}
@@ -599,10 +634,66 @@ void Domain::Decomp(std::string Filename)
 		//printf("Ready to recieve data %i at process %i \n", N,rank);
 		MPI_Recv(id,N,MPI_CHAR,0,15,Comm,MPI_STATUS_IGNORE);
 	}
+	//Comm.barrier();
 	MPI_Barrier(Comm);
+	// Compute the porosity
+	double sum;
+	double sum_local=0.0;
+	double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+	if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx*(Ny-2)*nprocy*((Nz-2)*nprocz-6));
+	//.........................................................
+	// If external boundary conditions are applied remove solid
+	if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == 0){
+    	if (inlet_layers_z < 4){
+            inlet_layers_z=4;
+            if(RANK==0){
+                printf("NOTE:Non-periodic BC is applied, but the number of Z-inlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-inlet layer is reset to %i voxels, saturated with phase label=%i \n",inlet_layers_z-1,inlet_layers_phase);
+            } 
+        }	
+		for (int k=0; k<inlet_layers_z; k++){
+			for (int j=0;j<Ny;j++){
+				for (int i=0;i<Nx;i++){
+					int n = k*Nx*Ny+j*Nx+i;
+					id[n] = inlet_layers_phase;
+				}                    
+			}
+ 		}
+ 	}
+    if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == nprocz-1){
+    	if (outlet_layers_z < 4){
+            outlet_layers_z=4;
+            if(RANK==nprocs-1){
+                printf("NOTE:Non-periodic BC is applied, but the number of Z-outlet layers is not specified (or is smaller than 3 voxels) \n     the number of Z-outlet layer is reset to %i voxels, saturated with phase label=%i \n",outlet_layers_z-1,outlet_layers_phase);
+            } 
+        }	
+ 		for (int k=Nz-outlet_layers_z; k<Nz; k++){
+ 			for (int j=0;j<Ny;j++){
+ 				for (int i=0;i<Nx;i++){
+ 					int n = k*Nx*Ny+j*Nx+i;
+ 					id[n] = outlet_layers_phase;
+ 				}                    
+ 			}
+ 		}
+ 	}
+    for (int k=inlet_layers_z+1; k<Nz-outlet_layers_z-1;k++){
+        for (int j=1;j<Ny-1;j++){
+            for (int i=1;i<Nx-1;i++){
+                int n = k*Nx*Ny+j*Nx+i;
+                if (id[n] > 0){
+                    sum_local+=1.0;
+                }
+            }
+        }
+    }
+    MPI_Allreduce(&sum_local,&sum,1,MPI_DOUBLE,MPI_SUM,Comm);
+    //sum = Comm.sumReduce(sum_local);
+    porosity = sum*iVol_global;
+    if (rank()==0) printf("Media porosity = %f \n",porosity);
+ 	//.........................................................
 }
 
-void Domain::AggregateLabels(char *FILENAME){
+
+void Domain::AggregateLabels( const std::string& filename ){
 	
 	int nx = Nx;
 	int ny = Ny;
@@ -622,7 +713,7 @@ void Domain::AggregateLabels(char *FILENAME){
 	int full_ny = npy*(ny-2);
 	int full_nz = npz*(nz-2);
 	int local_size = (nx-2)*(ny-2)*(nz-2);
-	long int full_size = long(full_nx)*long(full_ny)*long(full_nz);
+	unsigned long int full_size = long(full_nx)*long(full_ny)*long(full_nz);
 	
 	signed char *LocalID;
 	LocalID = new signed char [local_size];
@@ -652,7 +743,7 @@ void Domain::AggregateLabels(char *FILENAME){
 					int y = j-1;
 					int z = k-1;
 					int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1;
-					int n_full = z*full_nx*full_ny + y*full_nx + x;
+					unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x;
 					FullID[n_full] = LocalID[n_local];
 				}
 			}
@@ -672,15 +763,14 @@ void Domain::AggregateLabels(char *FILENAME){
 						int y = j-1 + ipy*(ny-2);
 						int z = k-1 + ipz*(nz-2);
 						int n_local = (k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1;
-						int n_full = z*full_nx*full_ny + y*full_nx + x;
+						unsigned long int n_full = z*long(full_nx)*long(full_ny) + y*long(full_nx) + x;
 						FullID[n_full] = LocalID[n_local];
 					}
 				}
 			}
 		}
 		// write the output
-		FILE *OUTFILE;
-		OUTFILE = fopen(FILENAME,"wb");
+		FILE *OUTFILE = fopen(filename.c_str(),"wb");
 		fwrite(FullID,1,full_size,OUTFILE);
 		fclose(OUTFILE);
 	}
@@ -1020,10 +1110,10 @@ void Domain::ReadIDs(){
 	double sum;
 	double sum_local=0.0;
 	double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
-	if (BoundaryCondition > 0) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6));
+	if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6));
 	//.........................................................
 	// If external boundary conditions are applied remove solid
-	if (BoundaryCondition >  0  && kproc() == 0){
+	if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == 0){
     	if (inlet_layers_z < 4)	inlet_layers_z=4;
 		for (int k=0; k<inlet_layers_z; k++){
 			for (int j=0;j<Ny;j++){
@@ -1034,7 +1124,7 @@ void Domain::ReadIDs(){
 			}
  		}
  	}
-    if (BoundaryCondition >  0  && kproc() == nprocz()-1){
+    if (BoundaryCondition >  0 && BoundaryCondition !=5 && kproc() == nprocz()-1){
     	if (outlet_layers_z < 4)	outlet_layers_z=4;
  		for (int k=Nz-outlet_layers_z; k<Nz; k++){
  			for (int j=0;j<Ny;j++){
@@ -1159,20 +1249,19 @@ void Domain::CommunicateMeshHalo(DoubleArray &Mesh)
 	UnpackMeshData(recvList_YZ, recvCount_YZ ,recvData_YZ, MeshData);
 }
 
-// Ideally stuff below here should be moved somewhere else -- doesn't really belong here
-void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np)
+// TODO Ideally stuff below here should be moved somewhere else -- doesn't really belong here
+void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np)
 {
-    int q,n;
     double value;
     ofstream File(FILENAME,ios::binary);
-    for (n=0; n<Np; n++){
+    for (size_t n=0; n<Np; n++){
         // Write the two density values
         value = cDen[n];
         File.write((char*) &value, sizeof(value));
         value = cDen[Np+n];
         File.write((char*) &value, sizeof(value));
         // Write the even distributions
-        for (q=0; q<19; q++){
+        for (size_t q=0; q<19; q++){
             value = cfq[q*Np+n];
             File.write((char*) &value, sizeof(value));
         }
@@ -1181,16 +1270,15 @@ void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq
 
 }
 
-void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np)
+void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, size_t Np)
 {
-    int q=0, n=0;
     double value=0;
     ifstream File(FILENAME,ios::binary);
-    for (n=0; n<Np; n++){
+    for (size_t n=0; n<Np; n++){
         File.read((char*) &value, sizeof(value));
         cPhi[n] = value;
         // Read the distributions
-        for (q=0; q<19; q++){
+        for (size_t q=0; q<19; q++){
             File.read((char*) &value, sizeof(value));
             cfq[q*Np+n] = value;
         }
@@ -1198,13 +1286,12 @@ void ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq, int Np)
     File.close();
 }
 
-void ReadBinaryFile(char *FILENAME, double *Data, int N)
+void ReadBinaryFile(char *FILENAME, double *Data, size_t N)
 {
-  int n;
   double value;
   ifstream File(FILENAME,ios::binary);
   if (File.good()){
-    for (n=0; n<N; n++){
+    for (size_t n=0; n<N; n++){
       // Write the two density values                                                                                
       File.read((char*) &value, sizeof(value));
       Data[n] = value;
@@ -1212,8 +1299,156 @@ void ReadBinaryFile(char *FILENAME, double *Data, int N)
     }
   }
   else {
-    for (n=0; n<N; n++) Data[n] = 1.2e-34;
+    for (size_t n=0; n<N; n++) Data[n] = 1.2e-34;
   }
   File.close();
 }
 
+
+void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData)
+{
+	//........................................................................................
+	// Reading the user-defined input file
+    // NOTE: so far it only supports BC=0 (periodic) and BC=5 (mixed reflection)
+    //       because if checkerboard or inlet/outlet buffer layers are added, the
+    //       value of the void space is undefined. 
+    // NOTE: if BC=5 is used, where the inlet and outlet layers of the domain are modified, 
+    //       user needs to modify the input file accordingly before LBPM simulator read
+    //       the input file.
+	//........................................................................................
+	int rank_offset = 0;
+	int RANK = rank();
+	int nprocs, nprocx, nprocy, nprocz, nx, ny, nz;
+	int64_t global_Nx,global_Ny,global_Nz;
+	int64_t i,j,k,n;
+    //TODO These offset we may still need them
+	int64_t xStart,yStart,zStart;
+	xStart=yStart=zStart=0;
+
+	// Read domain parameters
+    // TODO currently the size of the data is still read from Domain{}; 
+    //      but user may have a user-specified size
+	auto size = database->getVector<int>( "n" );
+	auto SIZE = database->getVector<int>( "N" );
+	auto nproc = database->getVector<int>( "nproc" );
+    //TODO currently the funcationality "offset" is disabled as the user-defined input data may have a different size from that of the input domain 
+	if (database->keyExists( "offset" )){
+		auto offset = database->getVector<int>( "offset" );
+		xStart = offset[0];
+		yStart = offset[1];
+		zStart = offset[2];
+	}
+	
+	nx = size[0];
+	ny = size[1];
+	nz = size[2];
+	nprocx = nproc[0];
+	nprocy = nproc[1];
+	nprocz = nproc[2];
+	global_Nx = SIZE[0];
+	global_Ny = SIZE[1];
+	global_Nz = SIZE[2];
+	nprocs=nprocx*nprocy*nprocz;
+
+	double *SegData = NULL;
+	if (RANK==0){
+		printf("User-defined input file: %s (data type: %s)\n",Filename.c_str(),Datatype.c_str());
+        printf("NOTE: currently only BC=0 or 5 supports user-defined input file!\n");
+		// Rank=0 reads the entire segmented data and distributes to worker processes
+		printf("Dimensions of the user-defined input file: %ld x %ld x %ld \n",global_Nx,global_Ny,global_Nz);
+		int64_t SIZE = global_Nx*global_Ny*global_Nz;
+
+		if (Datatype == "double"){
+			printf("Reading input data as double precision floating number\n");
+		    SegData = new double[SIZE];
+			FILE *SEGDAT = fopen(Filename.c_str(),"rb");
+			if (SEGDAT==NULL) ERROR("Domain.cpp: Error reading user-defined file!\n");
+			size_t ReadSeg;
+			ReadSeg=fread(SegData,8,SIZE,SEGDAT);
+			if (ReadSeg != size_t(SIZE)) printf("Domain.cpp: Error reading file: %s\n",Filename.c_str());
+			fclose(SEGDAT);
+		}
+        else{
+            ERROR("Error: User-defined input file only supports double-precision floating number!\n"); 
+        }
+		printf("Read file successfully from %s \n",Filename.c_str());
+	}
+	
+	// Get the rank info
+	int64_t N = (nx+2)*(ny+2)*(nz+2);
+
+	// number of sites to use for periodic boundary condition transition zone
+	//int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2;
+	//if (z_transition_size < 0) z_transition_size=0;
+	int64_t z_transition_size = 0;
+
+	//char LocalRankFilename[1000];//just for debug
+	double *loc_id;
+	loc_id = new double [(nx+2)*(ny+2)*(nz+2)];
+
+	// Set up the sub-domains
+	if (RANK==0){
+        printf("Decomposing user-defined input file\n");
+		printf("Distributing subdomains across %i processors \n",nprocs);
+		printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz);
+		printf("Subdomain size: %i x %i x %i \n",nx,ny,nz);
+		printf("Size of transition region: %ld \n", z_transition_size);
+
+		for (int kp=0; kp<nprocz; kp++){
+			for (int jp=0; jp<nprocy; jp++){
+				for (int ip=0; ip<nprocx; ip++){
+					// rank of the process that gets this subdomain
+					int rnk = kp*nprocx*nprocy + jp*nprocx + ip;
+					// Pack and send the subdomain for rnk
+					for (k=0;k<nz+2;k++){
+						for (j=0;j<ny+2;j++){
+							for (i=0;i<nx+2;i++){
+								int64_t x = xStart + ip*nx + i-1;
+								int64_t y = yStart + jp*ny + j-1;
+								// int64_t z = zStart + kp*nz + k-1;
+								int64_t z = zStart + kp*nz + k-1 - z_transition_size;
+								if (x<xStart) 	x=xStart;
+								if (!(x<global_Nx))	x=global_Nx-1;
+								if (y<yStart) 	y=yStart;
+								if (!(y<global_Ny))	y=global_Ny-1;
+								if (z<zStart) 	z=zStart;
+								if (!(z<global_Nz))	z=global_Nz-1;
+								int64_t nlocal = k*(nx+2)*(ny+2) + j*(nx+2) + i;
+								int64_t nglobal = z*global_Nx*global_Ny+y*global_Nx+x;
+								loc_id[nlocal] = SegData[nglobal];
+							}
+						}
+					}
+					if (rnk==0){
+						for (k=0;k<nz+2;k++){
+							for (j=0;j<ny+2;j++){
+								for (i=0;i<nx+2;i++){
+									int nlocal = k*(nx+2)*(ny+2) + j*(nx+2) + i;
+									UserData[nlocal] = loc_id[nlocal];
+								}
+							}
+						}
+					}
+					else{
+						//printf("Sending data to process %i \n", rnk);
+						MPI_Send(loc_id,N,MPI_DOUBLE,rnk,15,Comm);
+					}
+					// Write the data for this rank data 
+                    // NOTE just for debug
+					//sprintf(LocalRankFilename,"%s.%05i",Filename.c_str(),rnk+rank_offset);
+					//FILE *ID = fopen(LocalRankFilename,"wb");
+					//fwrite(loc_id,8,(nx+2)*(ny+2)*(nz+2),ID);
+					//fclose(ID);
+				}
+			}
+		}
+
+	}
+	else{
+		// Recieve the subdomain from rank = 0
+		//printf("Ready to recieve data %i at process %i \n", N,rank);
+		MPI_Recv(UserData,N,MPI_DOUBLE,0,15,Comm,MPI_STATUS_IGNORE);
+	}
+	//Comm.barrier();
+	MPI_Barrier(Comm);
+}

common/Domain.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -31,7 +32,6 @@
 #include "common/Communication.h"
 #include "common/Database.h"
 
-
 class Domain;
 template<class TYPE> class PatchData;
 
@@ -193,11 +193,12 @@ public: // Public variables (need to create accessors instead)
     signed char *id;
 
     void ReadIDs();
-    void Decomp(std::string Filename);
+    void Decomp( const std::string& filename );
+    void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
     void CommunicateMeshHalo(DoubleArray &Mesh);
     void CommInit(); 
     int PoreCount();
-    void AggregateLabels(char *FILENAME);
+    void AggregateLabels( const std::string& filename );
 
 private:
 
@@ -260,10 +261,13 @@ private:
 
 };
 
-void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np);
+//void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
+//void ReadFromFile(const std::string& Filename, DoubleArray &Mesh);
 
-void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np);
+void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np);
 
-void ReadBinaryFile(char *FILENAME, double *Data, int N);
+void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, size_t Np);
+
+void ReadBinaryFile(char *FILENAME, double *Data, size_t N);
 
 #endif

common/FunctionTable.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/FunctionTable.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/MPI_Helpers.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/MPI_Helpers.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -203,6 +204,12 @@ inline int sumReduce( MPI_Comm comm, int x )
 	MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm);
     return y;
 }
+inline long long sumReduce( MPI_Comm comm, long long x )
+{
+    long long y = 0;
+	MPI_Allreduce(&x,&y,1,MPI_LONG_LONG,MPI_SUM,comm);
+    return y;
+}
 inline bool sumReduce( MPI_Comm comm, bool x )
 {
     int y = sumReduce( comm, x?1:0 );

common/MPI_Helpers.hpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/ReadMicroCT.cpp

@@ -0,0 +1,118 @@
+#include "common/ReadMicroCT.h"
+#include "common/Utilities.h"
+
+#include "zlib.h"
+
+#include <cstring>
+
+
+// Read a file into memory
+std::vector<char> readFile( const std::string& filename )
+{
+    auto fid = fopen( filename.c_str(), "rb" );
+    INSIST( fid, "File does not exist: " + filename );
+    fseek( fid, 0, SEEK_END );
+    size_t bytes = ftell(fid);
+    fseek( fid, 0, SEEK_SET );
+    std::vector<char> data( bytes );
+    size_t bytes2 = fread( data.data(), 1, bytes, fid );
+    ASSERT( bytes == bytes2 );
+    fclose( fid );
+    return data;
+}
+
+
+// Decompress a gzip buffer
+std::vector<char> gunzip( const std::vector<char>& in )
+{
+    z_stream stream;
+    std::vector<char> out( 1000000 );
+    stream.next_in = (Bytef*) in.data();
+    stream.avail_in = in.size();
+    stream.total_in = 0;
+    stream.zalloc = Z_NULL;
+    stream.zfree = Z_NULL;
+    stream.opaque = Z_NULL;
+    stream.next_out = (Bytef*) out.data();
+    stream.avail_out = out.size();
+    stream.total_out = 0;
+    ASSERT( inflateInit2(&stream, 16+MAX_WBITS) == Z_OK );
+    bool finished = inflate(&stream, Z_SYNC_FLUSH) == Z_STREAM_END;
+    while ( !finished && stream.msg == Z_NULL ) {
+        out.resize( 2 * out.size() );
+        stream.next_out = (Bytef*) &out[stream.total_out];
+        stream.avail_out = out.size() - stream.total_out;
+        finished = inflate(&stream, Z_SYNC_FLUSH) == Z_STREAM_END;
+    }
+    ASSERT( stream.msg == Z_NULL );
+    out.resize( stream.total_out );
+    inflateEnd(&stream);
+    return out;
+}
+
+
+// Read the compressed micro CT data
+Array<uint8_t> readMicroCT( const std::string& filename )
+{
+    auto in  = readFile( filename );
+    auto out = gunzip( in );
+    ASSERT( out.size() == 1024*1024*1024 );
+    Array<uint8_t> data( 1024, 1024, 1024 );
+    memcpy( data.data(), out.data(), data.length() );
+    return data;
+}
+
+
+// Read the compressed micro CT data and distribute
+Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
+{
+    // Get the local problem info
+    auto n = domain.getVector<int>( "n" );
+    int rank = comm_rank(MPI_COMM_WORLD);
+    auto nproc = domain.getVector<int>( "nproc" );
+    RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );
+    
+    // Determine the largest file number to get
+    int Nfx = ( n[0] * rankInfo.nx + 1023 ) / 1024;
+    int Nfy = ( n[1] * rankInfo.ny + 1023 ) / 1024;
+    int Nfz = ( n[2] * rankInfo.nz + 1023 ) / 1024;
+
+    // Load one of the files if rank < largest file
+    Array<uint8_t> data;
+    RankInfoStruct srcRankInfo( rank, Nfx, Nfy, Nfz );
+    if ( srcRankInfo.ix >= 0 ) {
+    	auto filename = domain.getScalar<std::string>( "Filename" );
+    	char tmp[100];
+    	if ( filename.find( "0x_0y_0z.gbd.gz" ) != std::string::npos ) {
+    		sprintf( tmp, "%ix_%iy_%iz.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
+    		filename = filename.replace( filename.find( "0x_0y_0z.gbd.gz" ), 15, std::string( tmp ) );
+    	} else if ( filename.find( "x0_y0_z0.gbd.gz" ) != std::string::npos ) {
+    		sprintf( tmp, "x%i_y%i_z%i.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
+    		filename = filename.replace( filename.find( "x0_y0_z0.gbd.gz" ), 15, std::string( tmp ) );
+    	} else {
+    		ERROR( "Invalid name for first file" );
+    	}
+    	data = readMicroCT( filename );
+    }
+
+    // Redistribute the data
+    data = redistribute( srcRankInfo, data, rankInfo, { n[0], n[1], n[2] }, comm );
+
+	// Relabel the data
+    auto ReadValues = domain.getVector<int>( "ReadValues" );
+    auto WriteValues = domain.getVector<int>( "WriteValues" );
+    ASSERT( ReadValues.size() == WriteValues.size() );
+    int readMaxValue = 0;
+    for ( auto v : ReadValues )
+        readMaxValue = std::max( data.max()+1, v );
+    std::vector<int> map( readMaxValue + 1, -1 );
+    for ( size_t i=0; i<ReadValues.size(); i++ )
+        map[ReadValues[i]] = WriteValues[i];
+    for ( size_t i=0; i<data.length(); i++ ) {
+        int t = data(i);
+        ASSERT( t >= 0 && t <= readMaxValue );
+        data(i) = map[t];
+    }
+
+    return data;
+}

common/ReadMicroCT.h

@@ -0,0 +1,15 @@
+#ifndef READMICROCT_H
+#define READMICROCT_H
+
+
+#include "common/Array.h"
+#include "common/Communication.h"
+#include "common/Database.h"
+
+
+Array<uint8_t> readMicroCT( const std::string& filename );
+
+Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm );
+
+
+#endif

common/ScaLBL.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -95,43 +96,43 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
 	BoundaryCondition = Dm->BoundaryCondition;
 	//......................................................................................
 
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 5*sendCount_X*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 5*sendCount_y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 5*sendCount_z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, sendCount_xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, sendCount_xY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, sendCount_XY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, sendCount_xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, sendCount_yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 2*5*sendCount_x*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 2*5*sendCount_X*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 2*5*sendCount_y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 2*5*sendCount_Y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 2*5*sendCount_z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 2*5*sendCount_Z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, 2*sendCount_xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, 2*sendCount_xY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, 2*sendCount_Xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, 2*sendCount_XY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, 2*sendCount_xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, 2*sendCount_xZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, 2*sendCount_Xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, 2*sendCount_XZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, 2*sendCount_yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, 2*sendCount_yZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, 2*sendCount_Yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, 2*sendCount_YZ*sizeof(double));	// Allocate device memory
 	//......................................................................................
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 5*recvCount_x*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 5*recvCount_X*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 5*recvCount_y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 5*recvCount_z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, recvCount_xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, recvCount_xY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, recvCount_XY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, recvCount_xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, recvCount_yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 2*5*recvCount_x*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 2*5*recvCount_X*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 2*5*recvCount_y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 2*5*recvCount_Y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 2*5*recvCount_z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 2*5*recvCount_Z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, 2*recvCount_xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, 2*recvCount_xY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, 2*recvCount_Xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, 2*recvCount_XY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, 2*recvCount_xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, 2*recvCount_xZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, 2*recvCount_Xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, 2*recvCount_XZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, 2*recvCount_yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, 2*recvCount_yZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, 2*recvCount_Yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, 2*recvCount_YZ*sizeof(double));	// Allocate device memory
 	//......................................................................................
 	ScaLBL_AllocateZeroCopy((void **) &dvcSendList_x, sendCount_x*sizeof(int));	// Allocate device memory
 	ScaLBL_AllocateZeroCopy((void **) &dvcSendList_X, sendCount_X*sizeof(int));	// Allocate device memory
@@ -1026,19 +1027,6 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
 	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N);
 	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N);
 	//...................................................................................
-	//...Packing for z face(6,12,13,16,17)................................
-	ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	//...Packing for Z face(5,11,14,15,18)................................
-	ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	//..................................................................................
 	//...Pack the xy edge (8)................................
 	ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,dist,N);
 	//...Pack the Xy edge (9)................................
@@ -1047,22 +1035,75 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
 	ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,dist,N);
 	//...Pack the XY edge (7)................................
 	ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,dist,N);
-	//...Pack the xz edge (12)................................
-	ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
-	//...Pack the xZ edge (14)................................
-	ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
-	//...Pack the Xz edge (13)................................
-	ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
-	//...Pack the XZ edge (11)................................
-	ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
-	//...Pack the yz edge (16)................................
-	ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
-	//...Pack the yZ edge (18)................................
-	ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
-	//...Pack the Yz edge (17)................................
-	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
-	//...Pack the YZ edge (15)................................
-	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			//...Pack the xz edge (12)................................
+			ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
+			//...Pack the Xz edge (13)................................
+			ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
+			//...Pack the yz edge (16)................................
+			ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
+			//...Pack the Yz edge (17)................................
+			ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
+			//..................................................................................
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			//...Pack the xZ edge (14)................................
+			ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
+			//...Pack the XZ edge (11)................................
+			ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
+			//...Pack the yZ edge (18)................................
+			ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
+			//...Pack the YZ edge (15)................................
+			ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+			//..................................................................................
+		}
+	}
+	else {
+		//...Packing for z face(6,12,13,16,17)................................
+		ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		//...Packing for Z face(5,11,14,15,18)................................
+		ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		//..................................................................................
+		//...Pack the xz edge (12)................................
+		ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
+		//...Pack the xZ edge (14)................................
+		ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
+		//...Pack the Xz edge (13)................................
+		ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
+		//...Pack the XZ edge (11)................................
+		ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
+		//...Pack the yz edge (16)................................
+		ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
+		//...Pack the yZ edge (18)................................
+		ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
+		//...Pack the Yz edge (17)................................
+		ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
+		//...Pack the YZ edge (15)................................
+		ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+	}
+
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
@@ -1085,6 +1126,7 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
 	ScaLBL_Gradient_Unpack(1.0,-1,0,0,dvcRecvDist_x,0,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,-1,0,dvcRecvDist_x,recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,1,0,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
+	ScaLBL_Gradient_Unpack(0.5,-1,0,-1,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,0,1,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	//...................................................................................
 	//...Packing for X face(1,7,9,11,13)................................
@@ -1240,18 +1282,18 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N);
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,recvCount_Y,recvCount_Y,recvbuf_Y,Bq,N);
 	//...................................................................................
-	
-	if (BoundaryCondition > 0 && kproc == 0){
-		// don't unpack little z
-		//...Packing for Z face(5,11,14,15,18)................................
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
-	}
-	else if (BoundaryCondition > 0 && kproc == nprocz-1){
-		// don't unpack big z
-		//...Packing for z face(6,12,13,16,17)................................
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
+		}
 	}
 	else {
 		//...Packing for z face(6,12,13,16,17)................................
@@ -1261,7 +1303,17 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){
 		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
 		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
 	}
-	
+/*	if (BoundaryCondition == 5){
+		if (kproc == 0){
+			ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Aq, sendCount_z, N);
+			ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Bq, sendCount_z, N);
+		}
+		if (kproc == nprocz-1){
+			ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Aq, sendCount_Z, N);
+			ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Bq, sendCount_Z, N);
+		}
+	}
+	*/
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
@@ -1358,19 +1410,19 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Cq,N);
 	//...................................................................................
 	
-	if (BoundaryCondition > 0 && kproc == 0){
-		// don't unpack little z
-		//...Packing for Z face(5,11,14,15,18)................................
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N);
-	}
-	else if (BoundaryCondition > 0 && kproc == nprocz-1){
-		// don't unpack big z
-		//...Packing for z face(6,12,13,16,17)................................
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N);
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N);
+		}
 	}
 	else {
 		//...Packing for z face(6,12,13,16,17)................................
@@ -1472,30 +1524,57 @@ void ScaLBL_Communicator::RecvHalo(double *data){
 	//...................................................................................
 	ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+	
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+		}
+	}
+	else {
+		ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+	}
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
+	if (BoundaryCondition == 5 && kproc == 0){
+		ScaLBL_CopySlice_z(data,Nx,Ny,Nz,1,0);
+	}
+	if (BoundaryCondition == 5 && kproc == nprocz-1){
+		ScaLBL_CopySlice_z(data,Nx,Ny,Nz,Nz-2,Nz-1);
+	}	
 }
 
 void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray &regdata){
 	// Gets data from the device and stores in regular layout
-	int i,j,k,n,idx;
+	int i,j,k,idx;
 	int Nx = map.size(0);
 	int Ny = map.size(1);
 	int Nz = map.size(2);
@@ -1524,23 +1603,30 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double
 	delete [] TmpDat;
 }
 
-
 void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){
-	double Value=(vA-vB)/(vA+vB);
 	if (kproc == 0) {
-		// Set the phase indicator field and density on the z inlet
-		ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N);
+		if (BoundaryCondition == 5){
+			//ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,1,0);
+		}
+		else {
+			// Set the phase indicator field and density on the z inlet
+			ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N);
+		}
 		//ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,0);
 	}
 }
 
 void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){
-	double Value=(vA-vB)/(vA+vB);
 	if (kproc == nprocz-1){
+		if (BoundaryCondition == 5){
+			//ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,Nz-2,Nz-1);
+		}
+		else {
 		// Set the phase indicator field and density on the Z outlet
-		ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N);
-		//ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,Nz-1);
+			ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N);
+		}
 	}
+
 }
 
 void ScaLBL_Communicator::D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time){
@@ -1608,6 +1694,17 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl
 	return din;
 }
 
+void ScaLBL_Communicator::D3Q19_Reflection_BC_z(double *fq){
+	if (kproc == 0)
+		ScaLBL_D3Q19_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N);
+	
+}
+
+void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(double *fq){
+	if (kproc == nprocz-1)
+		ScaLBL_D3Q19_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N);
+}
+
 void ScaLBL_Communicator::PrintD3Q19(){
 	printf("Printing D3Q19 communication buffer contents \n");
 

common/ScaLBL.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -61,6 +62,7 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int
 
 extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np);
 
+
 extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np);
 
 extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np);
@@ -70,6 +72,136 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int
 
 extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
 
+// GREYSCALE MODEL (Single-component)
+
+extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+// GREYSCALE FREE-ENERGY MODEL (Two-component)
+
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den,
+//                double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,
+//                double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
+//                double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
+//                double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
+//
+//extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity,
+//                double *Pressure, double rhoA,double rhoB, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap,
+//      		     double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np);
+
+// GREYSCALE SHAN-CHEN MODEL (Two-component)
+
+//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
+//
+//extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
+
+// GREYSCALE COLOR MODEL (Two-component)
+//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np);
+
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+//        double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+//        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+//		double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+//        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+        double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure,
+        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure, 
+        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
 // MRT MODEL
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
 		double Fy, double Fz);
@@ -116,11 +248,6 @@ extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double
 
 // BOUNDARY CONDITION ROUTINES
 
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din,
-//		int Nx, int Ny, int Nz);
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout,
-//		int Nx, int Ny, int Nz, int outlet);
-
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np);
@@ -139,8 +266,18 @@ extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den,
 
 extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
 
+extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np);
+
 extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice);
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination);
+
 class ScaLBL_Communicator{
 public:
 	//......................................................................................
@@ -149,6 +286,7 @@ public:
 	//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
 	~ScaLBL_Communicator();
 	//......................................................................................
+	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator
 	unsigned long int CommunicationCount,SendCount,RecvCount;
 	int Nx,Ny,Nz,N;
 	int BoundaryCondition;
@@ -174,14 +312,6 @@ public:
 	int LastInterior();
 	
 	int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np);
-//	void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
-//	void SendD3Q19(double *f_even, double *f_odd);
-//	void RecvD3Q19(double *f_even, double *f_odd);
-//	void SendD3Q19AA(double *f_even, double *f_odd);
-//	void RecvD3Q19AA(double *f_even, double *f_odd);
 	void SendD3Q19AA(double *dist);
 	void RecvD3Q19AA(double *dist);
 //	void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
@@ -200,8 +330,13 @@ public:
 	void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB);
 	void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time);
 	void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
+	void D3Q19_Reflection_BC_z(double *fq);
+	void D3Q19_Reflection_BC_Z(double *fq);
 	double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
-
+	void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
+	void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
+    void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
+    void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
 //	void TestSendD3Q19(double *f_even, double *f_odd);
 //	void TestRecvD3Q19(double *f_even, double *f_odd);
 
@@ -222,7 +357,6 @@ private:
 	// Give the object it's own MPI communicator
 	RankInfoStruct rank_info;
 	MPI_Group Group;	// Group of processors associated with this domain
-	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator for this domain
 	MPI_Request req1[18],req2[18];
 	MPI_Status stat1[18],stat2[18];
 	//......................................................................................

common/SpherePack.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/SpherePack.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/UnitTest.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/UnitTest.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Units.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Units.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

common/Utilities.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -14,12 +15,132 @@
   along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 */
 #include "common/Utilities.h"
+#include "StackTrace/StackTrace.h"
+#include "StackTrace/ErrorHandlers.h"
+
+#ifdef USE_TIMER
+#include "MemoryApp.h"
+#include "ProfilerApp.h"
+#endif
+
+#ifdef USE_MPI
+#include "mpi.h"
+#endif
 
-#include <math.h>
 #include <algorithm>
+#include <math.h>
+#include <mutex>
 
 
-// Factor a number into it's prime factors
+// OS specific includes / definitions
+// clang-format off
+#if defined( WIN32 ) || defined( _WIN32 ) || defined( WIN64 ) || defined( _WIN64 )
+    #define USE_WINDOWS
+#elif defined( __APPLE__ )
+    #define USE_MAC
+#elif defined( __linux ) || defined( __linux__ ) || defined( __unix ) || defined( __posix )
+    #define USE_LINUX
+#else
+    #error Unknown OS
+#endif
+// clang-format on
+
+
+// Mutex for Utility functions
+static std::mutex Utilities_mutex;
+
+
+/****************************************************************************
+ *  Function to perform the default startup/shutdown sequences               *
+ ****************************************************************************/
+void Utilities::startup( int argc, char **argv )
+{
+    NULL_USE( argc );
+    NULL_USE( argv );
+    // Disable OpenMP
+    Utilities::setenv( "OMP_NUM_THREADS", "1" );
+    Utilities::setenv( "MKL_NUM_THREADS", "1" );
+    // Start MPI
+#ifdef USE_MPI
+    int provided;
+    MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+    if ( provided < MPI_THREAD_MULTIPLE ) {
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+        if ( rank == 0 )
+            std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
+    }
+    StackTrace::globalCallStackInitialize( MPI_COMM_WORLD );
+#endif
+    // Set the error handlers
+    Utilities::setAbortBehavior( true, 3 );
+    Utilities::setErrorHandlers();
+}
+void Utilities::shutdown()
+{
+    // Clear the error handlers
+    Utilities::clearErrorHandlers();
+    StackTrace::clearSignals();
+    StackTrace::clearSymbols();
+    int rank = 0;
+#ifdef USE_MPI
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+    StackTrace::globalCallStackFinalize();
+    MPI_Barrier( MPI_COMM_WORLD );
+    MPI_Finalize();
+#endif
+#ifdef USE_TIMER
+    PROFILE_DISABLE();
+    auto memory = MemoryApp::getMemoryStats();
+    if ( rank == 0 && memory.N_new > memory.N_delete )
+        MemoryApp::print( std::cout );
+#endif
+}
+
+
+/****************************************************************************
+ *  Function to set an environemental variable                               *
+ ****************************************************************************/
+void Utilities::setenv( const std::string &name, const std::string &value )
+{
+    Utilities_mutex.lock();
+#if defined( USE_LINUX ) || defined( USE_MAC )
+    bool pass = false;
+    if ( !value.empty() )
+        pass = ::setenv( name.data(), value.data(), 1 ) == 0;
+    else
+        pass = ::unsetenv( name.data() ) == 0;
+#elif defined( USE_WINDOWS )
+    bool pass = SetEnvironmentVariable( name.data(), value.data() ) != 0;
+#else
+#error Unknown OS
+#endif
+    Utilities_mutex.unlock();
+    if ( !pass ) {
+        char msg[1024];
+        if ( !value.empty() )
+            sprintf(
+                msg, "Error setting enviornmental variable: %s=%s\n", name.data(), value.data() );
+        else
+            sprintf( msg, "Error clearing enviornmental variable: %s\n", name.data() );
+        ERROR( msg );
+    }
+}
+std::string Utilities::getenv( const std::string &name )
+{
+    std::string var;
+    Utilities_mutex.lock();
+    auto tmp = std::getenv( name.data() );
+    if ( tmp )
+        var = std::string( tmp );
+    Utilities_mutex.unlock();
+    return var;
+}
+
+
+/****************************************************************************
+ *  Factor a number into it's prime factors                                  *
+ ****************************************************************************/
 std::vector<int> Utilities::factor(size_t number)
 {
     if ( number<=3 ) 
@@ -69,9 +190,13 @@ std::vector<int> Utilities::factor(size_t number)
 }
 
 
-// Dummy function to prevent compiler from optimizing away variable
+/****************************************************************************
+ *  Dummy function to prevent compiler from optimizing away variable         *
+ ****************************************************************************/
 void Utilities::nullUse( void* data )
 {
     NULL_USE(data);
 }
 
+
+

common/Utilities.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -40,6 +41,37 @@ using StackTrace::Utilities::sleep_ms;
 using StackTrace::Utilities::sleep_s;
 
 
+/*!
+ * \brief Start MPI, error handlers
+ * \details This routine will peform the default startup sequence
+ * \param argc              argc from main
+ * \param argv              argv from main
+ */
+void startup( int argc, char **argv );
+
+/*!
+ * \brief Stop MPI, error handlers
+ * \details This routine will peform the default shutdown sequence to match startup
+ */
+void shutdown();
+
+
+/*!
+ * Get an environmental variable
+ * @param name              The name of the environmental variable
+ * @return                  The value of the enviornmental variable
+ */
+std::string getenv( const std::string &name );
+
+
+/*!
+ * Set an environmental variable
+ * @param name              The name of the environmental variable
+ * @param value             The value to set
+ */
+void setenv( const std::string &name, const std::string &value );
+
+
 //! std::string version of sprintf
 inline std::string stringf( const char *format, ... );
 

common/UtilityMacros.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/BGK.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

cpu/Color.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -1379,11 +1380,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do
 
 		//...........Normalize the Color Gradient.................................
 		C = sqrt(nx*nx+ny*ny+nz*nz);
-		if (C==0.0) C=1.0;
-		nx = nx/C;
-		ny = ny/C;
-		nz = nz/C;		
-
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
+		
 		// q=0
 		fq = dist[n];
 		rho = fq;
@@ -1883,7 +1885,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 	const double mrt_V12=0.04166666666666666;
 
 	for (int n=start; n<finish; n++){
-		
+	
 		// read the component number densities
 		nA = Den[n];
 		nB = Den[Np + n];
@@ -1964,10 +1966,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 
 		//...........Normalize the Color Gradient.................................
 		C = sqrt(nx*nx+ny*ny+nz*nz);
-		if (C==0.0) C=1.0;
-		nx = nx/C;
-		ny = ny/C;
-		nz = nz/C;		
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
 
 		// q=0
 		fq = dist[n];
@@ -2823,3 +2826,11 @@ extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, doubl
 	}
 }
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	int n; double value;
+	for (n=0; n<Nx*Ny; n++){
+		value = Phi[Source*Nx*Ny+n];
+		Phi[Dest*Nx*Ny+n] = value;
+	}
+}
+

cpu/D3Q7.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -87,6 +88,23 @@ extern  "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int
 	}
 }
 
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+  int n;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		double f5 = 0.222222222222222222222222 - dist[6*Np+n];
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+  int n;
+  for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		double f6 = 0.222222222222222222222222 - dist[5*Np+n];
+		dist[5*Np+n] = f6;
+	}
+}
 extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
 {
 	int n,N;

cpu/dfh.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -53,11 +55,10 @@ extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, do
 	}
 }
 
-extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np){
-	int idx,n;
-	double phi,nA,nB;
-
-	for (idx=start; idx<finish; idx++){
+extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np)
+{
+	for (int idx=start; idx<finish; idx++){
+	    double phi,nA,nB;
 		phi = Phi[idx];
 		if (phi > 0.f){
 			nA = 1.0; nB = 0.f;
@@ -90,15 +91,13 @@ extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq
 // LBM based on density functional hydrodynamics
 extern "C" void ScaLBL_D3Q19_AAeven_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
-		double Fx, double Fy, double Fz, int start, int finish, int Np){
-
-	int ijk,nn,n;
+		double Fx, double Fy, double Fz, int start, int finish, int Np)
+{
 	double fq;
 	// conserved momemnts
 	double rho,jx,jy,jz;
 	// non-conserved moments
 	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
-	double m3,m5,m7;
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
@@ -616,7 +615,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 		double *Phi, double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int start, int finish, int Np){
 	
-	int n,nn,ijk,nread;
+	int nread;
 	int nr1,nr2,nr3,nr4,nr5,nr6;
 	int nr7,nr8,nr9,nr10;
 	int nr11,nr12,nr13,nr14;
@@ -626,7 +625,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 	double rho,jx,jy,jz;
 	// non-conserved moments
 	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
-	double m3,m5,m7;
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
@@ -1212,12 +1210,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 }
 
 extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq, 
-			double *Den, double *Phi, int start, int finish, int Np){
-
-	int idx,n,nread;
-	double fq,nA,nB;
+			double *Den, double *Phi, int start, int finish, int Np)
+{
 
 	for (int n=start; n<finish; n++){
+		int nread;
+		double fq,nA,nB;
 		
 		//..........Compute the number density for component A............
 		// q=0
@@ -1300,11 +1298,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
 }
 
 extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, double *Phi, 
-			int start, int finish, int Np){
-	int idx,n,nread;
-	double fq,nA,nB;
+			int start, int finish, int Np)
+{
 	for (int n=start; n<finish; n++){
-		
+		double fq,nA,nB;
 		// compute number density for component A
 		// q=0
 		fq = Aq[n];

example/Sph1896/input.db

@@ -26,27 +26,25 @@ Color {
 }
 
 Domain {
-    nproc = 2, 2, 2        // Number of processors (Npx,Npy,Npz)
-    n = 256, 256, 256      // Size of local domain (Nx,Ny,Nz)
-    N = 512, 512, 512      
+    nproc = 1, 1, 1        // Number of processors (Npx,Npy,Npz)
+    n = 320, 320, 320      // Size of local domain (Nx,Ny,Nz)
+    N = 320, 320, 320      
     nspheres = 1896        // Number of spheres (only needed if using a sphere packing)
     L = 1, 1, 1            // Length of domain (x,y,z)
     BC = 0                 // Boundary condition type 
                            //     BC = 0  for periodic BC
                            //     BC = 1  for pressure BC (applied in z direction)
                            //     BC = 4  for flux BC (applied in z direction
-    // Filename = "mineralmodel.raw"                    // name of raw binary file to read digital image from 
     ReadType = "8bit"
-    ReadValues = 0, 1, 2, 3, 4, 5, 6, 7, 8           // list of labels within the binary file (read)
-    WriteValues = 1, -1, -2, -2, -3, -3, -4, -4, -4  // list of labels within the output files (write)
+    ReadValues = 0, 1, 2           // list of labels within the binary file (read)
+    WriteValues = 0, 1, 2          // list of labels within the output files (write)
 
 }
 
 Analysis {
-    blobid_interval = 1000      // Frequency to perform blob identification
     analysis_interval = 1000    // Frequency to perform analysis
-    restart_interval = 2000    // Frequency to write restart data
-    visualization_interval = 2000        // Frequency to write visualization data
+    restart_interval = 50000    // Frequency to write restart data
+    visualization_interval = 50000        // Frequency to write visualization data
     restart_file = "Restart"    // Filename to use for restart file (will append rank)
     N_threads    = 4            // Number of threads to use
     load_balance = "independent" // Load balance method to use: "none", "default", "independent"

example/Workflow/ComputeSaturation.py

@@ -0,0 +1,37 @@
+import sys
+import numpy as np
+import matplotlib.pylab as plt
+
+FILENAME=sys.argv[1]
+Nx=int(sys.argv[2])
+Ny=int(sys.argv[3])
+Nz=int(sys.argv[4])
+
+# read the input image
+Output = np.fromfile(FILENAME,dtype = np.uint8)
+Output.shape = (Nz,Ny,Nx)
+
+Oil=np.count_nonzero(Output==1)
+Water=np.count_nonzero(Output==2)
+Sw=Water/(Oil+Water)
+
+Porosity=1.0-(Oil+Water)/(Nx*Ny*Nz)
+
+print(FILENAME,"Porosity=", Porosity)
+
+SaturationProfile=np.zeros(Nz)
+PorosityProfile=np.zeros(Nz)
+# Compute saturation slice by slice 
+for idx in range(0, Nz):
+   Slice = Output[idx,:,:]
+   Oil=np.count_nonzero(Slice==1)
+   Water=np.count_nonzero(Slice==2)
+   SaturationProfile[idx]=Water/(Oil+Water)
+   PorosityProfile[idx]=(Oil+Water)/(Nx*Ny)
+   
+
+plt.figure()
+plt.plot(SaturationProfile)
+plt.xlabel('Position (z)')
+plt.ylabel('Water Saturation')
+plt.show()

example/Workflow/HelperFunctions.R

@@ -2,16 +2,57 @@ require("ggplot2")
 
 GG_THEME=theme_bw()+theme(panel.grid.major = element_blank(),panel.grid.minor = element_blank())
 
+ReadDatabase<-function(FILE){
+
+    INPUT<-gsub(';','',readLines(FILE))
+
+    S<-gsub('tauA = ','',gsub("\\s+"," ",(grep("tauA",INPUT,value=TRUE))))
+    TAU_A = as.numeric(gsub("/.*","",S))
+    S<-gsub('tauB = ','',gsub("\\s+"," ",(grep("tauB",INPUT,value=TRUE))))
+    TAU_B = as.numeric(gsub("/.*","",S))
+    S<-gsub('rhoA = ','',gsub("\\s+"," ",(grep("rhoA",INPUT,value=TRUE))))
+    RHO_A = as.numeric(gsub("/.*","",S))
+    S<-gsub('rhoB = ','',gsub("\\s+"," ",(grep("rhoB",INPUT,value=TRUE))))
+    RHO_B = as.numeric(gsub("/.*","",S))
+
+    S<-gsub('alpha = ','',gsub("\\s+"," ",(grep("alpha",INPUT,value=TRUE))))
+    ALPHA = as.numeric(gsub("/.*","",S))
+
+    # Read the affinity
+    S<-gsub('ComponentAffinity = ','',gsub("\\s+"," ",(grep("ComponentAffinity",INPUT,value=TRUE))))
+    S<-gsub("/.*","",S)
+    AFFINITY<-as.numeric(unlist(strsplit(S,", ")))
+
+    PARAMETERS<-c(TAU_A,TAU_B,RHO_A,RHO_B,ALPHA,AFFINITY)
+    
+    return(PARAMETERS)
+}
+
 ReadSubphase<-function(PATH){
   FILE=paste0(PATH,"/subphase.csv")
   S<-read.csv(FILE,head=TRUE,sep=" ")
   S$Vw<-S$Vwc+S$Vwd
   S$Vn<-S$Vnc+S$Vnd
+  S$Aw<-S$Awc+S$Awd
+  S$An<-S$Anc+S$And
+  S$Hw<-S$Hwc+S$Hwd
+  S$Hn<-S$Hnc+S$Hnd
+  S$Xw<-S$Xwc+S$Xwd
+  S$Xn<-S$Xnc+S$Xnd
+
   S$Sw<-S$Vw/(S$Vn+S$Vw)
+  S$pw<-(S$pwc*S$Vwc+S$pwd*S$Vwd) / (S$Vwc+S$Vwd)
+  S$pn<-(S$pnc*S$Vnc+S$pnd*S$Vnd) / (S$Vnc+S$Vnd)
+
   S$Qwx<-S$Vw*(S$Pwc_x+S$Pwd_x)/(S$Mwc+S$Mwd)
   S$Qnx<-S$Vn*(S$Pnc_x+S$Pnd_x)/(S$Mnc+S$Mnd)
-  S$Krn<-S$nun*S$Qnx/S$Fx
-  S$Krw<-S$nuw*S$Qwx/S$Fx
+  S$Qwy<-S$Vw*(S$Pwc_y+S$Pwd_y)/(S$Mwc+S$Mwd)
+  S$Qny<-S$Vn*(S$Pnc_y+S$Pnd_y)/(S$Mnc+S$Mnd)
+  S$Qwz<-S$Vw*(S$Pwc_z+S$Pwd_z)/(S$Mwc+S$Mwd)
+  S$Qnz<-S$Vn*(S$Pnc_z+S$Pnd_z)/(S$Mnc+S$Mnd)
+
+  S$Krn<-S$nun*S$Qnz/S$Fz
+  S$Krw<-S$nuw*S$Qwz/S$Fz
   S$Case<-PATH
   return(S)
 }

gpu/BGK.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/Color.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -143,7 +144,7 @@ __global__  void dvc_ScaLBL_Color_InitDistance(char *ID, double *Den, double *Ph
 
 __global__  void dvc_ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np)
 {
-	int idx,n,nm;
+	int idx,n;
 	// Fill the outlet with component b
 	idx = blockIdx.x*blockDim.x + threadIdx.x;
 	if (idx < count){
@@ -1255,6 +1256,15 @@ __global__  void dvc_ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny
 }
 
 
+__global__  void dvc_ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	double value;
+	int n =  blockIdx.x*blockDim.x + threadIdx.x;
+	if (n < Nx*Ny){
+		value = Phi[Source*Nx*Ny+n];
+		Phi[Dest*Nx*Ny+n] = value;
+	}
+}
+
 
 __global__  void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Velocity, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
@@ -1371,10 +1381,11 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *A
 
 			//...........Normalize the Color Gradient.................................
 			C = sqrt(nx*nx+ny*ny+nz*nz);
-			if (C==0.0) C=1.0;
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;		
+			double ColorMag = C;
+			if (C==0.0) ColorMag=1.0;
+			nx = nx/ColorMag;
+			ny = ny/ColorMag;
+			nz = nz/ColorMag;		
 
 			// q=0
 			fq = dist[n];
@@ -1957,10 +1968,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double
 
 			//...........Normalize the Color Gradient.................................
 			C = sqrt(nx*nx+ny*ny+nz*nz);
-			if (C==0.0) C=1.0;
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;		
+			double ColorMag = C;
+			if (C==0.0) ColorMag=1.0;
+			nx = nx/ColorMag;
+			ny = ny/ColorMag;
+			nz = nz/ColorMag;		
 
 			// q=0
 			fq = dist[n];
@@ -3484,13 +3496,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_ColorMass(double *Aq, double *Bq, double
 		double *Velocity, double *ColorGrad, double beta, int start, int finish, int Np){
 
 	int n;
-	double fq;
 	// non-conserved moments
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
 	double ux,uy,uz;
-	double phi,tau,rho0,rlx_setA,rlx_setB;
 
 	int S = Np/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -3578,13 +3588,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_ColorMass(int *neighborList, double *Aq,
 		double *Velocity, double *ColorGrad, double beta, int start, int finish, int Np){
 
 	int n,nread;
-	double fq;
 	// non-conserved moments
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
 	double ux,uy,uz;
-	double phi,tau,rho0,rlx_setA,rlx_setB;
 
 	int S = Np/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -4151,5 +4159,9 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den,
 	}
 }
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	int GRID = Nx*Ny / 512 + 1;
+	dvc_ScaLBL_CopySlice_z<<<GRID,512>>>(Phi,Nx,Ny,Nz,Source,Dest);
+}
 
 

gpu/CudaExtras.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

gpu/D3Q19.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -282,7 +283,6 @@ __global__ void dvc_ScaLBL_D3Q19_Init(double *dist, int Np)
 	}
 }
 
-
 //*************************************************************************
 __global__  void dvc_ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np, int q){
 	int n,nn;
@@ -1553,7 +1553,6 @@ __global__  void dvc_ScaLBL_D3Q19_Momentum(double *dist, double *vel, int N)
 	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
 	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
 	double vx,vy,vz;
-	char id;
 
 	int S = N/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -1744,6 +1743,43 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist,
 		//...................................................
 	}
 }
+__global__  void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f5 = 0.111111111111111111111111 - dist[6*Np+n];
+		double f11 = 0.05555555555555555555556 - dist[12*Np+n];
+		double f14 = 0.05555555555555555555556 - dist[13*Np+n];
+		double f15 = 0.05555555555555555555556 - dist[16*Np+n];
+		double f18 = 0.05555555555555555555556 - dist[17*Np+n];
+		
+		dist[6*Np+n] = f5;
+		dist[12*Np+n] = f11;
+		dist[13*Np+n] = f14;
+		dist[16*Np+n] = f15;
+		dist[17*Np+n] = f18;
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f6 = 0.111111111111111111111111 - dist[5*Np+n];
+		double f12 = 0.05555555555555555555556 - dist[11*Np+n];
+		double f13 = 0.05555555555555555555556 - dist[14*Np+n] ;
+		double f16 = 0.05555555555555555555556 - dist[15*Np+n];
+		double f17 = 0.05555555555555555555556 - dist[18*Np+n];
+		
+		dist[5*Np+n] = f6;
+		dist[11*Np+n] = f12;
+		dist[14*Np+n] = f13;
+		dist[15*Np+n] = f16;
+		dist[18*Np+n] = f17;
+	}
+}
 
 __global__  void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
 {
@@ -2340,10 +2376,11 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){
 	dvc_ScaLBL_D3Q19_Init<<<NBLOCKS,NTHREADS >>>(dist, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err));
 	}
 }
 
+
 extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){
 	dvc_ScaLBL_D3Q19_Swap<<<NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Nx, Ny, Nz);
 	cudaError_t err = cudaGetLastError();
@@ -2630,11 +2667,23 @@ extern "C" double deviceReduce(double *in, double* out, int N) {
 	return sum;
 }
 
-//
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np){
-//	int GRID = count / 512 + 1;
-//	dvc_ScaLBL_D3Q19_Pressure_BC_Z<<<GRID,512>>>(disteven, distodd, dout, Nx, Ny, Nz, outlet);
-//}
+extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q19_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q19_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
 
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
        double Fy, double Fz){

gpu/D3Q7.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -14,6 +15,7 @@
   along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 */
 // GPU Functions for D3Q7 Lattice Boltzmann Methods
+#include <stdio.h>
 
 #define NBLOCKS 560
 #define NTHREADS 128
@@ -94,6 +96,25 @@ __global__ void dvc_ScaLBL_D3Q7_Unpack(int q,  int *list,  int start, int count,
 	}
 }
 
+__global__  void dvc_ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f5 = 0.222222222222222222222222 - dist[6*Np+n];
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f6 = 0.222222222222222222222222 - dist[5*Np+n];
+		dist[5*Np+n] = f6;
+	}
+}
 __global__ void dvc_ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
 {
 	int n,N;
@@ -222,6 +243,24 @@ __global__  void dvc_ScaLBL_D3Q7_Density(char *ID, double *disteven, double *dis
 	}
 }
 
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
 extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list,  int start, int count, double *recvbuf, double *dist, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q7_Unpack <<<GRID,512 >>>(q, list, start, count, recvbuf, dist, N);

gpu/dfh.cu

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 */
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

models/ColorModel.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -19,6 +20,8 @@ color lattice boltzmann model
 #include "models/ColorModel.h"
 #include "analysis/distance.h"
 #include "analysis/morphology.h"
+#include "common/Communication.h"
+#include "common/ReadMicroCT.h"
 #include <stdlib.h>
 #include <time.h>
 
@@ -69,8 +72,6 @@ void ScaLBL_ColorModel::ReadCheckpoint(char *FILENAME, double *cPhi, double *cfq
     File.close();
 }
  */
-
-
 void ScaLBL_ColorModel::ReadParams(string filename){
 	// read the input database 
 	db = std::make_shared<Database>( filename );
@@ -133,7 +134,7 @@ void ScaLBL_ColorModel::ReadParams(string filename){
 	inletB=0.f;
 	outletA=0.f;
 	outletB=1.f;
-	if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details)
+	//if (BoundaryCondition==4) flux *= rhoA; // mass flux must adjust for density (see formulation for details)
 
 	BoundaryCondition = 0;
 	if (domain_db->keyExists( "BC" )){
@@ -143,21 +144,21 @@ void ScaLBL_ColorModel::ReadParams(string filename){
 	// Override user-specified boundary condition for specific protocols
 	auto protocol = color_db->getWithDefault<std::string>( "protocol", "none" );
 	if (protocol == "seed water"){
-		if (BoundaryCondition != 0 ){
+		if (BoundaryCondition != 0 && BoundaryCondition != 5){
 			BoundaryCondition = 0;
 			if (rank==0) printf("WARNING: protocol (seed water) supports only full periodic boundary condition \n");
 		}
 		domain_db->putScalar<int>( "BC", BoundaryCondition );
 	}
 	else if (protocol == "open connected oil"){
-		if (BoundaryCondition != 0 ){
+		if (BoundaryCondition != 0 && BoundaryCondition != 5){
 			BoundaryCondition = 0;
 			if (rank==0) printf("WARNING: protocol (open connected oil) supports only full periodic boundary condition \n");
 		}
 		domain_db->putScalar<int>( "BC", BoundaryCondition );
 	}
 	else if (protocol == "shell aggregation"){
-		if (BoundaryCondition != 0 ){
+		if (BoundaryCondition != 0 && BoundaryCondition != 5){
 			BoundaryCondition = 0;
 			if (rank==0) printf("WARNING: protocol (shell aggregation) supports only full periodic boundary condition \n");
 		}
@@ -199,11 +200,23 @@ void ScaLBL_ColorModel::ReadInput(){
 	if (color_db->keyExists( "image_sequence" )){
 		auto ImageList = color_db->getVector<std::string>( "image_sequence");
 		int IMAGE_INDEX = color_db->getWithDefault<int>( "image_index", 0 );
-		int IMAGE_COUNT = ImageList.size();
 		std::string first_image = ImageList[IMAGE_INDEX];
 		Mask->Decomp(first_image);
 		IMAGE_INDEX++;
 	}
+	else if (domain_db->keyExists( "GridFile" )){
+        // Read the local domain data
+	    auto input_id = readMicroCT( *domain_db, MPI_COMM_WORLD );
+        // Fill the halo (assuming GCW of 1)
+        array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
+        ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
+        ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
+        fillHalo<signed char> fill( MPI_COMM_WORLD, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
+        Array<signed char> id_view;
+        id_view.viewRaw( size1, Mask->id );
+        fill.copy( input_id, id_view );
+        fill.fill( id_view );
+	}
 	else if (domain_db->keyExists( "Filename" )){
 		auto Filename = domain_db->getScalar<std::string>( "Filename" );
 		Mask->Decomp(Filename);
@@ -216,7 +229,6 @@ void ScaLBL_ColorModel::ReadInput(){
 	// Generate the signed distance map
 	// Initialize the domain and communication
 	Array<char> id_solid(Nx,Ny,Nz);
-	int count = 0;
 	// Solve for the position of the solid phase
 	for (int k=0;k<Nz;k++){
 		for (int j=0;j<Ny;j++){
@@ -233,7 +245,6 @@ void ScaLBL_ColorModel::ReadInput(){
 	for (int k=0;k<Nz;k++){
 		for (int j=0;j<Ny;j++){
 			for (int i=0;i<Nx;i++){
-				int n=k*Nx*Ny+j*Nx+i;
 				// Initialize distance to +/- 1
 				Averages->SDs(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
 			}
@@ -266,7 +277,7 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
 	double label_count_global[NLABELS];
 	// Assign the labels
 
-	for (int idx=0; idx<NLABELS; idx++) label_count[idx]=0;
+	for (size_t idx=0; idx<NLABELS; idx++) label_count[idx]=0;
 
 	for (int k=0;k<Nz;k++){
 		for (int j=0;j<Ny;j++){
@@ -294,7 +305,8 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
 	// Set Dm to match Mask
 	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i]; 
 	
-	for (int idx=0; idx<NLABELS; idx++)		label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
+	for (size_t idx=0; idx<NLABELS; idx++)
+		label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
 
 	if (rank==0){
 		printf("Component labels: %lu \n",NLABELS);
@@ -373,16 +385,16 @@ void ScaLBL_ColorModel::Create(){
 	}
 	// check that TmpMap is valid
 	for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
-		int n = TmpMap[idx];
+		auto n = TmpMap[idx];
 		if (n > Nx*Ny*Nz){
-			printf("Bad value! idx=%i \n");
+			printf("Bad value! idx=%i \n", n);
 			TmpMap[idx] = Nx*Ny*Nz-1;
 		}
 	}
 	for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
-		int n = TmpMap[idx];
-		if (n > Nx*Ny*Nz){
-			printf("Bad value! idx=%i \n");
+		auto n = TmpMap[idx];
+		if ( n > Nx*Ny*Nz ){
+			printf("Bad value! idx=%i \n",n);
 			TmpMap[idx] = Nx*Ny*Nz-1;
 		}
 	}
@@ -474,7 +486,8 @@ void ScaLBL_ColorModel::Initialize(){
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
 
-	if (BoundaryCondition >0 ){
+	// establish reservoirs for external bC
+	if (BoundaryCondition == 1 || BoundaryCondition == 2 ||  BoundaryCondition == 3 || BoundaryCondition == 4 ){
 		if (Dm->kproc()==0){
 			ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
 			ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
@@ -497,6 +510,7 @@ void ScaLBL_ColorModel::Run(){
 	int IMAGE_COUNT = 0;
 	std::vector<std::string> ImageList;
 	bool SET_CAPILLARY_NUMBER = false;
+	bool RESCALE_FORCE = false;
 	bool MORPH_ADAPT = false;
 	bool USE_MORPH = false;
 	bool USE_SEED = false;
@@ -505,10 +519,11 @@ void ScaLBL_ColorModel::Run(){
 	int MAX_MORPH_TIMESTEPS = 50000; // maximum number of LBM timesteps to spend in morphological adaptation routine
 	int MIN_STEADY_TIMESTEPS = 100000;
 	int MAX_STEADY_TIMESTEPS = 200000;
+	int RESCALE_FORCE_AFTER_TIMESTEP = 0;
 	int RAMP_TIMESTEPS = 0;//50000;		 // number of timesteps to run initially (to get a reasonable velocity field before other pieces kick in)
 	int CURRENT_MORPH_TIMESTEPS=0;   // counter for number of timesteps spent in  morphological adaptation routine (reset each time)
 	int CURRENT_STEADY_TIMESTEPS=0;   // counter for number of timesteps spent in  morphological adaptation routine (reset each time)
-	int morph_interval = 1000000;
+	int morph_interval = 100000;
 	int analysis_interval = 1000; 	// number of timesteps in between in situ analysis 
 	int morph_timesteps = 0;
 	double morph_delta = 0.0;
@@ -519,10 +534,25 @@ void ScaLBL_ColorModel::Run(){
 	double initial_volume = 0.0;
 	double delta_volume = 0.0;
 	double delta_volume_target = 0.0;
-	double RESIDUAL_ENDPOINT_THRESHOLD = 0.04;
-	double NOISE_THRESHOLD = 1.0e-6;
-	double BUMP_RATE = 10.0;
 	
+	/* history for morphological algoirthm */
+	double KRA_MORPH_FACTOR=0.5;
+	double volA_prev = 0.0; 
+	double log_krA_prev = 1.0;
+	double log_krA_target = 1.0;
+	double log_krA = 1.0;
+	double slope_krA_volume = 0.0;
+	if (color_db->keyExists( "vol_A_previous" )){
+		volA_prev  = color_db->getScalar<double>( "vol_A_previous" );
+	}
+	if (color_db->keyExists( "log_krA_previous" )){
+		log_krA_prev  = color_db->getScalar<double>( "log_krA_previous" );
+	}
+	if (color_db->keyExists( "krA_morph_factor" )){
+		KRA_MORPH_FACTOR  = color_db->getScalar<double>( "krA_morph_factor" );
+	}
+	
+	/* defaults for simulation protocols */
 	auto protocol = color_db->getWithDefault<std::string>( "protocol", "none" );
 	if (protocol == "image sequence"){
 		// Get the list of images
@@ -534,39 +564,33 @@ void ScaLBL_ColorModel::Run(){
 		USE_MORPH = true;
 	}
 	else if (protocol == "seed water"){
-		morph_delta = 0.05;
+		morph_delta = -0.05;
 		seed_water = 0.01;
 		USE_SEED = true;
 		USE_MORPH = true;
 	}
 	else if (protocol == "open connected oil"){
-		morph_delta = 0.05;
+		morph_delta = -0.05;
 		USE_MORPH = true;
 		USE_MORPHOPEN_OIL = true;
 	}
 	else if (protocol == "shell aggregation"){
-		morph_delta = 0.05;
+		morph_delta = -0.05;
 		USE_MORPH = true;
 	}  
-	
-	if (color_db->keyExists( "residual_endpoint_threshold" )){
-		RESIDUAL_ENDPOINT_THRESHOLD  = color_db->getScalar<double>( "residual_endpoint_threshold" );
-	}
-	if (color_db->keyExists( "noise_threshold" )){
-		NOISE_THRESHOLD  = color_db->getScalar<double>( "noise_threshold" );
-	}
-	if (color_db->keyExists( "bump_rate" )){
-		BUMP_RATE  = color_db->getScalar<double>( "bump_rate" );
-	}
 	if (color_db->keyExists( "capillary_number" )){
 		capillary_number = color_db->getScalar<double>( "capillary_number" );
 		SET_CAPILLARY_NUMBER=true;
 	}
+	if (color_db->keyExists( "rescale_force_after_timestep" )){
+		RESCALE_FORCE_AFTER_TIMESTEP = color_db->getScalar<int>( "rescale_force_after_timestep" );
+		RESCALE_FORCE = true;
+	}
 	if (color_db->keyExists( "timestep" )){
 		timestep = color_db->getScalar<int>( "timestep" );
 	}
-	if (BoundaryCondition != 0 && SET_CAPILLARY_NUMBER==true){
-		if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 \n");
+	if (BoundaryCondition != 0 && BoundaryCondition != 5 && SET_CAPILLARY_NUMBER==true){
+		if (rank == 0) printf("WARINING: capillary number target only supported for BC = 0 or 5 \n");
 		SET_CAPILLARY_NUMBER=false;
 	}
 	if (analysis_db->keyExists( "seed_water" )){
@@ -603,6 +627,7 @@ void ScaLBL_ColorModel::Run(){
 		MAX_MORPH_TIMESTEPS = analysis_db->getScalar<int>( "max_morph_timesteps" );
 	}
 
+
 	if (rank==0){
 		printf("********************************************************\n");
 		if (protocol == "image sequence"){
@@ -668,7 +693,7 @@ void ScaLBL_ColorModel::Run(){
 
 		// Perform the collision operation
 		ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
-		if (BoundaryCondition > 0){
+		if (BoundaryCondition > 0 && BoundaryCondition < 5){
 			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
 			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
 		}
@@ -689,9 +714,14 @@ void ScaLBL_ColorModel::Run(){
 			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
 			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
 		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
 		ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
 				alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); 
+		MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 
 		// *************EVEN TIMESTEP*************
 		timestep++;
@@ -705,7 +735,7 @@ void ScaLBL_ColorModel::Run(){
 		// Perform the collision operation
 		ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
 		// Halo exchange for phase field
-		if (BoundaryCondition > 0){
+		if (BoundaryCondition > 0 && BoundaryCondition < 5){
 			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
 			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
 		}
@@ -724,21 +754,23 @@ void ScaLBL_ColorModel::Run(){
 			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
 			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
 		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
 		ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
 				alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
-		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		ScaLBL_DeviceBarrier(); 
+		MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 		//************************************************************************
-		
-		MPI_Barrier(comm);
 		PROFILE_STOP("Update");
 
-		if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition > 0){
+		if (rank==0 && timestep%analysis_interval == 0 && BoundaryCondition == 4){
 			printf("%i %f \n",timestep,din);
 		}
 		// Run the analysis
 		analysis.basic(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den );
 
-		
 		// allow initial ramp-up to get closer to steady state
 		if (timestep > RAMP_TIMESTEPS && timestep%analysis_interval == 0 && USE_MORPH){
 			analysis.finish();
@@ -748,7 +780,7 @@ void ScaLBL_ColorModel::Run(){
 			double volA = Averages->gnb.V; 
 			volA /= Dm->Volume;
 			volB /= Dm->Volume;;
-			initial_volume = volA*Dm->Volume;
+			//initial_volume = volA*Dm->Volume;
 			double vA_x = Averages->gnb.Px/Averages->gnb.M; 
 			double vA_y = Averages->gnb.Py/Averages->gnb.M; 
 			double vA_z = Averages->gnb.Pz/Averages->gnb.M; 
@@ -780,11 +812,50 @@ void ScaLBL_ColorModel::Run(){
 					isSteady = true;
 				if (CURRENT_STEADY_TIMESTEPS > MAX_STEADY_TIMESTEPS)
 					isSteady = true;
-
+				if (RESCALE_FORCE == true && SET_CAPILLARY_NUMBER == true && CURRENT_STEADY_TIMESTEPS > RESCALE_FORCE_AFTER_TIMESTEP){
+				  RESCALE_FORCE = false;
+				  double RESCALE_FORCE_FACTOR = capillary_number / Ca;
+				  if (RESCALE_FORCE_FACTOR > 2.0) RESCALE_FORCE_FACTOR = 2.0;
+				  if (RESCALE_FORCE_FACTOR < 0.5) RESCALE_FORCE_FACTOR = 0.5;
+				  Fx *= RESCALE_FORCE_FACTOR;
+				  Fy *= RESCALE_FORCE_FACTOR;
+				  Fz *= RESCALE_FORCE_FACTOR;
+				  force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+				  if (force_mag > 1e-3){
+				    Fx *= 1e-3/force_mag;   // impose ceiling for stability
+				    Fy *= 1e-3/force_mag;   
+				    Fz *= 1e-3/force_mag;   
+				  }
+				  if (rank == 0) printf("    -- adjust force by factor %f \n ",capillary_number / Ca);
+				  Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
+				  color_db->putVector<double>("F",{Fx,Fy,Fz});
+				}
 				if ( isSteady ){
 					MORPH_ADAPT = true;
 					CURRENT_MORPH_TIMESTEPS=0;
 					delta_volume_target = Dm->Volume*volA *morph_delta; // set target volume change
+					//****** ENDPOINT ADAPTATION ********/
+					double krA_TMP= fabs(muA*flow_rate_A / force_mag);
+					double krB_TMP= fabs(muB*flow_rate_B / force_mag);
+					log_krA = log(krA_TMP);
+					if (krA_TMP < 0.0){
+						// cannot do endpoint adaptation if kr is negative
+						log_krA = log_krA_prev;
+					}
+					else if (krA_TMP < krB_TMP && morph_delta > 0.0){
+						/** morphological target based on relative permeability for A **/
+						log_krA_target = log(KRA_MORPH_FACTOR*(krA_TMP));
+						slope_krA_volume = (log_krA - log_krA_prev)/(Dm->Volume*(volA - volA_prev));
+						delta_volume_target=min(delta_volume_target,Dm->Volume*(volA+(log_krA_target - log_krA)/slope_krA_volume));
+						if (rank==0){
+							printf("    Enabling endpoint adaptation: krA = %f, krB = %f \n",krA_TMP,krB_TMP);	
+							printf("    log(kr)=%f, volume=%f, TARGET log(kr)=%f, volume change=%f \n",log_krA, volA, log_krA_target, delta_volume_target/(volA*Dm->Volume));							
+						}
+					}
+					log_krA_prev = log_krA;
+					volA_prev = volA;
+					//******************************** **/
+					/**  compute averages & write data **/
 					Averages->Full();
 					Averages->Write(timestep);
 					analysis.WriteVisData(timestep, current_db, *Averages, Phi, Pressure, Velocity, fq, Den );
@@ -799,8 +870,8 @@ void ScaLBL_ColorModel::Run(){
 						double pB = Averages->gwb.p;
 						double pAc = Averages->gnc.p;
 						double pBc = Averages->gwc.p;
-						double pAB = (pA-pB)/(h*5.796*alpha);
-						double pAB_connected = (pAc-pBc)/(h*5.796*alpha);
+						double pAB = (pA-pB)/(h*6.0*alpha);
+						double pAB_connected = (pAc-pBc)/(h*6.0*alpha);
 						// connected contribution
 						double Vol_nc = Averages->gnc.V/Dm->Volume;
 						double Vol_wc = Averages->gwc.V/Dm->Volume;
@@ -847,7 +918,7 @@ void ScaLBL_ColorModel::Run(){
 							WriteHeader=true;
 						kr_log_file = fopen("relperm.csv","a");
 						if (WriteHeader)
-							fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,pAB,viscous_pressure_drop,Ca,Mobility);
+							fprintf(kr_log_file,"timesteps sat.water eff.perm.oil eff.perm.water eff.perm.oil.connected eff.perm.water.connected eff.perm.oil.disconnected eff.perm.water.disconnected cap.pressure cap.pressure.connected pressure.drop Ca M\n");
 
 						fprintf(kr_log_file,"%i %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",CURRENT_STEADY_TIMESTEPS,current_saturation,kAeff,kBeff,kAeff_connected,kBeff_connected,kAeff_disconnected,kBeff_disconnected,pAB,pAB_connected,viscous_pressure_drop,Ca,Mobility);
 						fclose(kr_log_file);
@@ -858,26 +929,16 @@ void ScaLBL_ColorModel::Run(){
 						Fx *= capillary_number / Ca;
 						Fy *= capillary_number / Ca;
 						Fz *= capillary_number / Ca;
-
 						if (force_mag > 1e-3){
 							Fx *= 1e-3/force_mag;   // impose ceiling for stability
 							Fy *= 1e-3/force_mag;   
 							Fz *= 1e-3/force_mag;   
 						}
-						if (flow_rate_A < NOISE_THRESHOLD){
-							if (rank==0) printf("Hit noise threshold (%f): bumping capillary number by %f X \n",NOISE_THRESHOLD,BUMP_RATE);
-							Fx *= BUMP_RATE;   // impose bump condition
-							Fy *= BUMP_RATE;   
-							Fz *= BUMP_RATE;   
-							capillary_number *= BUMP_RATE;
-							color_db->putScalar<int>("capillary_number",capillary_number);
-							current_db->putDatabase("Color", color_db);
-							MORPH_ADAPT = false; // re-run current point if below noise threshold
-						}
 						if (rank == 0) printf("    -- adjust force by factor %f \n ",capillary_number / Ca);
 						Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
 						color_db->putVector<double>("F",{Fx,Fy,Fz});
 					}
+					
 					CURRENT_STEADY_TIMESTEPS = 0;
 				}
 				else{
@@ -911,7 +972,7 @@ void ScaLBL_ColorModel::Run(){
 					delta_volume = volA*Dm->Volume - initial_volume;
 					CURRENT_MORPH_TIMESTEPS += analysis_interval;
 					double massChange = SeedPhaseField(seed_water);
-					if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", seed_water, delta_volume, delta_volume_target);
+					if (rank==0) printf("***Seed water in oil %f, volume change %f / %f ***\n", massChange, delta_volume, delta_volume_target);
 				}
 				else if (USE_MORPHOPEN_OIL){
 					delta_volume = volA*Dm->Volume - initial_volume;
@@ -929,41 +990,29 @@ void ScaLBL_ColorModel::Run(){
 					CURRENT_STEADY_TIMESTEPS=0;
 					initial_volume = volA*Dm->Volume;
 					delta_volume = 0.0;
-					if (USE_DIRECT){
-						//BoundaryCondition = 0;
-						//ScaLBL_Comm->BoundaryCondition = 0;
-						//ScaLBL_Comm_Regular->BoundaryCondition = 0;
-						//Fx = capillary_number*dir_x*force_mag / Ca;
-						//Fy = capillary_number*dir_y*force_mag / Ca;
-						//Fz = capillary_number*dir_z*force_mag / Ca;
-					}
+					if (RESCALE_FORCE_AFTER_TIMESTEP > 0)
+						RESCALE_FORCE = true;
 				}
 				else if (!(USE_DIRECT) && CURRENT_MORPH_TIMESTEPS > MAX_MORPH_TIMESTEPS) {
 					MORPH_ADAPT = false;
 					CURRENT_STEADY_TIMESTEPS=0;
 					initial_volume = volA*Dm->Volume;
 					delta_volume = 0.0;
+					RESCALE_FORCE = true;
+					if (RESCALE_FORCE_AFTER_TIMESTEP > 0)
+						RESCALE_FORCE = true;
 				}
-				if ( REVERSE_FLOW_DIRECTION ){
-					//if (rank==0) printf("*****REVERSE FLOW DIRECTION***** \n");
-					delta_volume = 0.0;
-					// flow direction will reverse after next steady point
-					MORPH_ADAPT = false;
-					CURRENT_STEADY_TIMESTEPS=0;
-					//morph_delta *= (-1.0);
-					REVERSE_FLOW_DIRECTION = false;
-				}
-				MPI_Barrier(comm);
 			}
 			morph_timesteps += analysis_interval;
 		}
+		MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 	}
 	analysis.finish();
 	PROFILE_STOP("Loop");
 	PROFILE_SAVE("lbpm_color_simulator",1);
 	//************************************************************************
 	ScaLBL_DeviceBarrier();
-	MPI_Barrier(comm);
+	MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 	stoptime = MPI_Wtime();
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
@@ -983,7 +1032,6 @@ void ScaLBL_ColorModel::Run(){
 
 double ScaLBL_ColorModel::ImageInit(std::string Filename){
 	
-	bool suppress = false;
 	if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str());
 	Mask->Decomp(Filename);
 	for (int i=0; i<Nx*Ny*Nz; i++) id[i] = Mask->id[i];  // save what was read
@@ -1014,12 +1062,12 @@ double ScaLBL_ColorModel::ImageInit(std::string Filename){
 	
 	if (rank==0) printf("   new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount);
 	ScaLBL_CopyToDevice(Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double));
-	MPI_Barrier(comm);
+	MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 	
 	ScaLBL_D3Q19_Init(fq, Np);
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
-	MPI_Barrier(comm);
+	MPI_Barrier(ScaLBL_Comm->MPI_COMM_SCALBL);
 	
 	ScaLBL_CopyToHost(Averages->Phi.data(),Phi,Nx*Ny*Nz*sizeof(double));
 
@@ -1053,10 +1101,9 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
 		ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm);
 		MPI_Barrier(Dm->Comm);
 
-		int count_oil=0;
-		int count_connected=0;
-		int count_porespace=0;
-		int count_water=0;
+		long long count_connected=0;
+		long long count_porespace=0;
+		long long count_water=0;
 		for (int k=1; k<nz-1; k++){
 			for (int j=1; j<ny-1; j++){
 				for (int i=1; i<nx-1; i++){
@@ -1156,7 +1203,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
 		ScaLBL_CopyToDevice(Phi,phase.data(),N*sizeof(double));
 		ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
 		ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
-		if (BoundaryCondition >0 ){
+		if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){
 			if (Dm->kproc()==0){
 				ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
 				ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
@@ -1171,99 +1218,79 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
 	}
 	return(volume_change);
 }
-
 double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){
-	srand(time(NULL));
-	double mass_loss =0.f;
-	double count =0.f;
-	double *Aq_tmp, *Bq_tmp;
-	
-	Aq_tmp = new double [7*Np];
-	Bq_tmp = new double [7*Np];
+  srand(time(NULL));
+  double mass_loss =0.f;
+  double count =0.f;
+  double *Aq_tmp, *Bq_tmp;
+  
+  Aq_tmp = new double [7*Np];
+  Bq_tmp = new double [7*Np];
 
-	ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double));
-	ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double));
-	
-/*	for (int k=1; k<Nz-1; k++){
-		for (int j=1; j<Ny-1; j++){
-			for (int i=1; i<Nx-1; i++){
-				double random_value = double(rand())/ RAND_MAX;
+  ScaLBL_CopyToHost(Aq_tmp, Aq, 7*Np*sizeof(double));
+  ScaLBL_CopyToHost(Bq_tmp, Bq, 7*Np*sizeof(double));
+  
+ 
+   for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){
+    double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
+    double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
+    double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
+    double phase_id = (dA - dB) / (dA + dB);
+    if (phase_id > 0.0){
+      Aq_tmp[n] -= 0.3333333333333333*random_value;
+      Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
+      
+      Bq_tmp[n] += 0.3333333333333333*random_value;
+      Bq_tmp[n+Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
+    }
+    mass_loss += random_value*seed_water_in_oil;
+  }
 
-				if (Averages->SDs(i,j,k) < 0.f){
-					// skip
-				}
-				else if (phase(i,j,k) > 0.f ){
-					phase(i,j,k) -= random_value*seed_water_in_oil;
-					mass_loss += random_value*seed_water_in_oil;
-					count++;
-				}
-				else {
+  for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){
+    double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
+    double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
+    double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
+    double phase_id = (dA - dB) / (dA + dB);
+    if (phase_id > 0.0){
+      Aq_tmp[n] -= 0.3333333333333333*random_value;
+      Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
+      
+      Bq_tmp[n] += 0.3333333333333333*random_value;
+      Bq_tmp[n+Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
+    }
+    mass_loss += random_value*seed_water_in_oil;
+  }
 
-				}
-			}
-		}
-	}
-	*/
-	for (int n=0; n < ScaLBL_Comm->LastExterior(); n++){
-		double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
-		double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
-		double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
-		double phase_id = (dA - dB) / (dA + dB);
-		if (phase_id > 0.0){
-			Aq_tmp[n] -= 0.3333333333333333*random_value;
-			Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
-			
-			Bq_tmp[n] += 0.3333333333333333*random_value;
-			Bq_tmp[n+Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
-		}
-		mass_loss += random_value*seed_water_in_oil;
-	}
+  count= sumReduce( Dm->Comm, count);
+  mass_loss= sumReduce( Dm->Comm, mass_loss);
+  if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count);
 
-	for (int n=ScaLBL_Comm->FirstInterior(); n < ScaLBL_Comm->LastInterior(); n++){
-		double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
-		double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
-		double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
-		double phase_id = (dA - dB) / (dA + dB);
-		if (phase_id > 0.0){
-			Aq_tmp[n] -= 0.3333333333333333*random_value;
-			Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
-			Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
-			
-			Bq_tmp[n] += 0.3333333333333333*random_value;
-			Bq_tmp[n+Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
-			Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
-		}
-		mass_loss += random_value*seed_water_in_oil;
-	}
+  // Need to initialize Aq, Bq, Den, Phi directly
+  //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double));
+  ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double));
+  ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double));
 
-	count= sumReduce( Dm->Comm, count);
-	mass_loss= sumReduce( Dm->Comm, mass_loss);
-	if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count);
-
-	// Need to initialize Aq, Bq, Den, Phi directly
-	//ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double));
-	ScaLBL_CopyToDevice(Aq, Aq_tmp, 7*Np*sizeof(double));
-	  ScaLBL_CopyToDevice(Bq, Bq_tmp, 7*Np*sizeof(double));
-
-	return(mass_loss);
+  return(mass_loss);
 }
 
 double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta_volume){
@@ -1272,6 +1299,8 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 	double vF = 0.f;
 	double vS = 0.f;
 	double delta_volume;
+	double WallFactor = 0.0;
+	bool USE_CONNECTED_NWP = false;
 
 	DoubleArray phase(Nx,Ny,Nz);
 	IntArray phase_label(Nx,Ny,Nz);;
@@ -1284,8 +1313,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 	// 1. Copy phase field to CPU
 	ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double));
 
-	double count,count_global,volume_initial,volume_final,volume_connected;
-	count = 0.f;
+	double count = 0.f;
 	for (int k=1; k<Nz-1; k++){
 		for (int j=1; j<Ny-1; j++){
 			for (int i=1; i<Nx-1; i++){
@@ -1293,7 +1321,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 			}
 		}
 	}
-	volume_initial = sumReduce( Dm->Comm, count);
+	double volume_initial = sumReduce( Dm->Comm, count);
 	/*
 	sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank);
 	FILE *INPUT = fopen(LocalRankFilename,"wb");
@@ -1301,41 +1329,65 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 	fclose(INPUT);
 	*/
 	// 2. Identify connected components of phase field -> phase_label
-	BlobIDstruct new_index;
-	ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm);
-	MPI_Barrier(comm);
 	
-	// only operate on component "0"
-	count = 0.0;
-	double second_biggest = 0.0;
+	double volume_connected = 0.0;
+       	double second_biggest = 0.0;
+	if (USE_CONNECTED_NWP){
+		BlobIDstruct new_index;
+		ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm);
+		MPI_Barrier(Dm->Comm);
 
-	for (int k=0; k<Nz; k++){
-		for (int j=0; j<Ny; j++){
-			for (int i=0; i<Nx; i++){
-				int label = phase_label(i,j,k);
-				if (label == 0 ){
-					phase_id(i,j,k) = 0;
-					count += 1.0;
-				}
-				else 		
-					phase_id(i,j,k) = 1;
-				if (label == 1 ){
-					second_biggest += 1.0;
+		// only operate on component "0"
+		count = 0.0;
+
+		for (int k=0; k<Nz; k++){
+			for (int j=0; j<Ny; j++){
+				for (int i=0; i<Nx; i++){
+					int label = phase_label(i,j,k);
+					if (label == 0 ){
+						phase_id(i,j,k) = 0;
+						count += 1.0;
+					}
+					else 		
+						phase_id(i,j,k) = 1;
+					if (label == 1 ){
+						second_biggest += 1.0;
+					}
 				}
 			}
-		}
-	}	
-	volume_connected = sumReduce( Dm->Comm, count);
-	second_biggest = sumReduce( Dm->Comm, second_biggest);
-
-	int reach_x, reach_y, reach_z;
-	for (int k=0; k<Nz; k++){
-		for (int j=0; j<Ny; j++){
-			for (int i=0; i<Nx; i++){
+		}	
+		volume_connected = sumReduce( Dm->Comm, count);
+		second_biggest = sumReduce( Dm->Comm, second_biggest);
+	}
+	else {
+		// use the whole NWP 
+		for (int k=0; k<Nz; k++){
+			for (int j=0; j<Ny; j++){
+				for (int i=0; i<Nx; i++){
+					if (Averages->SDs(i,j,k) > 0.f){
+						if (phase(i,j,k) > 0.f ){
+							phase_id(i,j,k) = 0;
+						}
+						else {
+							phase_id(i,j,k) = 1;
+						}
+					}
+					else {
+						phase_id(i,j,k) = 1;
+					}
+				}
 			}
 		}
 	}
 
+	/*int reach_x, reach_y, reach_z;
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+			}
+		}
+	}*/
+
 	// 3. Generate a distance map to the largest object -> phase_distance
 	CalcDist(phase_distance,phase_id,*Dm);
 
@@ -1361,18 +1413,21 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 		}
 	}
 
+	if (USE_CONNECTED_NWP){
 	if (volume_connected - second_biggest < 2.0*fabs(target_delta_volume) && target_delta_volume < 0.0){
 		// if connected volume is less than 2% just delete the whole thing
 		if (rank==0) printf("Connected region has shrunk! \n");
 		REVERSE_FLOW_DIRECTION = true;
 	}
+	
 /*	else{*/
 		if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest );
+	}
 		if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial);
 		double target_delta_volume_incremental = target_delta_volume;
 		if (fabs(target_delta_volume) > 0.01*volume_initial)  
 			target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume);
-		delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental);
+		delta_volume = MorphGrow(Averages->SDs,phase_distance,phase_id,Averages->Dm, target_delta_volume_incremental, WallFactor);
 
 		for (int k=0; k<Nz; k++){
 			for (int j=0; j<Ny; j++){
@@ -1390,7 +1445,6 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 		for (int k=0; k<Nz; k++){
 			for (int j=0; j<Ny; j++){
 				for (int i=0; i<Nx; i++){
-					int n = k*Nx*Ny + j*Nx + i;
 					double d = phase_distance(i,j,k);
 					if (Averages->SDs(i,j,k) > 0.f){
 						if (d < 3.f){
@@ -1414,7 +1468,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 			}
 		}
 	}
-	volume_final= sumReduce( Dm->Comm, count);
+	double volume_final= sumReduce( Dm->Comm, count);
 
 	delta_volume = (volume_final-volume_initial);
 	if (rank == 0)  printf("MorphInit: change fluid volume fraction by %f \n", delta_volume/volume_initial);
@@ -1437,7 +1491,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 	// 7. Re-initialize phase field and density
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
 	ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
-	if (BoundaryCondition >0 ){
+	if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){
 		if (Dm->kproc()==0){
 			ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
 			ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
@@ -1463,4 +1517,68 @@ void ScaLBL_ColorModel::WriteDebug(){
 	OUTFILE = fopen(LocalRankFilename,"wb");
 	fwrite(PhaseField.data(),8,N,OUTFILE);
 	fclose(OUTFILE);
+
+    ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField);
+	FILE *AFILE;
+	sprintf(LocalRankFilename,"A.%05i.raw",rank);
+	AFILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,AFILE);
+	fclose(AFILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField);
+	FILE *BFILE;
+	sprintf(LocalRankFilename,"B.%05i.raw",rank);
+	BFILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,BFILE);
+	fclose(BFILE);
+
+	ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField);
+	FILE *PFILE;
+	sprintf(LocalRankFilename,"Pressure.%05i.raw",rank);
+	PFILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,PFILE);
+	fclose(PFILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField);
+	FILE *VELX_FILE;
+	sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank);
+	VELX_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELX_FILE);
+	fclose(VELX_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField);
+	FILE *VELY_FILE;
+	sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank);
+	VELY_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELY_FILE);
+	fclose(VELY_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField);
+	FILE *VELZ_FILE;
+	sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank);
+	VELZ_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELZ_FILE);
+	fclose(VELZ_FILE);
+
+/*	ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField);
+	FILE *CGX_FILE;
+	sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank);
+	CGX_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,CGX_FILE);
+	fclose(CGX_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&ColorGrad[Np],PhaseField);
+	FILE *CGY_FILE;
+	sprintf(LocalRankFilename,"Gradient_Y.%05i.raw",rank);
+	CGY_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,CGY_FILE);
+	fclose(CGY_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&ColorGrad[2*Np],PhaseField);
+	FILE *CGZ_FILE;
+	sprintf(LocalRankFilename,"Gradient_Z.%05i.raw",rank);
+	CGZ_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,CGZ_FILE);
+	fclose(CGZ_FILE);
+*/
 }

models/ColorModel.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify

models/DFHModel.cpp

@@ -114,7 +114,6 @@ void ScaLBL_DFHModel::SetDomain(){
 }
 
 void ScaLBL_DFHModel::ReadInput(){
-	size_t readID;
 	//.......................................................................
 	if (rank == 0)    printf("Read input media... \n");
 	//.......................................................................

models/GreyscaleColorModel.h

@@ -0,0 +1,94 @@
+/*
+Implementation of two-fluid greyscale color lattice boltzmann model
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+
+#include "common/Communication.h"
+#include "analysis/GreyPhase.h"
+#include "common/MPI_Helpers.h"
+#include "ProfilerApp.h"
+#include "threadpool/thread_pool.h"
+
+class ScaLBL_GreyscaleColorModel{
+public:
+	ScaLBL_GreyscaleColorModel(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_GreyscaleColorModel();	
+	
+	// functions in they should be run
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+	void SetDomain();
+	void ReadInput();
+	void Create();
+	void Initialize();
+	void Run();
+	void WriteDebug();
+	
+	bool Restart,pBC;
+	bool REVERSE_FLOW_DIRECTION;
+	int timestep,timestepMax;
+	int BoundaryCondition;
+	double tauA,tauB,rhoA,rhoB,alpha,beta;
+    double tauA_eff,tauB_eff;
+	double Fx,Fy,Fz,flux;
+	double din,dout,inletA,inletB,outletA,outletB;
+    double GreyPorosity;
+	
+	int Nx,Ny,Nz,N,Np;
+	int rank,nprocx,nprocy,nprocz,nprocs;
+	double Lx,Ly,Lz;
+
+	std::shared_ptr<Domain> Dm;   // this domain is for analysis
+	std::shared_ptr<Domain> Mask; // this domain is for lbm
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm_Regular;
+        std::shared_ptr<GreyPhaseAnalysis> Averages;
+    
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> domain_db;
+    std::shared_ptr<Database> greyscaleColor_db;
+    std::shared_ptr<Database> analysis_db;
+    std::shared_ptr<Database> vis_db;
+
+    IntArray Map;
+    signed char *id;    
+	int *NeighborList;
+	int *dvcMap;
+	double *fq, *Aq, *Bq;
+	double *Den, *Phi;
+    //double *GreySolidPhi; //Model 2 & 3
+    double *GreySolidGrad;//Model 1 & 4
+	//double *ColorGrad;
+	double *Velocity;
+	double *Pressure;
+    double *Porosity_dvc;
+    double *Permeability_dvc;
+		
+private:
+	MPI_Comm comm;
+    
+	int dist_mem_size;
+	int neighborSize;
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+   
+    //int rank,nprocs;
+    void LoadParams(std::shared_ptr<Database> db0);
+    void AssignComponentLabels();
+    void AssignGreySolidLabels();
+    void AssignGreyPoroPermLabels();
+    void ImageInit(std::string filename);
+    double MorphInit(const double beta, const double morph_delta);
+    double SeedPhaseField(const double seed_water_in_oil);
+    double MorphOpenConnected(double target_volume_change);
+};
+

models/GreyscaleModel.cpp

@@ -0,0 +1,959 @@
+/*
+  Copyright 2020 Equinor ASA
+  Copyright Equnior ASA
+
+  This file is part of the Open Porous Media project (OPM).
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+
+  Greyscale lattice boltzmann model
+ */
+#include "models/GreyscaleModel.h"
+#include "analysis/distance.h"
+#include "analysis/morphology.h"
+#include <stdlib.h>
+#include <time.h>
+
+template<class TYPE>
+void DeleteArray( const TYPE *p )
+{
+    delete [] p;
+}
+
+ScaLBL_GreyscaleModel::ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM):
+rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),tau_eff(0),Den(0),Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),GreyPorosity(0),
+Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
+{
+	SignDist.resize(Nx,Ny,Nz);           
+    SignDist.fill(0);
+
+}
+ScaLBL_GreyscaleModel::~ScaLBL_GreyscaleModel(){
+
+}
+
+void ScaLBL_GreyscaleModel::ReadParams(string filename){
+	// read the input database 
+	db = std::make_shared<Database>( filename );
+	domain_db = db->getDatabase( "Domain" );
+	greyscale_db =  db->getDatabase( "Greyscale" );
+	analysis_db = db->getDatabase( "Analysis" );
+	vis_db = db->getDatabase( "Visualization" );
+
+	// set defaults
+	timestepMax = 100000;
+	tau = 1.0;
+    tau_eff = tau;
+    Den = 1.0;//constant density
+	tolerance = 0.01;
+	Fx = Fy = Fz = 0.0;
+	Restart=false;
+	din=dout=1.0;
+	flux=0.0;
+    dp = 10.0; //unit of 'dp': voxel
+    CollisionType = 1; //1: IMRT; 2: BGK; 3: MRT
+	
+	// ---------------------- Greyscale Model parameters -----------------------//
+	if (greyscale_db->keyExists( "timestepMax" )){
+		timestepMax = greyscale_db->getScalar<int>( "timestepMax" );
+	}
+	if (greyscale_db->keyExists( "tau" )){
+		tau = greyscale_db->getScalar<double>( "tau" );
+	}
+	tau_eff = greyscale_db->getWithDefault<double>( "tau_eff", tau );
+	if (greyscale_db->keyExists( "Den" )){
+		Den = greyscale_db->getScalar<double>( "Den" );
+	}
+	if (greyscale_db->keyExists( "dp" )){
+		dp = greyscale_db->getScalar<double>( "dp" );
+	}
+	if (greyscale_db->keyExists( "F" )){
+		Fx = greyscale_db->getVector<double>( "F" )[0];
+		Fy = greyscale_db->getVector<double>( "F" )[1];
+		Fz = greyscale_db->getVector<double>( "F" )[2];
+	}
+	if (greyscale_db->keyExists( "Restart" )){
+		Restart = greyscale_db->getScalar<bool>( "Restart" );
+	}
+	if (greyscale_db->keyExists( "din" )){
+		din = greyscale_db->getScalar<double>( "din" );
+	}
+	if (greyscale_db->keyExists( "dout" )){
+		dout = greyscale_db->getScalar<double>( "dout" );
+	}
+	if (greyscale_db->keyExists( "flux" )){
+		flux = greyscale_db->getScalar<double>( "flux" );
+	}
+	if (greyscale_db->keyExists( "tolerance" )){
+		tolerance = greyscale_db->getScalar<double>( "tolerance" );
+	}
+	auto collision = greyscale_db->getWithDefault<std::string>( "collision", "IMRT" );
+	if (collision == "BGK"){
+        CollisionType=2;
+	}
+    else if (collision == "MRT"){
+        CollisionType=3;
+    }
+	// ------------------------------------------------------------------------//
+    
+    //------------------------ Other Domain parameters ------------------------//
+	BoundaryCondition = 0;
+	if (domain_db->keyExists( "BC" )){
+		BoundaryCondition = domain_db->getScalar<int>( "BC" );
+	}
+	// ------------------------------------------------------------------------//
+}
+
+void ScaLBL_GreyscaleModel::SetDomain(){
+	Dm  = std::shared_ptr<Domain>(new Domain(domain_db,comm));      // full domain for analysis
+	Mask  = std::shared_ptr<Domain>(new Domain(domain_db,comm));    // mask domain removes immobile phases
+	// domain parameters
+	Nx = Dm->Nx;
+	Ny = Dm->Ny;
+	Nz = Dm->Nz;
+	Lx = Dm->Lx;
+	Ly = Dm->Ly;
+	Lz = Dm->Lz;
+	N = Nx*Ny*Nz;
+
+	SignDist.resize(Nx,Ny,Nz);
+	Velocity_x.resize(Nx,Ny,Nz);
+	Velocity_y.resize(Nx,Ny,Nz);
+	Velocity_z.resize(Nx,Ny,Nz);
+	PorosityMap.resize(Nx,Ny,Nz);
+	Pressure.resize(Nx,Ny,Nz);
+
+	id = new signed char [N];
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1;               // initialize this way
+	MPI_Barrier(comm);
+	Dm->CommInit();
+	MPI_Barrier(comm);
+	// Read domain parameters
+	rank = Dm->rank();	
+	nprocx = Dm->nprocx();
+	nprocy = Dm->nprocy();
+	nprocz = Dm->nprocz();
+}
+
+void ScaLBL_GreyscaleModel::ReadInput(){
+	
+	sprintf(LocalRankString,"%05d",rank);
+	sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
+	sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
+	
+	if (domain_db->keyExists( "Filename" )){
+		auto Filename = domain_db->getScalar<std::string>( "Filename" );
+		Mask->Decomp(Filename);
+	}
+	else{
+        if (rank==0) printf("Filename of input image is not found, reading ID.0* instead.");
+		Mask->ReadIDs();
+	}
+	for (int i=0; i<Nx*Ny*Nz; i++) id[i] = Mask->id[i];  // save what was read
+	
+	// Generate the signed distance map
+	// Initialize the domain and communication
+	Array<char> id_solid(Nx,Ny,Nz);
+	int count = 0;
+	// Solve for the position of the solid phase
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				// Initialize the solid phase
+				signed char label = Mask->id[n];
+				if (label > 0)		id_solid(i,j,k) = 1;
+				else	     		id_solid(i,j,k) = 0;
+			}
+		}
+	}
+	// Initialize the signed distance function
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n=k*Nx*Ny+j*Nx+i;
+				// Initialize distance to +/- 1
+				SignDist(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
+			}
+		}
+	}
+//	MeanFilter(SignDist);
+	if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
+	CalcDist(SignDist,id_solid,*Mask);
+	
+	if (rank == 0) cout << "Domain set." << endl;
+}
+
+/********************************************************
+ * AssignComponentLabels                                 *
+ ********************************************************/
+void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity, double *Permeability)
+{
+	size_t NLABELS=0;
+	signed char VALUE=0;
+	double POROSITY=0.f;
+	double PERMEABILITY=0.f;
+
+	auto LabelList = greyscale_db->getVector<int>( "ComponentLabels" );
+	auto PorosityList = greyscale_db->getVector<double>( "PorosityList" );
+	auto PermeabilityList = greyscale_db->getVector<double>( "PermeabilityList" );
+
+	NLABELS=LabelList.size();
+	if (NLABELS != PorosityList.size()){
+		ERROR("Error: ComponentLabels and PorosityList must be the same length! \n");
+	}
+
+	double label_count[NLABELS];
+	double label_count_global[NLABELS];
+	// Assign the labels
+
+	for (int idx=0; idx<NLABELS; idx++) label_count[idx]=0;
+
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				VALUE=id[n];
+				// Assign the affinity from the paired list
+				for (unsigned int idx=0; idx < NLABELS; idx++){
+				      //printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]);
+					if (VALUE == LabelList[idx]){
+						POROSITY=PorosityList[idx];
+						label_count[idx] += 1.0;
+						idx = NLABELS;
+						//Mask->id[n] = 0; // set mask to zero since this is an immobile component
+					}
+				}
+				int idx = Map(i,j,k);
+				if (!(idx < 0)){
+                    if (POROSITY<=0.0){
+                        ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n");
+                    }
+                    else{
+					    Porosity[idx] = POROSITY;
+                    }
+                }
+			}
+		}
+	}
+
+	if (NLABELS != PermeabilityList.size()){
+		ERROR("Error: ComponentLabels and PermeabilityList must be the same length! \n");
+	}
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int n = k*Nx*Ny+j*Nx+i;
+				VALUE=id[n];
+				// Assign the affinity from the paired list
+				for (unsigned int idx=0; idx < NLABELS; idx++){
+					//printf("idx=%i, value=%i, %i, \n",idx, VALUE,LabelList[idx]);
+					if (VALUE == LabelList[idx]){
+						PERMEABILITY=PermeabilityList[idx];
+						idx = NLABELS;
+						//Mask->id[n] = 0; // set mask to zero since this is an immobile component
+					}
+				}
+				int idx = Map(i,j,k);
+				if (!(idx < 0)){
+                    if (PERMEABILITY<=0.0){
+                        ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n");
+                    }
+                    else{
+					    Permeability[idx] = PERMEABILITY/Dm->voxel_length/Dm->voxel_length;
+                    }
+                }
+			}
+		}
+	}
+
+
+	// Set Dm to match Mask
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i]; 
+	
+	for (int idx=0; idx<NLABELS; idx++)		label_count_global[idx]=sumReduce( Dm->Comm, label_count[idx]);
+    //Initialize a weighted porosity after considering grey voxels
+    GreyPorosity=0.0;
+	for (unsigned int idx=0; idx<NLABELS; idx++){
+		double volume_fraction  = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+        GreyPorosity+=volume_fraction*PorosityList[idx];
+    }
+
+	if (rank==0){
+        printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length);
+		printf("Number of component labels: %lu \n",NLABELS);
+		for (unsigned int idx=0; idx<NLABELS; idx++){
+			VALUE=LabelList[idx];
+			POROSITY=PorosityList[idx];
+			PERMEABILITY=PermeabilityList[idx];
+			double volume_fraction  = double(label_count_global[idx])/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+			printf("   label=%d: porosity=%.3g, permeability=%.3g [um^2] (=%.3g [voxel^2]), volume fraction=%.3g\n",
+                    VALUE,POROSITY,PERMEABILITY,PERMEABILITY/Dm->voxel_length/Dm->voxel_length,volume_fraction); 
+            printf("             effective porosity=%.3g\n",volume_fraction*POROSITY);
+		}
+        printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity);
+	}
+}
+
+void ScaLBL_GreyscaleModel::AssignComponentLabels(double *Porosity,double *Permeability,const vector<std::string> &File_poro,const vector<std::string> &File_perm)
+{
+    double *Porosity_host, *Permeability_host;
+    Porosity_host = new double[N];
+    Permeability_host = new double[N];
+	double POROSITY=0.f;
+	double PERMEABILITY=0.f;
+    //Initialize a weighted porosity after considering grey voxels
+    double GreyPorosity_loc=0.0;
+    GreyPorosity=0.0;
+    //double label_count_loc = 0.0;
+    //double label_count_glb = 0.0;
+
+    Mask->ReadFromFile(File_poro[0],File_poro[1],Porosity_host);
+    Mask->ReadFromFile(File_perm[0],File_perm[1],Permeability_host);
+
+	for (int k=0;k<Nz;k++){
+		for (int j=0;j<Ny;j++){
+			for (int i=0;i<Nx;i++){
+				int idx = Map(i,j,k);
+				if (!(idx < 0)){
+				    int n = k*Nx*Ny+j*Nx+i;
+                    POROSITY = Porosity_host[n];
+                    PERMEABILITY = Permeability_host[n];
+                    if (POROSITY<=0.0){
+                        ERROR("Error: Porosity for grey voxels must be 0.0 < Porosity <= 1.0 !\n");
+                    }
+                    else if (PERMEABILITY<=0.0){
+                        ERROR("Error: Permeability for grey voxel must be > 0.0 ! \n");
+                    }
+                    else{
+                        Porosity[idx] = POROSITY;
+                        Permeability[idx] = PERMEABILITY;
+                        GreyPorosity_loc += POROSITY;
+                        //label_count_loc += 1.0;
+                    }
+                }
+			}
+		}
+	}
+    GreyPorosity = sumReduce( Dm->Comm, GreyPorosity_loc);
+    GreyPorosity = GreyPorosity/double((Nx-2)*(Ny-2)*(Nz-2)*nprocs);
+
+	if (rank==0){
+        printf("Image resolution: %.5g [um/voxel]\n",Dm->voxel_length);
+        printf("The weighted porosity, considering both open and grey voxels, is %.3g\n",GreyPorosity);
+	}
+    delete [] Porosity_host;
+    delete [] Permeability_host;
+}
+
+void ScaLBL_GreyscaleModel::Create(){
+	/*
+	 *  This function creates the variables needed to run a LBM 
+	 */
+	//.........................................................
+	// don't perform computations at the eight corners
+	//id[0] = id[Nx-1] = id[(Ny-1)*Nx] = id[(Ny-1)*Nx + Nx-1] = 0;
+	//id[(Nz-1)*Nx*Ny] = id[(Nz-1)*Nx*Ny+Nx-1] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx] = id[(Nz-1)*Nx*Ny+(Ny-1)*Nx + Nx-1] = 0;
+
+	//.........................................................
+	// Initialize communication structures in averaging domain
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
+	Mask->CommInit();
+	Np=Mask->PoreCount();
+	//...........................................................................
+	if (rank==0)    printf ("Create ScaLBL_Communicator \n");
+	// Create a communicator for the device (will use optimized layout)
+	// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
+	ScaLBL_Comm  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+
+	int Npad=(Np/16 + 2)*16;
+	if (rank==0)    printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
+	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
+	auto neighborList= new int[18*Npad];
+	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
+	MPI_Barrier(comm);
+
+	//...........................................................................
+	//                MAIN  VARIABLES ALLOCATED HERE
+	//...........................................................................
+	// LBM variables
+	if (rank==0)    printf ("Allocating distributions \n");
+	//......................device distributions.................................
+	dist_mem_size = Np*sizeof(double);
+	neighborSize=18*(Np*sizeof(int));
+	//...........................................................................
+	ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
+	ScaLBL_AllocateDeviceMemory((void **) &fq, 19*dist_mem_size);
+	ScaLBL_AllocateDeviceMemory((void **) &Permeability, sizeof(double)*Np);		
+	ScaLBL_AllocateDeviceMemory((void **) &Porosity, sizeof(double)*Np);		
+	ScaLBL_AllocateDeviceMemory((void **) &Pressure_dvc, sizeof(double)*Np);
+	ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
+	//...........................................................................
+	// Update GPU data structures
+	if (rank==0)	printf ("Setting up device neighbor list \n");
+	fflush(stdout);
+	// copy the neighbor list 
+	ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
+	// initialize phi based on PhaseLabel (include solid component labels)
+	double *Poros, *Perm;
+	Poros = new double[Np];
+	Perm  = new double[Np];
+    if (greyscale_db->keyExists("FileVoxelPorosityMap")){
+        //NOTE: FileVoxel**Map is a vector, including "file_name, datatype"
+		auto File_poro = greyscale_db->getVector<std::string>( "FileVoxelPorosityMap" );
+		auto File_perm = greyscale_db->getVector<std::string>( "FileVoxelPermeabilityMap" );
+	    AssignComponentLabels(Poros,Perm,File_poro,File_perm);
+    }
+    else if (greyscale_db->keyExists("PorosityList")){
+        //initialize voxel porosity and perm from the input list
+	    AssignComponentLabels(Poros,Perm);
+    }
+    else {
+		ERROR("Error: PorosityList or FilenameVoxelPorosityMap cannot be found! \n");
+    }
+	ScaLBL_CopyToDevice(Porosity, Poros, Np*sizeof(double));
+	ScaLBL_CopyToDevice(Permeability, Perm, Np*sizeof(double));
+    delete [] Poros;
+    delete [] Perm;
+}        
+
+
+void ScaLBL_GreyscaleModel::Initialize(){
+	if (rank==0)	printf ("Initializing distributions \n");
+    //TODO: for BGK, you need to consider voxel porosity
+    //      for IMRT, the whole set of feq is different
+    //      if in the future you have different collison mode, need to write two set of initialization functions
+    if (CollisionType==1){
+	    ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den);
+        if (rank==0) printf("Collision model: Incompressible MRT.\n");
+    }
+    else if (CollisionType==2){
+	    ScaLBL_D3Q19_Init(fq, Np);
+        if (rank==0) printf("Collision model: BGK.\n");
+    }
+    else if (CollisionType==3){
+	    ScaLBL_D3Q19_Init(fq, Np);
+        if (rank==0) printf("Collision model: MRT.\n");
+    }
+    else{
+        if (rank==0) printf("Unknown collison type! IMRT collision is used.\n"); 
+	    ScaLBL_D3Q19_GreyIMRT_Init(fq, Np, Den);
+        CollisionType=1;
+		greyscale_db->putScalar<std::string>( "collision", "IMRT" );
+    }
+
+	if (Restart == true){
+		if (rank==0){
+			printf("Initializing distributions from Restart! \n");
+		}
+		// Read in the restart file to CPU buffers
+        std::shared_ptr<double> cfq;
+        cfq = std::shared_ptr<double>(new double[19*Np],DeleteArray<double>);
+        FILE *File;
+        File=fopen(LocalRestartFile,"rb");
+        fread(cfq.get(),sizeof(double),19*Np,File);
+        fclose(File);
+
+		// Copy the restart data to the GPU
+		ScaLBL_CopyToDevice(fq,cfq.get(),19*Np*sizeof(double));
+		ScaLBL_DeviceBarrier();
+
+		MPI_Barrier(comm);
+	}
+}
+
+void ScaLBL_GreyscaleModel::Run(){
+	int nprocs=nprocx*nprocy*nprocz;
+	const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
+	
+	int analysis_interval = 1000; 	// number of timesteps in between in situ analysis 
+	int visualization_interval = 1000; 	 
+	int restart_interval = 10000; 	// number of timesteps in between in saving distributions for restart 
+	if (analysis_db->keyExists( "analysis_interval" )){
+		analysis_interval = analysis_db->getScalar<int>( "analysis_interval" );
+	}
+	if (analysis_db->keyExists( "visualization_interval" )){
+		visualization_interval = analysis_db->getScalar<int>( "visualization_interval" );
+	}
+	if (analysis_db->keyExists( "restart_interval" )){
+		restart_interval = analysis_db->getScalar<int>( "restart_interval" );
+	}
+	if (greyscale_db->keyExists( "timestep" )){
+		timestep = greyscale_db->getScalar<int>( "timestep" );
+	}
+
+	if (rank==0){
+		printf("********************************************************\n");
+		printf("No. of timesteps: %i \n", timestepMax);
+		fflush(stdout);
+	}
+
+	//.......create and start timer............
+	double starttime,stoptime,cputime;
+	ScaLBL_DeviceBarrier();
+	MPI_Barrier(comm);
+	starttime = MPI_Wtime();
+	//.........................................
+	
+	Minkowski Morphology(Mask);
+
+	//************ MAIN ITERATION LOOP ***************************************/
+	PROFILE_START("Loop");
+	auto current_db = db->cloneDatabase();
+	double rlx = 1.0/tau;
+    double rlx_eff = 1.0/tau_eff;
+	double error = 1.0;
+	double flow_rate_previous = 0.0;
+	while (timestep < timestepMax && error > tolerance) {
+		//************************************************************************/
+		// *************ODD TIMESTEP*************//
+		timestep++;
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
+        switch (CollisionType){
+            case 1: 
+                    ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            case 2: 
+                    ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
+                    break;
+            case 3: 
+                    ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            default: 
+                    ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+        }
+		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		ScaLBL_DeviceBarrier();
+		// Set BCs
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+        switch (CollisionType){
+            case 1: 
+		            ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            case 2: 
+		            ScaLBL_D3Q19_AAodd_Greyscale(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
+                    break;
+            case 3: 
+		            ScaLBL_D3Q19_AAodd_Greyscale_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            default: 
+		            ScaLBL_D3Q19_AAodd_Greyscale_IMRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+        }
+		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+
+		// *************EVEN TIMESTEP*************//
+		timestep++;
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
+        switch (CollisionType){
+            case 1: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            case 2: 
+		            ScaLBL_D3Q19_AAeven_Greyscale(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
+                    break;
+            case 3: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            default: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+        }
+        ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		ScaLBL_DeviceBarrier();
+		// Set BCs
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+        switch (CollisionType){
+            case 1: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            case 2: 
+		            ScaLBL_D3Q19_AAeven_Greyscale(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Pressure_dvc);
+                    break;
+            case 3: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+            default: 
+		            ScaLBL_D3Q19_AAeven_Greyscale_IMRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx, rlx_eff, Fx, Fy, Fz,Porosity,Permeability,Velocity,Den,Pressure_dvc);
+                    break;
+        }
+        ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+		//************************************************************************/
+		
+		if (timestep%analysis_interval==0){
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x);
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y);
+			ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z);
+			//ScaLBL_Comm->RegularLayout(Map,Porosity,PorosityMap);
+			//ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure);
+			
+			double count_loc=0;
+			double count;
+			double vax,vay,vaz;
+			double vax_loc,vay_loc,vaz_loc;
+            //double px_loc,py_loc,pz_loc;
+            //double px,py,pz;
+            //double mass_loc,mass_glb;
+            
+            //parameters for domain average
+	        int64_t i,j,k,n,imin,jmin,kmin,kmax;
+            // If external boundary conditions are set, do not average over the inlet and outlet
+            kmin=1; kmax=Nz-1;
+            //In case user forgets to specify the inlet/outlet buffer layers for BC>0
+            if (BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
+            if (BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
+
+            imin=jmin=1;
+            // If inlet/outlet layers exist use these as default
+            //if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
+            //if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
+            if (BoundaryCondition > 0 && Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin = 1 + Dm->inlet_layers_z;//"1" indicates the halo layer
+            if (BoundaryCondition > 0 && Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax = Nz-1 - Dm->outlet_layers_z; 
+
+//			px_loc = py_loc = pz_loc = 0.f;
+//            mass_loc = 0.f;
+//			for (int k=kmin; k<kmax; k++){
+//				for (int j=jmin; j<Ny-1; j++){
+//					for (int i=imin; i<Nx-1; i++){
+//						if (SignDist(i,j,k) > 0){
+//							px_loc   += Velocity_x(i,j,k)*Den*PorosityMap(i,j,k);
+//							py_loc   += Velocity_y(i,j,k)*Den*PorosityMap(i,j,k);
+//							pz_loc   += Velocity_z(i,j,k)*Den*PorosityMap(i,j,k);
+//							mass_loc += Den*PorosityMap(i,j,k);
+//						}
+//					}
+//				}
+//			}
+//			MPI_Allreduce(&px_loc,  &px,      1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+//			MPI_Allreduce(&py_loc,  &py,      1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+//			MPI_Allreduce(&pz_loc,  &pz,      1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+//			MPI_Allreduce(&mass_loc,&mass_glb,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+//			
+//			vax = px/mass_glb;
+//			vay = py/mass_glb;
+//			vaz = pz/mass_glb;
+            
+			vax_loc = vay_loc = vaz_loc = 0.f;
+			for (int k=kmin; k<kmax; k++){
+				for (int j=jmin; j<Ny-1; j++){
+					for (int i=imin; i<Nx-1; i++){
+						if (SignDist(i,j,k) > 0){
+							vax_loc += Velocity_x(i,j,k);
+							vay_loc += Velocity_y(i,j,k);
+							vaz_loc += Velocity_z(i,j,k);
+							count_loc+=1.0;
+						}
+					}
+				}
+			}
+            vax = sumReduce( Mask->Comm, vax_loc);
+            vay = sumReduce( Mask->Comm, vay_loc);
+            vaz = sumReduce( Mask->Comm, vaz_loc);
+            count = sumReduce( Mask->Comm, count_loc);
+
+			vax /= count;
+			vay /= count;
+			vaz /= count;
+
+			double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+			double dir_x = Fx/force_mag;
+			double dir_y = Fy/force_mag;
+			double dir_z = Fz/force_mag;
+			if (force_mag == 0.0){
+				// default to z direction
+				dir_x = 0.0;
+				dir_y = 0.0;
+				dir_z = 1.0;
+				force_mag = 1.0;
+			}
+			//double flow_rate = (px*dir_x + py*dir_y + pz*dir_z)/mass_glb;
+			double flow_rate = (vax*dir_x + vay*dir_y + vaz*dir_z);
+			
+			error = fabs(flow_rate - flow_rate_previous) / fabs(flow_rate);
+			flow_rate_previous = flow_rate;
+			
+			//if (rank==0) printf("Computing Minkowski functionals \n");
+			Morphology.ComputeScalar(SignDist,0.f);
+			//Morphology.PrintAll();
+			double mu = (tau-0.5)/3.f;
+			double Vs = Morphology.V();
+			double As = Morphology.A();
+			double Hs = Morphology.H();
+			double Xs = Morphology.X();
+			Vs = sumReduce( Dm->Comm, Vs);
+			As = sumReduce( Dm->Comm, As);
+			Hs = sumReduce( Dm->Comm, Hs);
+			Xs = sumReduce( Dm->Comm, Xs);
+
+			double h = Dm->voxel_length;
+			//double absperm = h*h*mu*Mask->Porosity()*flow_rate / force_mag;
+			double absperm = h*h*mu*GreyPorosity*flow_rate / force_mag;
+
+            if (rank==0){
+				printf("     AbsPerm = %.5g [micron^2]\n",absperm);
+                bool WriteHeader=false;
+                FILE * log_file = fopen("Permeability.csv","r");
+                if (log_file != NULL)
+                    fclose(log_file);
+                else
+                    WriteHeader=true;
+                log_file = fopen("Permeability.csv","a");
+                if (WriteHeader)
+                    fprintf(log_file,"timestep Fx Fy Fz mu Vs As Hs Xs vax vay vaz AbsPerm \n",
+                            timestep,Fx,Fy,Fz,mu,h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz,absperm);
+
+                fprintf(log_file,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",timestep, Fx, Fy, Fz, mu, 
+                        h*h*h*Vs,h*h*As,h*Hs,Xs,vax,vay,vaz, absperm);
+                fclose(log_file);
+            }
+		}
+
+		if (timestep%visualization_interval==0){
+            VelocityField();
+        }
+
+		if (timestep%restart_interval==0){
+            //Use rank=0 write out Restart.db
+            if (rank==0) {
+                greyscale_db->putScalar<int>("timestep",timestep);    		
+                greyscale_db->putScalar<bool>( "Restart", true );
+                current_db->putDatabase("Greyscale", greyscale_db);
+                std::ofstream OutStream("Restart.db");
+                current_db->print(OutStream, "");
+                OutStream.close();
+      
+            }
+            //Write out Restart data.
+            std::shared_ptr<double> cfq;
+            cfq = std::shared_ptr<double>(new double[19*Np],DeleteArray<double>);
+            ScaLBL_CopyToHost(cfq.get(),fq,19*Np*sizeof(double));// Copy restart data to the CPU
+
+            FILE *RESTARTFILE;
+            RESTARTFILE=fopen(LocalRestartFile,"wb");
+            fwrite(cfq.get(),sizeof(double),19*Np,RESTARTFILE);
+            fclose(RESTARTFILE);
+		    MPI_Barrier(comm);
+        }
+	}
+
+	PROFILE_STOP("Loop");
+	PROFILE_SAVE("lbpm_greyscale_simulator",1);
+	//************************************************************************
+	ScaLBL_DeviceBarrier();
+	MPI_Barrier(comm);
+	stoptime = MPI_Wtime();
+	if (rank==0) printf("-------------------------------------------------------------------\n");
+	// Compute the walltime per timestep
+	cputime = (stoptime - starttime)/timestep;
+	// Performance obtained from each node
+	double MLUPS = double(Np)/cputime/1000000;
+
+	if (rank==0) printf("********************************************************\n");
+	if (rank==0) printf("CPU time = %f \n", cputime);
+	if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
+	MLUPS *= nprocs;
+	if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
+	if (rank==0) printf("********************************************************\n");
+
+	// ************************************************************************
+}
+
+void ScaLBL_GreyscaleModel::VelocityField(){
+
+/*	Minkowski Morphology(Mask);
+	int SIZE=Np*sizeof(double);
+	ScaLBL_D3Q19_Momentum(fq,Velocity, Np);
+	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
+	ScaLBL_CopyToHost(&VELOCITY[0],&Velocity[0],3*SIZE);
+
+	memcpy(Morphology.SDn.data(), Distance.data(), Nx*Ny*Nz*sizeof(double));
+	Morphology.Initialize();
+	Morphology.UpdateMeshValues();
+	Morphology.ComputeLocal();
+	Morphology.Reduce();
+	
+	double count_loc=0;
+	double count;
+	double vax,vay,vaz;
+	double vax_loc,vay_loc,vaz_loc;
+	vax_loc = vay_loc = vaz_loc = 0.f;
+	for (int n=0; n<ScaLBL_Comm->LastExterior(); n++){
+		vax_loc += VELOCITY[n];
+		vay_loc += VELOCITY[Np+n];
+		vaz_loc += VELOCITY[2*Np+n];
+		count_loc+=1.0;
+	}
+	
+	for (int n=ScaLBL_Comm->FirstInterior(); n<ScaLBL_Comm->LastInterior(); n++){
+		vax_loc += VELOCITY[n];
+		vay_loc += VELOCITY[Np+n];
+		vaz_loc += VELOCITY[2*Np+n];
+		count_loc+=1.0;
+	}
+	MPI_Allreduce(&vax_loc,&vax,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vay_loc,&vay,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&vaz_loc,&vaz,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	MPI_Allreduce(&count_loc,&count,1,MPI_DOUBLE,MPI_SUM,Mask->Comm);
+	
+	vax /= count;
+	vay /= count;
+	vaz /= count;
+	
+	double mu = (tau-0.5)/3.f;
+	if (rank==0) printf("Fx Fy Fz mu Vs As Js Xs vx vy vz\n");
+	if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, 
+						Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz);
+						*/
+	
+	std::vector<IO::MeshDataStruct> visData;
+	fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
+
+	auto VxVar = std::make_shared<IO::Variable>();
+	auto VyVar = std::make_shared<IO::Variable>();
+	auto VzVar = std::make_shared<IO::Variable>();
+	auto SignDistVar = std::make_shared<IO::Variable>();
+	auto PressureVar = std::make_shared<IO::Variable>();
+
+	IO::initialize("","silo","false");
+	// Create the MeshDataStruct	
+	visData.resize(1);
+	visData[0].meshName = "domain";
+	visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+	SignDistVar->name = "SignDist";
+	SignDistVar->type = IO::VariableType::VolumeVariable;
+	SignDistVar->dim = 1;
+	SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(SignDistVar);
+	
+	VxVar->name = "Velocity_x";
+	VxVar->type = IO::VariableType::VolumeVariable;
+	VxVar->dim = 1;
+	VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VxVar);
+	VyVar->name = "Velocity_y";
+	VyVar->type = IO::VariableType::VolumeVariable;
+	VyVar->dim = 1;
+	VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VyVar);
+	VzVar->name = "Velocity_z";
+	VzVar->type = IO::VariableType::VolumeVariable;
+	VzVar->dim = 1;
+	VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(VzVar);
+	
+	PressureVar->name = "Pressure";
+	PressureVar->type = IO::VariableType::VolumeVariable;
+	PressureVar->dim = 1;
+	PressureVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+	visData[0].vars.push_back(PressureVar);
+
+	Array<double>& SignData  = visData[0].vars[0]->data;
+	Array<double>& VelxData = visData[0].vars[1]->data;
+	Array<double>& VelyData = visData[0].vars[2]->data;
+	Array<double>& VelzData = visData[0].vars[3]->data;
+	Array<double>& PressureData = visData[0].vars[4]->data;
+	
+    ASSERT(visData[0].vars[0]->name=="SignDist");
+    ASSERT(visData[0].vars[1]->name=="Velocity_x");
+    ASSERT(visData[0].vars[2]->name=="Velocity_y");
+    ASSERT(visData[0].vars[3]->name=="Velocity_z");
+    ASSERT(visData[0].vars[4]->name=="Pressure");
+	
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Velocity_x);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Velocity_y);
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Velocity_z);
+	ScaLBL_Comm->RegularLayout(Map,Pressure_dvc,Pressure);
+
+    fillData.copy(SignDist,SignData);
+    fillData.copy(Velocity_x,VelxData);
+    fillData.copy(Velocity_y,VelyData);
+    fillData.copy(Velocity_z,VelzData);
+    fillData.copy(Pressure,PressureData);
+	
+    IO::writeData( timestep, visData, Dm->Comm );
+
+}
+
+void ScaLBL_GreyscaleModel::WriteDebug(){
+	// Copy back final phase indicator field and convert to regular layout
+	DoubleArray PhaseField(Nx,Ny,Nz);
+
+	//ScaLBL_CopyToHost(Porosity.data(), Poros, sizeof(double)*N);
+
+//	FILE *OUTFILE;
+//	sprintf(LocalRankFilename,"Phase.%05i.raw",rank);
+//	OUTFILE = fopen(LocalRankFilename,"wb");
+//	fwrite(PhaseField.data(),8,N,OUTFILE);
+//	fclose(OUTFILE);
+//
+//    ScaLBL_Comm->RegularLayout(Map,&Den[0],PhaseField);
+//	FILE *AFILE;
+//	sprintf(LocalRankFilename,"A.%05i.raw",rank);
+//	AFILE = fopen(LocalRankFilename,"wb");
+//	fwrite(PhaseField.data(),8,N,AFILE);
+//	fclose(AFILE);
+//
+//	ScaLBL_Comm->RegularLayout(Map,&Den[Np],PhaseField);
+//	FILE *BFILE;
+//	sprintf(LocalRankFilename,"B.%05i.raw",rank);
+//	BFILE = fopen(LocalRankFilename,"wb");
+//	fwrite(PhaseField.data(),8,N,BFILE);
+//	fclose(BFILE);
+//
+//	ScaLBL_Comm->RegularLayout(Map,Pressure,PhaseField);
+//	FILE *PFILE;
+//	sprintf(LocalRankFilename,"Pressure.%05i.raw",rank);
+//	PFILE = fopen(LocalRankFilename,"wb");
+//	fwrite(PhaseField.data(),8,N,PFILE);
+//	fclose(PFILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],PhaseField);
+	FILE *VELX_FILE;
+	sprintf(LocalRankFilename,"Velocity_X.%05i.raw",rank);
+	VELX_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELX_FILE);
+	fclose(VELX_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],PhaseField);
+	FILE *VELY_FILE;
+	sprintf(LocalRankFilename,"Velocity_Y.%05i.raw",rank);
+	VELY_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELY_FILE);
+	fclose(VELY_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],PhaseField);
+	FILE *VELZ_FILE;
+	sprintf(LocalRankFilename,"Velocity_Z.%05i.raw",rank);
+	VELZ_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,VELZ_FILE);
+	fclose(VELZ_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Porosity[0],PhaseField);
+	FILE *POROS_FILE;
+	sprintf(LocalRankFilename,"Porosity.%05i.raw",rank);
+	POROS_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,POROS_FILE);
+	fclose(POROS_FILE);
+
+	ScaLBL_Comm->RegularLayout(Map,&Permeability[0],PhaseField);
+	FILE *PERM_FILE;
+	sprintf(LocalRankFilename,"Permeability.%05i.raw",rank);
+	PERM_FILE = fopen(LocalRankFilename,"wb");
+	fwrite(PhaseField.data(),8,N,PERM_FILE);
+	fclose(PERM_FILE);
+}

models/GreyscaleModel.h

@@ -0,0 +1,106 @@
+/*
+  Copyright 2020 Equinor ASA                                                          
+  Copyright Equnior ASA
+                                                                                      
+  This file is part of the Open Porous Media project (OPM).                           
+  OPM is free software: you can redistribute it and/or modify                         
+  it under the terms of the GNU General Public License as published by                
+  the Free Software Foundation, either version 3 of the License, or                   
+  (at your option) any later version.                                                 
+  OPM is distributed in the hope that it will be useful,                              
+  but WITHOUT ANY WARRANTY; without even the implied warranty of                      
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                       
+  GNU General Public License for more details.                                        
+  You should have received a copy of the GNU General Public License                   
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.       
+
+ greyscale lattice boltzmann model
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "common/Database.h"
+#include "common/ScaLBL.h"
+#include "ProfilerApp.h"
+#include "threadpool/thread_pool.h"
+
+class ScaLBL_GreyscaleModel{
+public:
+	ScaLBL_GreyscaleModel(int RANK, int NP, MPI_Comm COMM);
+	~ScaLBL_GreyscaleModel();	
+	
+	// functions in they should be run
+	void ReadParams(string filename);
+	void ReadParams(std::shared_ptr<Database> db0);
+	void SetDomain();
+	void ReadInput();
+	void Create();
+	void Initialize();
+	void Run();
+	void WriteDebug();
+	void VelocityField();
+	
+	bool Restart,pBC;
+	int timestep,timestepMax;
+	int BoundaryCondition;
+    int CollisionType;
+	double tau;
+    double tau_eff;
+    double Den;//constant density
+	double tolerance;
+	double Fx,Fy,Fz,flux;
+	double din,dout;
+    double dp;//solid particle diameter, unit in voxel
+    double GreyPorosity;
+	
+	int Nx,Ny,Nz,N,Np;
+	int rank,nprocx,nprocy,nprocz,nprocs;
+	double Lx,Ly,Lz;
+
+	std::shared_ptr<Domain> Dm;   // this domain is for analysis
+	std::shared_ptr<Domain> Mask; // this domain is for lbm
+	std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm;
+    
+    // input database
+    std::shared_ptr<Database> db;
+    std::shared_ptr<Database> domain_db;
+    std::shared_ptr<Database> greyscale_db;
+    std::shared_ptr<Database> analysis_db;
+    std::shared_ptr<Database> vis_db;
+
+    signed char *id;    
+	int *NeighborList;
+	double *fq;
+	double *Permeability;//grey voxel permeability
+	double *Porosity;
+	double *Velocity;
+	double *Pressure_dvc;
+    IntArray Map;
+    DoubleArray SignDist;
+    DoubleArray Velocity_x;
+    DoubleArray Velocity_y;
+    DoubleArray Velocity_z;
+    DoubleArray PorosityMap;
+    DoubleArray Pressure;
+		
+private:
+	MPI_Comm comm;
+    
+	int dist_mem_size;
+	int neighborSize;
+	// filenames
+    char LocalRankString[8];
+    char LocalRankFilename[40];
+    char LocalRestartFile[40];
+   
+    void AssignComponentLabels(double *Porosity, double *Permeablity);
+    void AssignComponentLabels(double *Porosity,double *Permeability,const vector<std::string> &File_poro,const vector<std::string> &File_perm);
+};
+

models/MRTModel.cpp

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -18,6 +19,7 @@
  */
 #include "models/MRTModel.h"
 #include "analysis/distance.h"
+#include "common/ReadMicroCT.h"
 
 ScaLBL_MRTModel::ScaLBL_MRTModel(int RANK, int NP, MPI_Comm COMM):
 rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tau(0),
@@ -47,7 +49,7 @@ void ScaLBL_MRTModel::ReadParams(string filename){
 		timestepMax = mrt_db->getScalar<int>( "timestepMax" );
 	}
 	if (mrt_db->keyExists( "tolerance" )){
-		tolerance = mrt_db->getScalar<int>( "timestepMax" );
+		tolerance = mrt_db->getScalar<double>( "tolerance" );
 	}
 	if (mrt_db->keyExists( "tau" )){
 		tau = mrt_db->getScalar<double>( "tau" );
@@ -108,20 +110,36 @@ void ScaLBL_MRTModel::SetDomain(){
 }
 
 void ScaLBL_MRTModel::ReadInput(){
-    int rank=Dm->rank();
-    size_t readID;
-    //.......................................................................
-    //.......................................................................
-    Mask->ReadIDs();
     
     sprintf(LocalRankString,"%05d",Dm->rank());
     sprintf(LocalRankFilename,"%s%s","ID.",LocalRankString);
     sprintf(LocalRestartFile,"%s%s","Restart.",LocalRankString);
 
-	// Generate the signed distance map
+    
+    if (domain_db->keyExists( "Filename" )){
+    	auto Filename = domain_db->getScalar<std::string>( "Filename" );
+    	Mask->Decomp(Filename);
+    }
+    else if (domain_db->keyExists( "GridFile" )){
+    	// Read the local domain data
+    	auto input_id = readMicroCT( *domain_db, comm );
+    	// Fill the halo (assuming GCW of 1)
+    	array<int,3> size0 = { (int) input_id.size(0), (int) input_id.size(1), (int) input_id.size(2) };
+    	ArraySize size1 = { (size_t) Mask->Nx, (size_t) Mask->Ny, (size_t) Mask->Nz };
+    	ASSERT( (int) size1[0] == size0[0]+2 && (int) size1[1] == size0[1]+2 && (int) size1[2] == size0[2]+2 );
+    	fillHalo<signed char> fill( comm, Mask->rank_info, size0, { 1, 1, 1 }, 0, 1 );
+    	Array<signed char> id_view;
+    	id_view.viewRaw( size1, Mask->id );
+    	fill.copy( input_id, id_view );
+    	fill.fill( id_view );
+    }
+    else{
+    	Mask->ReadIDs();
+    }
+
+    // Generate the signed distance map
 	// Initialize the domain and communication
 	Array<char> id_solid(Nx,Ny,Nz);
-	int count = 0;
 	// Solve for the position of the solid phase
 	for (int k=0;k<Nz;k++){
 		for (int j=0;j<Ny;j++){
@@ -137,7 +155,6 @@ void ScaLBL_MRTModel::ReadInput(){
 	for (int k=0;k<Nz;k++){
 		for (int j=0;j<Ny;j++){
 			for (int i=0;i<Nx;i++){
-				int n=k*Nx*Ny+j*Nx+i;
 				// Initialize distance to +/- 1
 				Distance(i,j,k) = 2.0*double(id_solid(i,j,k))-1.0;
 			}
@@ -206,12 +223,20 @@ void ScaLBL_MRTModel::Run(){
 	double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
 	
 	Minkowski Morphology(Mask);
-	int SIZE=Np*sizeof(double);
 
 	if (rank==0){
-		FILE * log_file = fopen("Permeability.csv","a");
-		fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n");
-		fclose(log_file);
+		bool WriteHeader=false;
+		FILE *log_file = fopen("Permeability.csv","r");
+		if (log_file != NULL)
+			fclose(log_file);
+		else
+			WriteHeader=true;
+
+		if (WriteHeader){
+			log_file = fopen("Permeability.csv","a+");
+			fprintf(log_file,"time Fx Fy Fz mu Vs As Js Xs vx vy vz k\n");
+			fclose(log_file);
+		}
 	}
 
 	//.......create and start timer............
@@ -229,12 +254,38 @@ void ScaLBL_MRTModel::Run(){
 		ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
 		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		// Set boundary conditions
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
 		ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
 		timestep++;
 		ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
 		ScaLBL_D3Q19_AAeven_MRT(fq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		// Set boundary conditions
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
 		ScaLBL_D3Q19_AAeven_MRT(fq, 0, ScaLBL_Comm->LastExterior(), Np, rlx_setA, rlx_setB, Fx, Fy, Fz);
 		ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
 		//************************************************************************/
@@ -374,7 +425,9 @@ void ScaLBL_MRTModel::VelocityField(){
 	if (rank==0) printf("%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",Fx, Fy, Fz, mu, 
 						Morphology.V(),Morphology.A(),Morphology.J(),Morphology.X(),vax,vay,vaz);
 						*/
-	
+        vis_db = db->getDatabase( "Visualization" );
+	if (vis_db->getWithDefault<bool>( "write_silo", false )){
+  
 	std::vector<IO::MeshDataStruct> visData;
 	fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
 
@@ -426,5 +479,5 @@ void ScaLBL_MRTModel::VelocityField(){
     fillData.copy(Velocity_z,VelzData);
 	
     IO::writeData( timestep, visData, Dm->Comm );
-
+    }
 }

models/MRTModel.h

@@ -1,5 +1,6 @@
 /*
   Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
+  Copyright Equnior ASA
 
   This file is part of the Open Porous Media project (OPM).
   OPM is free software: you can redistribute it and/or modify
@@ -64,6 +65,7 @@ public:
     std::shared_ptr<Database> db;
     std::shared_ptr<Database> domain_db;
     std::shared_ptr<Database> mrt_db;
+    std::shared_ptr<Database> vis_db;
 
     IntArray Map;
     DoubleArray Distance;

sample_scripts/configure_huckleberry

@@ -28,5 +28,5 @@ cmake                                    \
     -D USE_TIMER=0				 \
     $LBPM_SOURCE_DIR
 
-make VERBOSE=1 -j1 && make install
+make VERBOSE=1 -j8 && make install
 

sample_scripts/configure_summit

@@ -7,6 +7,7 @@ module load cuda
 
 export HDF5_DIR=/ccs/proj/csc380/mcclurej/install/hdf5/1.8.12/
 export SILO_DIR=/ccs/proj/csc380/mcclurej/install/silo/4.10.2/
+export NETCDF_DIR=/ccs/proj/geo136/install/netcdf/4.6.1
 
 # configure
 rm -rf CMake*
@@ -28,6 +29,8 @@ cmake                                       \
      -D USE_SILO=1                        \
         -D SILO_LIB="$SILO_DIR/lib/libsiloh5.a"    \
    	-D SILO_DIRECTORY="$SILO_DIR"		     \
+    -D USE_NETCDF=1				     \
+        -D NETCDF_DIRECTORY="$NETCDF_DIR"	     \
     -D USE_DOXYGEN:BOOL=false		        \
     -D USE_TIMER=0			                \
     ~/LBPM-WIA

tests/CMakeLists.txt

@@ -3,6 +3,8 @@
 #ADD_LBPM_EXECUTABLE( lbpm_nondarcy_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
+ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator )
+ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator )
 #ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
 #ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
 ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )
@@ -35,6 +37,7 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest )
 #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )
 ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar )
 
+
 CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cylindertest ${CMAKE_CURRENT_BINARY_DIR}/cylindertest COPYONLY )
 
 # Add the tests
@@ -69,6 +72,8 @@ ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 )
 ADD_LBPM_TEST_1_2_4( testCommunication )
 ADD_LBPM_TEST( TestWriter )
 ADD_LBPM_TEST( TestDatabase )
+ADD_LBPM_TEST( TestSetDevice )
+ADD_LBPM_PROVISIONAL_TEST( TestMicroCTReader )
 IF ( USE_NETCDF )
     ADD_LBPM_TEST_PARALLEL( TestNetcdf 8 )
     ADD_LBPM_EXECUTABLE( lbpm_uCT_pp )

tests/GenerateSphereTest.cpp

@@ -145,8 +145,8 @@ inline void MorphOpen(DoubleArray SignDist, char *id, Domain &Dm, int nx, int ny
 
 	// Increase the critical radius until the target saturation is met
 	double deltaR=0.05; // amount to change the radius in voxel units
-	double Rcrit_old;
-	double Rcrit_new;
+	double Rcrit_old=0.0;
+	double Rcrit_new=0.0;
 
 	double GlobalNumber = 1.f;
 	int imin,jmin,kmin,imax,jmax,kmax;
@@ -363,7 +363,7 @@ int main(int argc, char **argv)
 	        nspheres = domain_db->getScalar<int>( "nspheres");
 
 		//printf("Set domain \n");
-		int BoundaryCondition=1;
+		//int BoundaryCondition=1;
 		//Nz += 2;
 		//Nx = Ny = Nz;	// Cubic domain
 		int N = Nx*Ny*Nz;
@@ -396,7 +396,7 @@ int main(int argc, char **argv)
 		int sum = 0;
 		double sum_local;
 		double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocs);
-		double porosity, pore_vol;
+		double porosity;
 		//...........................................................................
 		DoubleArray SignDist(Nx,Ny,Nz);
 		//.......................................................................
@@ -450,7 +450,6 @@ int main(int argc, char **argv)
 			}
 		}
 		sum=0;
-		pore_vol = 0.0;
 		for ( k=1;k<Nz-1;k++){
 			for ( j=1;j<Ny-1;j++){
 				for ( i=1;i<Nx-1;i++){

tests/TestBubbleDFH.cpp

@@ -74,11 +74,6 @@ int main(int argc, char **argv)
         ScaLBL_DeviceBarrier();
         MPI_Barrier(comm);
 
-        PROFILE_ENABLE(1);
-        //PROFILE_ENABLE_TRACE();
-        //PROFILE_ENABLE_MEMORY();
-        PROFILE_SYNCHRONIZE();
-        PROFILE_START("Main");
         Utilities::setErrorHandlers();
 
         // Variables that specify the computational domain  
@@ -164,14 +159,14 @@ int main(int argc, char **argv)
             pBC=false;
 
         // Full domain used for averaging (do not use mask for analysis)
-        std::shared_ptr<Domain> Dm(new Domain(domain_db,comm));
+        auto Dm = std::make_shared<Domain>(domain_db,comm);
         for (int i=0; i<Dm->Nx*Dm->Ny*Dm->Nz; i++) Dm->id[i] = 1;
-        std::shared_ptr<TwoPhase> Averages( new TwoPhase(Dm) );
+        auto Averages = std::make_shared<TwoPhase>(Dm);
         //   TwoPhase Averages(Dm);
         Dm->CommInit();
 
         // Mask that excludes the solid phase
-        std::shared_ptr<Domain> Mask(new Domain(domain_db,comm));
+        auto Mask = std::make_shared<Domain>(domain_db,comm);
         MPI_Barrier(comm);
 
         Nx+=2; Ny+=2; Nz += 2;
@@ -191,9 +186,8 @@ int main(int argc, char **argv)
 		//	printf("Local File Name =  %s \n",LocalRankFilename);
 		// .......... READ THE INPUT FILE .......................................
 		//	char value;
-		char *id;
-		id = new char[N];
-		double sum, sum_local;
+		auto id = new char[N];
+		double sum;
 		//...........................................................................
 		if (rank == 0) cout << "Setting up bubble..." << endl;
 	    double BubbleRadius = 15.5; // Radius of the capillary tube
@@ -244,19 +238,17 @@ int main(int argc, char **argv)
 		// Initialize communication structures in averaging domain
 		for (i=0; i<Mask->Nx*Mask->Ny*Mask->Nz; i++) Mask->id[i] = id[i];
 		Mask->CommInit();
-		double *PhaseLabel;
-		PhaseLabel = new double[N];
+		auto PhaseLabel = new double[N];
 		
 		//...........................................................................
 		if (rank==0)	printf ("Create ScaLBL_Communicator \n");
 		// Create a communicator for the device (will use optimized layout)
-		std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm(new ScaLBL_Communicator(Mask));
+		auto ScaLBL_Comm = std::make_shared<ScaLBL_Communicator>(Mask);
 		
 		int Npad=(Np/16 + 2)*16;
 		if (rank==0)	printf ("Set up memory efficient layout Npad=%i \n",Npad);
-		int *neighborList;
 		IntArray Map(Nx,Ny,Nz);
-		neighborList= new int[18*Npad];
+		auto neighborList= new int[18*Npad];
 		Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id,Np);
 		MPI_Barrier(comm);
 
@@ -515,9 +507,8 @@ int main(int argc, char **argv)
 		// Copy back final phase indicator field and convert to regular layout
 		DoubleArray PhaseField(Nx,Ny,Nz);
         ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField);
-    	FILE *OUTFILE;
-		sprintf(LocalRankFilename,"Phase.raw",rank);
-		OUTFILE = fopen(LocalRankFilename,"wb");
+		sprintf(LocalRankFilename,"Phase.raw");
+		auto OUTFILE = fopen(LocalRankFilename,"wb");
     	fwrite(PhaseField.data(),8,N,OUTFILE);
     	fclose(OUTFILE);
     	
@@ -535,9 +526,8 @@ int main(int argc, char **argv)
 				}
 			}
 		}
-    	FILE *GFILE;
 		sprintf(LocalRankFilename,"Gradient.raw");
-		GFILE = fopen(LocalRankFilename,"wb");
+		auto GFILE = fopen(LocalRankFilename,"wb");
     	fwrite(GradNorm.data(),8,N,GFILE);
     	fclose(GFILE);
     	
@@ -545,14 +535,12 @@ int main(int argc, char **argv)
 		DoubleArray Rho2(Nx,Ny,Nz);
         ScaLBL_Comm->RegularLayout(Map,&Den[0],Rho1);
         ScaLBL_Comm->RegularLayout(Map,&Den[Np],Rho2);
-    	FILE *RFILE1;
 		sprintf(LocalRankFilename,"Rho1.raw");
-		RFILE1 = fopen(LocalRankFilename,"wb");
+		auto RFILE1 = fopen(LocalRankFilename,"wb");
     	fwrite(Rho1.data(),8,N,RFILE1);
     	fclose(RFILE1);
-    	FILE *RFILE2;
 		sprintf(LocalRankFilename,"Rho2.raw");
-		RFILE2 = fopen(LocalRankFilename,"wb");
+		auto RFILE2 = fopen(LocalRankFilename,"wb");
     	fwrite(Rho2.data(),8,N,RFILE2);
     	fclose(RFILE2);
     	

tests/TestColorGradDFH.cpp

@@ -53,9 +53,6 @@ int main(int argc, char **argv)
 	    int Nx = db->getVector<int>( "n" )[0];
 	    int Ny = db->getVector<int>( "n" )[1];
 	    int Nz = db->getVector<int>( "n" )[2];
-	    int nprocx = db->getVector<int>( "nproc" )[0];
-	    int nprocy = db->getVector<int>( "nproc" )[1];
-	    int nprocz = db->getVector<int>( "nproc" )[2];
 
 	    if (rank==0){
 	    	printf("********************************************************\n");
@@ -64,7 +61,7 @@ int main(int argc, char **argv)
 	    }
 
 	    // Get the rank info
-	    std::shared_ptr<Domain> Dm(new Domain(db,comm));
+		auto Dm = std::make_shared<Domain>(db,comm);
 		Nx += 2;
 		Ny += 2;
 		Nz += 2;
@@ -111,7 +108,6 @@ int main(int argc, char **argv)
 		MPI_Barrier(comm);
 
 		//......................device distributions.................................
-		int dist_mem_size = Np*sizeof(double);
 		int neighborSize=18*Np*sizeof(int);
 		if (rank==0)	printf ("Allocating distributions \n");
 		int *NeighborList;

tests/TestCommD3Q19.cpp

@@ -49,7 +49,7 @@ extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int
 	{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
 	{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
 	
-	int q,i,j,k,n,N;
+	int q,i,j,k;
 	int Cqx,Cqy,Cqz; // Discrete velocity
 	int x,y,z;		// Global indices
 	int xn,yn,zn; 	// Global indices of neighbor 
@@ -59,8 +59,6 @@ extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int
 	Y = Ny*nprocy;
 	Z = Nz*nprocz;
     NULL_USE(Z);
-	N = (Nx+2)*(Ny+2)*(Nz+2);	// size of the array including halo
-
 
 	for (k=0; k<Nz; k++){ 
 		for (j=0; j<Ny; j++){
@@ -104,16 +102,13 @@ extern int GlobalCheckDebugDist(double *dist, IntArray Map, int Np, int Nx, int
 {
 
 	int returnValue = 0;
-	int q,i,j,k,n,N,idx;
-	int Cqx,Cqy,Cqz; // Discrete velocity
+	int q,i,j,k,idx;
 	int x,y,z;		// Global indices
-	int xn,yn,zn; 	// Global indices of neighbor 
 	int X,Y,Z;		// Global size
 	X = Nx*nprocx;
 	Y = Ny*nprocy;
 	Z = Nz*nprocz;
 	NULL_USE(Z);
-	N = (Nx+2)*(Ny+2)*(Nz+2);	// size of the array including halo
 	for (k=0; k<Nz; k++){ 
 		for (j=0; j<Ny; j++){
 			for (i=0; i<Nx; i++){
@@ -168,9 +163,6 @@ inline void UnpackID(int *list, int count, char *recvbuf, char *ID){
 //***************************************************************************************
 int main(int argc, char **argv)
 {
-	//*****************************************
-	// ***** MPI STUFF ****************
-	//*****************************************
 	// Initialize MPI
 	int rank,nprocs;
 	MPI_Init(&argc,&argv);
@@ -178,10 +170,7 @@ int main(int argc, char **argv)
 	MPI_Comm_rank(comm,&rank);
 	MPI_Comm_size(comm,&nprocs);
 	int check;
-
 	{
-		MPI_Request req1[18],req2[18];
-		MPI_Status stat1[18],stat2[18];
 
 		if (rank == 0){
 			printf("********************************************************\n");
@@ -191,11 +180,8 @@ int main(int argc, char **argv)
 
 		// BGK Model parameters
 		string FILENAME;
-		unsigned int nBlocks, nthreads;
-		int timestepMax, interval;
-		double tau,Fx,Fy,Fz,tol;
 		// Domain variables
-		int i,j,k,n;
+		int i,j,k;
 
         // Load inputs
         auto db = loadInputs( nprocs );
@@ -223,8 +209,7 @@ int main(int argc, char **argv)
 		char LocalRankFilename[40];
 		sprintf(LocalRankFilename,"ID.%05i",rank);
 
-		char *id;
-		id = new char[Nx*Ny*Nz];
+		auto id = new char[Nx*Ny*Nz];
 
 		/*		if (rank==0) printf("Assigning phase ID from file \n");
 		if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n");
@@ -237,7 +222,7 @@ int main(int argc, char **argv)
 		for (k=0;k<Nz;k++){
 			for (j=0;j<Ny;j++){
 				for (i=0;i<Nx;i++){
-					n = k*Nx*Ny+j*Nx+i;
+					int n = k*Nx*Ny+j*Nx+i;
 					id[n] = 1;
 					Dm->id[n] = id[n];
 				}
@@ -270,7 +255,7 @@ int main(int argc, char **argv)
 		for (k=1;k<Nz-1;k++){
 			for (j=1;j<Ny-1;j++){
 				for (i=1;i<Nx-1;i++){
-					n = k*Nx*Ny+j*Nx+i;
+					int n = k*Nx*Ny+j*Nx+i;
 					if (id[n] == component){
 						sum_local+=1.0;
 					}

tests/TestFluxBC.cpp

@@ -32,21 +32,14 @@ int main (int argc, char **argv)
 	}
 	{
 	  int i,j,k,n,Np;
-		bool pBC=true;
-		double Lx,Ly,Lz;
-		Lx = Ly = Lz = 1.f;
 		double din,dout;
-		int BC=1;
 
 	    // Load inputs
 	    auto db = loadInputs( nprocs );
 	    int Nx = db->getVector<int>( "n" )[0];
 	    int Ny = db->getVector<int>( "n" )[1];
 	    int Nz = db->getVector<int>( "n" )[2];
-	    int nprocx = db->getVector<int>( "nproc" )[0];
-	    int nprocy = db->getVector<int>( "nproc" )[1];
-	    int nprocz = db->getVector<int>( "nproc" )[2];
-		std::shared_ptr<Domain> Dm(new Domain(db,comm));
+		auto Dm = std::make_shared<Domain>(db,comm);
 		
 		Nx += 2;   Ny+=2;	Nz += 2;
 		Nx = Ny = Nz;	// Cubic domain
@@ -55,8 +48,7 @@ int main (int argc, char **argv)
 		//.......................................................................
 		// Assign the phase ID
 		//.......................................................................
-		char *id;
-		id = new char[N];
+		auto id = new char[N];
 		for (k=0;k<Nz;k++){
 			for (j=0;j<Ny;j++){
 				for (i=0;i<Nx;i++){
@@ -160,9 +152,7 @@ int main (int argc, char **argv)
     	ScaLBL_DeviceBarrier(); MPI_Barrier(comm);
     	ScaLBL_CopyToHost(&VEL[0],&dvc_vel[0],SIZE);
 
-		double err,value,Q;
-
-    	Q = 0.f;    	
+    	double Q = 0.f;    	
     	k=1;
     	for (j=1;j<Ny-1;j++){
     		for (i=1;i<Nx-1;i++){
@@ -176,7 +166,7 @@ int main (int argc, char **argv)
 
     	// respect backwards read / write!!!
 		printf("Inlet Flux: input=%f, output=%f \n",flux,Q);
-		err = fabs(flux + Q);
+		double err = fabs(flux + Q);
 		if (err > 1e-12){
 			error = 1;
 			printf("  Inlet error %f \n",err);
@@ -185,7 +175,7 @@ int main (int argc, char **argv)
 		// Consider a larger number of timesteps and simulate flow
 		double Fx, Fy, Fz;
 		double tau = 1.0;
-		double mu=(tau-0.5)/3.0;
+		//double mu=(tau-0.5)/3.0;
 		double rlx_setA=1.0/tau;
 		double rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
 		dout=1.f;

tests/TestForceD3Q19.cpp

@@ -457,24 +457,16 @@ int main (int argc, char **argv)
 		double *x = new double[1];
 		ASSERT(x!=NULL);
 	}
-
-    // set the error code
-    // Note: the error code should be consistent across all processors
-    int error = 0;
     
     int Np = 1;
-    int Q = 9;
+    //int Q = 9;
 
     double Fx = 1.0;
     double Fy = 1.0;
     double Fz = 1.0;
     
-    double *dist;    
-    double * Velocity;
-    
-    dist = new double [19*Np];
-    Velocity = new double [3*Np];
-
+    auto dist = new double [19*Np];
+    //auto Velocity  = new double [3*Np
 
     for (int n=0; n<Np; n++){
     	dist[n] = 0.3333333333333333;