merge scaLBL routines

Merge branch 'electrokinetic' of github.com:JamesEMcClure/LBPM-WIA into electrokinetic

CMakeLists.txt

@@ -112,7 +112,7 @@ ENDIF()
 ADD_CUSTOM_TARGET( build-test )
 ADD_CUSTOM_TARGET( build-examples )
 ADD_CUSTOM_TARGET( check COMMAND  make test  )
-ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool )
+ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace )
 
 
 # Check for CUDA
@@ -133,8 +133,6 @@ IF ( NOT ONLY_BUILD_DOCS )
     CONFIGURE_LBPM()
     CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
     CONFIGURE_LINE_COVERAGE()
-    INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/SharedPtr.cmake" )
-    CONFIGURE_SHARED_PTR( "${${PROJ}_INSTALL_DIR}/include" "std" )
     # Set the external library link list
     SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
 ENDIF()
@@ -156,6 +154,7 @@ IF ( NOT ONLY_BUILD_DOCS )
     ADD_PACKAGE_SUBDIRECTORY( analysis )
     ADD_PACKAGE_SUBDIRECTORY( IO )
     ADD_PACKAGE_SUBDIRECTORY( threadpool )
+    ADD_PACKAGE_SUBDIRECTORY( StackTrace )
     ADD_PACKAGE_SUBDIRECTORY( models )
     IF ( USE_CUDA )
         ADD_PACKAGE_SUBDIRECTORY( gpu )
@@ -164,7 +163,6 @@ IF ( NOT ONLY_BUILD_DOCS )
     ENDIF()
     INSTALL_LBPM_TARGET( lbpm-wia-library  )
     ADD_SUBDIRECTORY( tests )
-    ADD_SUBDIRECTORY( threadpool/test )
     ADD_SUBDIRECTORY( example )
     #ADD_SUBDIRECTORY( workflows )
     INSTALL_PROJ_LIB()

IO/Mesh.cpp

@@ -30,9 +30,9 @@
 */
 #include "Mesh.h"
 #include "common/Utilities.h"
-#include "shared_ptr.h"
 
 #include <limits>
+#include <memory>
 #include <stdint.h>
 
 namespace IO {

IO/Mesh.h

@@ -17,14 +17,13 @@
 #define MESH_INC
 
 #include <iostream>
+#include <memory>
 #include <string.h>
 #include <vector>
 
 #include "common/Array.h"
 #include "common/Communication.h"
 #include "analysis/PointList.h"
-#include "shared_ptr.h"
-
 
 
 namespace IO {

IO/MeshDatabase.h

@@ -18,9 +18,9 @@
 
 #include "IO/Mesh.h" 
 #include "common/MPI_Helpers.h"
-#include "shared_ptr.h"
 
 #include <iostream>
+#include <memory>
 #include <string.h>
 #include <vector>
 #include <map>

IO/Reader.h

@@ -17,12 +17,12 @@
 #define READER_INC
 
 #include <iostream>
+#include <memory>
 #include <string.h>
 #include <vector>
 
 #include "IO/Mesh.h"
 #include "IO/MeshDatabase.h"
-#include "shared_ptr.h"
 
 
 namespace IO {

IO/Writer.cpp

@@ -19,12 +19,12 @@
 #include "IO/silo.h"
 #include "common/MPI_Helpers.h"
 #include "common/Utilities.h"
-#include "shared_ptr.h"
 
 #include <sys/stat.h>
 #include <algorithm>
 #include <vector>
 #include <set>
+#include <memory>
 
 
 

IO/netcdf.cpp

@@ -204,6 +204,7 @@ std::vector<size_t> getAttDim( int fid, const std::string& att )
 {
     std::vector<size_t> dim(1,0);
     int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
+    CHECK_NC_ERR( err );
     return dim;
 }
 std::vector<std::string> getVarNames( int fid )

README.md

@@ -23,7 +23,7 @@ Configure, build & install procedure
 
 * edit configure script from sample_scripts directory and configure (e.g.)
 
-   `/path/to/LBPM-WIA/sample_scripts/configure_desktop`
+   `/path/to/LBPM/sample_scripts/configure_desktop`
 
 * compile and install
 

StackTrace/ErrorHandlers.h

@@ -0,0 +1,42 @@
+#ifndef included_StackTraceErrorHandlers
+#define included_StackTraceErrorHandlers
+
+
+#include "StackTrace/StackTrace.h"
+
+#include <functional>
+
+#include "mpi.h"
+
+
+namespace StackTrace
+{
+
+
+    /*!
+     * Set the error handler
+     * @param[in] abort     Function to terminate the program: abort(msg,type)
+     */
+    void setErrorHandler( std::function<void( const StackTrace::abort_error& )> abort );
+
+    //! Clear the error handler
+    void clearErrorHandler();
+
+
+    //! Set an error handler for MPI
+    void setMPIErrorHandler( MPI_Comm comm );
+
+    //! Clear an error handler for MPI
+    void clearMPIErrorHandler( MPI_Comm comm );
+
+
+    //! Initialize globalCallStack functionallity
+    void globalCallStackInitialize( MPI_Comm comm );
+
+    //! Clean up globalCallStack functionallity
+    void globalCallStackFinalize();
+
+
+} // namespace StackTrace
+
+#endif

StackTrace/Readme.txt

@@ -0,0 +1,4 @@
+This directory contains code external code released with permission under the license of this project.
+
+Original code and license are availible at:
+https://bitbucket.org/mberrill/StackTrace

StackTrace/StackTrace.h

@@ -16,41 +16,30 @@
 #ifndef included_StackTrace
 #define included_StackTrace
 
+#include <array>
 #include <functional>
 #include <iostream>
 #include <set>
 #include <thread>
 #include <vector>
 
-
-// Check for and include MPI
-// clang-format off
-#if defined(USE_MPI) || defined(USE_EXT_MPI)
-    #include "mpi.h"
-#elif defined(__has_include)
-    #if __has_include("mpi.h")
-        #include "mpi.h"
-    #else
-        typedef int MPI_Comm;
-    #endif
-#else
-    typedef int MPI_Comm;
-#endif
-// clang-format on
+#include "StackTrace/string_view.h"
 
 
 namespace StackTrace {
 
-
+//! Class to contain stack trace info for a single thread/process
 struct stack_info {
+    uint32_t line;
     void *address;
     void *address2;
-    std::string object;
-    std::string function;
-    std::string filename;
-    int line;
+    std::array<char, 56> object;
+    std::array<char, 48> objectPath;
+    std::array<char, 64> filename;
+    std::array<char, 64> filenamePath;
+    std::array<char, 256> function;
     //! Default constructor
-    stack_info() : address( nullptr ), address2( nullptr ), line( 0 ) {}
+    stack_info();
     //! Reset the stack
     void clear();
     //! Operator==
@@ -61,19 +50,22 @@ struct stack_info {
     int getAddressWidth() const;
     //! Print the stack info
     std::string print( int widthAddress = 16, int widthObject = 20, int widthFunction = 32 ) const;
+    //! Print the stack info
+    static void print( std::ostream &out, const std::vector<stack_info> &stack,
+        const StackTrace::string_view &prefix = "" );
+    //! Print the stack info
+    void print2(
+        char *txt, int widthAddress = 16, int widthObject = 20, int widthFunction = 32 ) const;
     //! Compute the number of bytes needed to store the object
     size_t size() const;
     //! Pack the data to a byte array, returning a pointer to the end of the data
     char *pack( char *ptr ) const;
     //! Unpack the data from a byte array, returning a pointer to the end of the data
     const char *unpack( const char *ptr );
-    //! Pack a vector of data to a memory block
-    static std::vector<char> packArray( const std::vector<stack_info> &data );
-    //! Unpack a vector of data from a memory block
-    static std::vector<stack_info> unpackArray( const char *data );
 };
 
 
+//! Class to contain stack trace info for multiple threads/processes
 struct multi_stack_info {
     int N;                                  // Number of threads/processes
     stack_info stack;                       // Current stack item
@@ -86,19 +78,69 @@ struct multi_stack_info {
     multi_stack_info &operator=( const std::vector<stack_info> & );
     //! Reset the stack
     void clear();
+    //! Is the stack empty
+    bool empty() const { return N == 0; }
     //! Add the given stack to the multistack
     void add( size_t len, const stack_info *stack );
+    //! Add the given stack to the multistack
+    void add( const multi_stack_info &stack );
+    //! Compute the number of bytes needed to store the object
+    size_t size() const;
+    //! Pack the data to a byte array, returning a pointer to the end of the data
+    char *pack( char *ptr ) const;
+    //! Unpack the data from a byte array, returning a pointer to the end of the data
+    const char *unpack( const char *ptr );
     //! Print the stack info
-    std::vector<std::string> print( const std::string &prefix = std::string() ) const;
+    std::vector<std::string> print( const StackTrace::string_view &prefix = "" ) const;
+    //! Print the stack info
+    void print( std::ostream &out, const StackTrace::string_view &prefix = "" ) const;
+    //! Print the stack info
+    std::string printString( const StackTrace::string_view &prefix = "" ) const;
 
 private:
-    void print2( const std::string &prefix, int w[3], std::vector<std::string> &text ) const;
+    template<class FUN>
+    void print2( int Np, char *prefix, int w[3], bool c, FUN &fun ) const;
     int getAddressWidth() const;
     int getObjectWidth() const;
     int getFunctionWidth() const;
 };
 
 
+//!< Terminate type
+enum class terminateType : uint8_t { signal, exception, abort, MPI, unknown };
+enum class printStackType : uint8_t { local = 1, threaded = 2, global = 3 };
+
+//!< Class to contain exception info from abort
+class abort_error : public std::exception
+{
+public:
+    std::string message;       //!< Abort message
+    std::string filename;      //!< File where abort was called
+    terminateType type;        //!< What caused the termination
+    printStackType stackType;  //!< Print the local stack, all threads, or global call stack
+    uint8_t signal;            //!< Signal number
+    int line;                  //!< Line number where abort was called
+    size_t bytes;              //!< Memory in use during abort
+    std::vector<void *> stack; //!< Local call stack for abort
+public:
+    virtual const char *what() const noexcept override;
+    abort_error();
+    virtual ~abort_error() {}
+
+private:
+    mutable std::string d_msg;
+};
+
+
+//!< Class to contain symbol information
+struct symbols_struct {
+    char type;
+    void *address;
+    std::array<char, 56> obj;
+    std::array<char, 56> objPath;
+};
+
+
 /*!
  * @brief  Get the current call stack
  * @details  This function returns the current call stack for the current thread
@@ -167,16 +209,18 @@ std::vector<stack_info> getStackInfo( const std::vector<void *> &address );
 
 
 //! Function to return the signal name
-std::string signalName( int signal );
+const char *signalName( int signal );
 
 
 /*!
  * Return the symbols from the current executable (not availible for all platforms)
- * @return      Returns 0 if sucessful
+ * @return      Returns the symbols loaded
  */
-int getSymbols( std::vector<void *> &address,
-                std::vector<char> &type,
-                std::vector<std::string> &obj );
+std::vector<symbols_struct> getSymbols();
+
+
+//! Clear internal symbol data
+void clearSymbols();
 
 
 /*!
@@ -193,20 +237,10 @@ std::string getExecutable();
 std::string getSymPaths();
 
 
-//!< Terminate type
-enum class terminateType { signal, exception };
-
-/*!
- * Set the error handlers
- * @param[in] abort     Function to terminate the program: abort(msg,type)
- */
-void setErrorHandlers( std::function<void( std::string, terminateType )> abort );
-
-
 /*!
  * Set the given signals to the handler
  * @param[in] signals   Signals to handle
- * @param[in] handler   Function to terminate the program: abort(msg,type)
+ * @param[in] handler   Function to terminate the program: abort(signal)
  */
 void setSignals( const std::vector<int> &signals, void ( *handler )( int ) );
 
@@ -215,10 +249,22 @@ void setSignals( const std::vector<int> &signals, void ( *handler )( int ) );
 void clearSignal( int signal );
 
 
+//! Clear a signal set by setSignals
+void clearSignals( const std::vector<int> &signals );
+
+
 //! Clear all signals set by setSignals
 void clearSignals();
 
 
+//! Raise a signal
+void raiseSignal( int signal );
+
+
+//! Default function to abort after catching a signal
+void terminateFunctionSignal( int signal );
+
+
 //! Return a list of all signals that can be caught
 std::vector<int> allSignalsToCatch();
 
@@ -227,19 +273,12 @@ std::vector<int> defaultSignalsToCatch();
 
 
 //! Get a list of the active threads
-std::set<std::thread::native_handle_type> activeThreads();
+std::vector<std::thread::native_handle_type> activeThreads();
 
 //! Get a handle to this thread
 std::thread::native_handle_type thisThread();
 
 
-//! Initialize globalCallStack functionallity
-void globalCallStackInitialize( MPI_Comm comm );
-
-//! Clean up globalCallStack functionallity
-void globalCallStackFinalize();
-
-
 /*!
  * @brief  Call system command
  * @details  This function calls a system command, waits for the program
@@ -248,7 +287,32 @@ void globalCallStackFinalize();
  * @param[out] exit_code    Exit code returned from child process
  * @return                  Returns string containing the output
  */
-std::string exec( const std::string &cmd, int &exit_code );
+std::string exec( const string_view &cmd, int &exit_code );
+
+
+/*!
+ * @brief  Create stack from string
+ * @details  This function creates the call stack from the string generated by print
+ * @param[in] str           Vector of strings containing call stack
+ * @return                  Returns the call stack
+ */
+multi_stack_info generateFromString( const std::vector<std::string> &str );
+
+
+/*!
+ * @brief  Create stack from string
+ * @details  This function creates the call stack from the string
+ * @param[in] str           String containing call stack
+ * @return                  Returns the call stack
+ */
+multi_stack_info generateFromString( const std::string &str );
+
+
+//! Set default stack type
+void setDefaultStackType( StackTrace::printStackType );
+
+//! Get default stack type
+StackTrace::printStackType getDefaultStackType();
 
 
 } // namespace StackTrace

StackTrace/Utilities.cpp

@@ -0,0 +1,334 @@
+#define NOMINMAX
+#include "StackTrace/Utilities.h"
+#include "StackTrace/ErrorHandlers.h"
+#include "StackTrace/StackTrace.h"
+
+#include <algorithm>
+#include <csignal>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <mutex>
+#include <sstream>
+#include <stdexcept>
+#include <typeinfo>
+
+#ifdef USE_MPI
+#include "mpi.h"
+#endif
+
+#ifdef USE_TIMER
+#include "MemoryApp.h"
+#endif
+
+#ifdef USE_GCOV
+extern "C" void __gcov_flush( void );
+#endif
+
+
+#define perr std::cerr
+
+
+// Detect the OS
+// clang-format off
+#if defined( WIN32 ) || defined( _WIN32 ) || defined( WIN64 ) || defined( _WIN64 ) || defined( _MSC_VER )
+    #define USE_WINDOWS
+#elif defined( __APPLE__ )
+    #define USE_MAC
+#elif defined( __linux ) || defined( __linux__ ) || defined( __unix ) || defined( __posix )
+    #define USE_LINUX
+    #define USE_NM
+#else
+    #error Unknown OS
+#endif
+// clang-format on
+
+
+// Include system dependent headers
+// clang-format off
+#ifdef USE_WINDOWS
+    #include <process.h>
+    #include <psapi.h>
+    #include <stdio.h>
+    #include <tchar.h>
+    #include <windows.h>
+#else
+    #include <dlfcn.h>
+    #include <execinfo.h>
+    #include <sched.h>
+    #include <sys/time.h>
+    #include <ctime>
+    #include <unistd.h>
+#endif
+#ifdef USE_LINUX
+    #include <malloc.h>
+#endif
+#ifdef USE_MAC
+    #include <mach/mach.h>
+    #include <sys/sysctl.h>
+    #include <sys/types.h>
+#endif
+// clang-format on
+
+
+#ifdef __GNUC__
+#define USE_ABI
+#include <cxxabi.h>
+#endif
+
+
+namespace StackTrace {
+
+
+/****************************************************************************
+ *  Function to find an entry                                                *
+ ****************************************************************************/
+template<class TYPE>
+inline size_t findfirst( const std::vector<TYPE> &X, TYPE Y )
+{
+    if ( X.empty() )
+        return 0;
+    size_t lower = 0;
+    size_t upper = X.size() - 1;
+    if ( X[lower] >= Y )
+        return lower;
+    if ( X[upper] < Y )
+        return upper;
+    while ( ( upper - lower ) != 1 ) {
+        size_t value = ( upper + lower ) / 2;
+        if ( X[value] >= Y )
+            upper = value;
+        else
+            lower = value;
+    }
+    return upper;
+}
+
+
+/****************************************************************************
+ *  Function to terminate the program                                        *
+ ****************************************************************************/
+static bool abort_throwException = false;
+static int force_exit            = 0;
+void Utilities::setAbortBehavior( bool throwException, int stackType )
+{
+    abort_throwException = throwException;
+    StackTrace::setDefaultStackType( static_cast<printStackType>( stackType ) );
+}
+void Utilities::abort( const std::string &message, const std::string &filename, const int line )
+{
+    abort_error err;
+    err.message   = message;
+    err.filename  = filename;
+    err.type      = terminateType::abort;
+    err.line      = line;
+    err.bytes     = Utilities::getMemoryUsage();
+    err.stackType = StackTrace::getDefaultStackType();
+    err.stack     = StackTrace::backtrace();
+    throw err;
+}
+static std::mutex terminate_mutex;
+static inline void callAbort()
+{
+#ifdef USE_GCOV
+    __gcov_flush();
+#endif
+    terminate_mutex.unlock();
+    std::abort();
+}
+void Utilities::terminate( const StackTrace::abort_error &err )
+{
+    // Lock mutex to ensure multiple threads do not try to abort simultaneously
+    terminate_mutex.lock();
+    // Clear the error handlers
+    clearErrorHandler();
+    // Print the message and abort
+    if ( force_exit > 1 ) {
+        callAbort();
+    } else if ( !abort_throwException ) {
+        // Use MPI_abort (will terminate all processes)
+        force_exit = 2;
+        perr << err.what();
+#if defined( USE_MPI ) || defined( HAVE_MPI )
+        int initialized = 0, finalized = 0;
+        MPI_Initialized( &initialized );
+        MPI_Finalized( &finalized );
+        if ( initialized != 0 && finalized == 0 ) {
+            clearMPIErrorHandler( MPI_COMM_WORLD );
+            MPI_Abort( MPI_COMM_WORLD, -1 );
+        }
+#endif
+        callAbort();
+    } else {
+        perr << err.what();
+        perr.flush();
+        callAbort();
+    }
+}
+
+
+/****************************************************************************
+ *  Functions to set the error handler                                       *
+ ****************************************************************************/
+static void setTerminateErrorHandler()
+{
+    // Set the terminate routine for runtime errors
+    StackTrace::setErrorHandler( Utilities::terminate );
+}
+void Utilities::setErrorHandlers()
+{
+#ifdef USE_MPI
+    setMPIErrorHandler( MPI_COMM_WORLD );
+    setMPIErrorHandler( MPI_COMM_SELF );
+#endif
+    setTerminateErrorHandler();
+}
+void Utilities::clearErrorHandlers()
+{
+#ifdef USE_MPI
+    clearMPIErrorHandler( MPI_COMM_WORLD );
+    clearMPIErrorHandler( MPI_COMM_SELF );
+#endif
+}
+
+
+/****************************************************************************
+ *  Function to get the memory usage                                         *
+ *  Note: this function should be thread-safe                                *
+ ****************************************************************************/
+// clang-format off
+#if defined( USE_MAC ) || defined( USE_LINUX )
+    // Get the page size on mac or linux
+    static size_t page_size = static_cast<size_t>( sysconf( _SC_PAGESIZE ) );
+#endif
+size_t Utilities::getSystemMemory()
+{
+    #if defined( USE_LINUX )
+        static long pages = sysconf( _SC_PHYS_PAGES );
+        size_t N_bytes    = pages * page_size;
+    #elif defined( USE_MAC )
+        int mib[2]    = { CTL_HW, HW_MEMSIZE };
+        u_int namelen = sizeof( mib ) / sizeof( mib[0] );
+        uint64_t size;
+        size_t len = sizeof( size );
+        size_t N_bytes = 0;
+        if ( sysctl( mib, namelen, &size, &len, nullptr, 0 ) == 0 )
+            N_bytes = size;
+    #elif defined( USE_WINDOWS )
+        MEMORYSTATUSEX status;
+        status.dwLength = sizeof( status );
+        GlobalMemoryStatusEx( &status );
+        size_t N_bytes = status.ullTotalPhys;
+    #else
+        #error Unknown OS
+    #endif
+    return N_bytes;
+}
+size_t Utilities::getMemoryUsage()
+{
+    #ifdef USE_TIMER
+        size_t N_bytes = MemoryApp::getTotalMemoryUsage();
+    #else
+        #if defined( USE_LINUX )
+            struct mallinfo meminfo = mallinfo();
+            size_t size_hblkhd      = static_cast<unsigned int>( meminfo.hblkhd );
+            size_t size_uordblks    = static_cast<unsigned int>( meminfo.uordblks );
+            size_t N_bytes          = size_hblkhd + size_uordblks;
+        #elif defined( USE_MAC )
+            struct task_basic_info t_info;
+            mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
+            if ( KERN_SUCCESS !=
+                 task_info( mach_task_self(), TASK_BASIC_INFO, (task_info_t) &t_info, &t_info_count ) ) {
+                return 0;
+            }
+            size_t N_bytes = t_info.virtual_size;
+        #elif defined( USE_WINDOWS )
+            PROCESS_MEMORY_COUNTERS memCounter;
+            GetProcessMemoryInfo( GetCurrentProcess(), &memCounter, sizeof( memCounter ) );
+            size_t N_bytes = memCounter.WorkingSetSize;
+        #else
+            #error Unknown OS
+        #endif
+    #endif
+    return N_bytes;
+}
+// clang-format on
+
+
+/****************************************************************************
+ *  Functions to get the time and timer resolution                           *
+ ****************************************************************************/
+#if defined( USE_WINDOWS )
+double Utilities::time()
+{
+    LARGE_INTEGER end, f;
+    QueryPerformanceFrequency( &f );
+    QueryPerformanceCounter( &end );
+    double time = ( (double) end.QuadPart ) / ( (double) f.QuadPart );
+    return time;
+}
+double Utilities::tick()
+{
+    LARGE_INTEGER f;
+    QueryPerformanceFrequency( &f );
+    double resolution = ( (double) 1.0 ) / ( (double) f.QuadPart );
+    return resolution;
+}
+#elif defined( USE_LINUX ) || defined( USE_MAC )
+double Utilities::time()
+{
+    timeval current_time;
+    gettimeofday( &current_time, nullptr );
+    double time = ( (double) current_time.tv_sec ) + 1e-6 * ( (double) current_time.tv_usec );
+    return time;
+}
+double Utilities::tick()
+{
+    timeval start, end;
+    gettimeofday( &start, nullptr );
+    gettimeofday( &end, nullptr );
+    while ( end.tv_sec == start.tv_sec && end.tv_usec == start.tv_usec )
+        gettimeofday( &end, nullptr );
+    double resolution = ( (double) ( end.tv_sec - start.tv_sec ) ) +
+                        1e-6 * ( (double) ( end.tv_usec - start.tv_usec ) );
+    return resolution;
+}
+#else
+#error Unknown OS
+#endif
+
+
+/****************************************************************************
+ *  Cause a segfault                                                         *
+ ****************************************************************************/
+void Utilities::cause_segfault()
+{
+    int *ptr = nullptr;
+    ptr[0]   = 0;
+}
+
+
+/****************************************************************************
+ *  Call system command                                                      *
+ ****************************************************************************/
+std::string Utilities::exec( const string_view &cmd, int &exit_code )
+{
+    return StackTrace::exec( cmd, exit_code );
+}
+
+
+/****************************************************************************
+ *  Get the type name                                                        *
+ ****************************************************************************/
+std::string Utilities::getTypeName( const std::type_info &id )
+{
+    std::string name = id.name();
+#if defined( USE_ABI )
+    int status;
+    name = abi::__cxa_demangle( name.c_str(), 0, 0, &status );
+#endif
+    return name;
+}
+
+
+} // namespace StackTrace

StackTrace/Utilities.h

@@ -0,0 +1,117 @@
+#ifndef included_StackTrace_Utilities
+#define included_StackTrace_Utilities
+
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <typeinfo>
+
+#include "StackTrace/StackTrace.h"
+#include "StackTrace/string_view.h"
+
+
+namespace StackTrace {
+namespace Utilities {
+
+
+/*!
+ * Aborts the run after printing an error message with file and
+ * line number information.
+ */
+void abort( const std::string &message, const std::string &filename, const int line );
+
+
+/*!
+ * Set the behavior of abort
+ * @param throwException    Throw an exception instead of MPI_Abort (default is false)
+ * @param stackType         Type of stack to get (1: thread local stack, 2: all threads, 3: global)
+ */
+void setAbortBehavior( bool throwException, int stackType = 2 );
+
+
+//! Function to terminate the application
+void terminate( const StackTrace::abort_error &err );
+
+
+//! Function to set the error handlers
+void setErrorHandlers();
+
+
+//! Function to clear the error handlers
+void clearErrorHandlers();
+
+
+/*!
+ * Function to get the memory availible.
+ * This function will return the total memory availible
+ * Note: depending on the implimentation, this number may be rounded to
+ * to a multiple of the page size.
+ * If this function fails, it will return 0.
+ */
+size_t getSystemMemory();
+
+
+/*!
+ * Function to get the memory usage.
+ * This function will return the total memory used by the application.
+ * Note: depending on the implimentation, this number may be rounded to
+ * to a multiple of the page size.
+ * If this function fails, it will return 0.
+ */
+size_t getMemoryUsage();
+
+
+//! Function to get an arbitrary point in time
+double time();
+
+
+//! Function to get the resolution of time
+double tick();
+
+
+/*!
+ * Sleep for X ms
+ * @param N         Time to sleep (ms)
+ */
+inline void sleep_ms( int N ) { std::this_thread::sleep_for( std::chrono::milliseconds( N ) ); }
+
+
+/*!
+ * Sleep for X s
+ * @param N         Time to sleep (s)
+ */
+inline void sleep_s( int N ) { std::this_thread::sleep_for( std::chrono::seconds( N ) ); }
+
+
+//! Cause a segfault
+void cause_segfault();
+
+
+/*!
+ * @brief  Call system command
+ * @details  This function calls a system command, waits for the program
+ *   to execute, captures and returns the output and exit code.
+ * @param[in] cmd           Command to execute
+ * @param[out] exit_code    Exit code returned from child process
+ * @return                  Returns string containing the output
+ */
+std::string exec( const StackTrace::string_view &cmd, int &exit_code );
+
+
+//! Return the hopefully demangled name of the given type
+std::string getTypeName( const std::type_info &id );
+
+
+//! Return the hopefully demangled name of the given type
+template<class TYPE>
+inline std::string getTypeName()
+{
+    return getTypeName( typeid( TYPE ) );
+}
+
+
+} // namespace Utilities
+} // namespace StackTrace
+
+
+#endif

StackTrace/string_view.h

@@ -0,0 +1,193 @@
+#ifndef included_StackTrace_stringView
+#define included_StackTrace_stringView
+
+#include <cstring>
+#include <ostream>
+
+namespace StackTrace {
+
+// string_view
+class string_view
+{
+public:
+    // Constants:
+    static constexpr size_t npos = size_t( -1 );
+
+    // Constructions
+    constexpr string_view() noexcept : d_data( nullptr ), d_size( 0 ) {}
+    constexpr string_view( string_view&& ) noexcept      = default;
+    constexpr string_view( const string_view& ) noexcept = default;
+    constexpr string_view( const char* s ) : d_data( s ), d_size( s ? strlen( s ) : 0 ) {}
+    constexpr string_view( const char* s, size_t count ) : d_data( s ), d_size( count ) {}
+    inline string_view( const std::string& s ) : d_data( s.data() ), d_size( s.size() ) {}
+
+    // Assignment
+    constexpr string_view& operator=( string_view&& other ) noexcept = default;
+    constexpr string_view& operator=( const string_view& other ) noexcept = default;
+
+    // Iterators
+    constexpr const char* begin() const noexcept { return d_data; }
+    constexpr const char* end() const noexcept { return d_data + d_size; }
+    constexpr const char* cbegin() const noexcept { return begin(); }
+    constexpr const char* cend() const noexcept { return end(); }
+
+    // capacity
+    constexpr size_t size() const noexcept { return d_size; }
+    constexpr size_t length() const noexcept { return d_size; }
+    constexpr bool empty() const noexcept { return d_size == 0; }
+
+    // Element access
+    constexpr const char& operator[]( size_t pos ) const
+    {
+        if ( pos >= d_size )
+            throw std::out_of_range( "string_view[]" );
+        return d_data[pos];
+    }
+    constexpr const char& at( size_t pos ) const
+    {
+        if ( pos >= d_size )
+            throw std::out_of_range( "string_view::at()" );
+        return d_data[pos];
+    }
+    constexpr const char& front() const
+    {
+        if ( d_size == 0 )
+            throw std::out_of_range( "front()" );
+        return d_data[0];
+    }
+    constexpr const char& back() const
+    {
+        if ( d_size == 0 )
+            throw std::out_of_range( "back()" );
+        return d_data[size() - 1];
+    }
+    constexpr const char* data() const noexcept { return d_data; }
+
+    // Swap data
+    void swap( string_view& other ) noexcept
+    {
+        std::swap( d_data, other.d_data );
+        std::swap( d_size, other.d_size );
+    }
+
+    // String operations
+    size_t copy( char* dest, size_t n, size_t pos = 0 ) const
+    {
+        if ( pos > size() )
+            throw std::out_of_range( "string_view::copy()" );
+        const size_t rlen = std::min( n, size() - pos );
+        memcpy( dest, data() + pos, rlen );
+        return rlen;
+    }
+    constexpr string_view substr( size_t pos = 0, size_t n = npos ) const
+    {
+        if ( pos > size() )
+            throw std::out_of_range( "string_view::substr()" );
+        return string_view( data() + pos, std::min( n, size() - pos ) );
+    }
+
+    // Find
+    constexpr size_t find( char ch, size_t pos = 0 ) const noexcept
+    {
+        for ( size_t i = pos; i < d_size; i++ )
+            if ( d_data[i] == ch )
+                return i;
+        return std::string::npos;
+    }
+    constexpr size_t find( string_view v, size_t pos = 0 ) const noexcept
+    {
+        size_t i = pos;
+        size_t N = v.size();
+        if ( N == 0 || N > ( d_size - pos ) )
+            return std::string::npos;
+        while ( i < ( d_size - N + 1 ) ) {
+            size_t j = 0;
+            for ( j = 0; j < N && i + j < d_size; j++ )
+                if ( d_data[i + j] != v[j] )
+                    break;
+            if ( j == N )
+                return i;
+            i++;
+        }
+        return std::string::npos;
+    }
+
+    // compare()
+    constexpr int compare( const string_view& other ) const noexcept
+    {
+        int N      = std::min( size(), other.size() );
+        int result = 0;
+        for ( int i = 0; i < N && result == 0; i++ )
+            if ( d_data[i] != other[i] )
+                result = d_data[i] < other[i] ? -( i + 1 ) : ( i + 1 );
+        if ( result == 0 )
+            result = size() == other.size() ? 0 : size() < other.size() ? -1 : 1;
+        return result;
+    }
+    constexpr int compare( size_t pos1, size_t n1, string_view other ) const
+    {
+        return substr( pos1, n1 ).compare( other );
+    }
+    constexpr int compare( size_t pos1, size_t n1, string_view other, size_t pos2, size_t n2 ) const
+    {
+        return substr( pos1, n1 ).compare( other.substr( pos2, n2 ) );
+    }
+    constexpr int compare( char const* s ) const { return compare( string_view( s ) ); }
+    constexpr int compare( size_t pos1, size_t n1, char const* s ) const
+    {
+        return substr( pos1, n1 ).compare( string_view( s ) );
+    }
+    constexpr int compare( size_t pos1, size_t n1, char const* s, size_t n2 ) const
+    {
+        return substr( pos1, n1 ).compare( string_view( s, n2 ) );
+    }
+
+    explicit operator std::string() const { return std::string( begin(), end() ); }
+    std::string to_string() const { return std::string( begin(), end() ); }
+
+private:
+    const char* d_data;
+    size_t d_size;
+};
+
+
+// Non-member functions:
+constexpr inline bool operator==( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) == 0;
+}
+constexpr inline bool operator!=( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) != 0;
+}
+constexpr inline bool operator<( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) < 0;
+}
+
+constexpr inline bool operator<=( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) <= 0;
+}
+constexpr inline bool operator>( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) > 0;
+}
+constexpr inline bool operator>=( const string_view& lhs, const string_view& rhs ) noexcept
+{
+    return lhs.compare( rhs ) >= 0;
+}
+inline std::string to_string( const string_view& v ) { return std::string( v.begin(), v.end() ); }
+inline string_view to_string_view( std::string const& s )
+{
+    return string_view( s.data(), s.size() );
+}
+inline std::ostream& operator<<( std::ostream& out, const string_view& s )
+{
+    out << s.data();
+    return out;
+}
+
+} // namespace StackTrace
+
+#endif

analysis/ElectroChemistry.cpp

@@ -0,0 +1,228 @@
+#include "analysis/ElectroChemistry.h"
+
+ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr <Domain> dm):
+	Dm(dm),
+	fillData(dm->Comm,dm->rank_info,{dm->Nx-2,dm->Ny-2,dm->Nz-2},{1,1,1},0,1)
+{
+	
+    MPI_Comm_dup(dm->Comm,&comm);
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	ChemicalPotential.resize(Nx,Ny,Nz);       ChemicalPotential.fill(0);
+	ElectricalPotential.resize(Nx,Ny,Nz);      ElectricalPotential.fill(0);
+	Pressure.resize(Nx,Ny,Nz);     	Pressure.fill(0);
+	Rho.resize(Nx,Ny,Nz);       	Rho.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("electrokinetic.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("electrokinetic.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"TBD TBD\n");				
+		}
+	}
+
+}
+
+ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){
+	if (Dm->rank()==0){
+		fclose(TIMELOG);
+	}
+}
+
+void ElectroChemistryAnalyzer::SetParams(){
+	
+}
+
+void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep){
+
+	int i,j,k;
+	Poisson.getElectricPotential(ElectricalPotential);
+	
+	/* local sub-domain averages */
+	double rho_avg_local[Ion.number_ion_species];
+	double rho_mu_avg_local[Ion.number_ion_species];
+	double rho_mu_fluctuation_local[Ion.number_ion_species];	
+	double rho_psi_avg_local[Ion.number_ion_species];
+	double rho_psi_fluctuation_local[Ion.number_ion_species];
+	/* global averages */
+	double rho_avg_global[Ion.number_ion_species];
+	double rho_mu_avg_global[Ion.number_ion_species];
+	double rho_mu_fluctuation_global[Ion.number_ion_species];
+	double rho_psi_avg_global[Ion.number_ion_species];
+	double rho_psi_fluctuation_global[Ion.number_ion_species];
+	
+	for (int ion=0; ion<Ion.number_ion_species; ion++){
+		rho_avg_local[ion] = 0.0;
+		rho_mu_avg_local[ion] = 0.0;
+		rho_psi_avg_local[ion] = 0.0;
+		Ion.getIonConcentration(Rho,ion);
+		/* Compute averages for each ion */
+		for (k=1; k<Nz; k++){
+			for (j=1; j<Ny; j++){
+				for (i=1; i<Nx; i++){
+					rho_avg_local[ion] += Rho(i,j,k);
+					rho_mu_avg_local[ion] += Rho(i,j,k)*Rho(i,j,k);
+					rho_psi_avg_local[ion] += Rho(i,j,k)*ElectricalPotential(i,j,k);
+				}
+			}
+		}
+		rho_avg_global[ion]=sumReduce( Dm->Comm, rho_avg_local[ion]);
+		rho_mu_avg_global[ion]=sumReduce( Dm->Comm, rho_mu_avg_local[ion]);
+		rho_psi_avg_global[ion]=sumReduce( Dm->Comm, rho_psi_avg_local[ion]);
+		
+		rho_mu_avg_global[ion] /= rho_avg_global[ion];
+		rho_psi_avg_global[ion] /= rho_avg_global[ion];
+	}
+	
+	for (int ion=0; ion<Ion.number_ion_species; ion++){
+		rho_mu_fluctuation_local[ion] = 0.0;
+		rho_psi_fluctuation_local[ion] = 0.0;
+		/* Compute averages for each ion */
+		for (k=1; k<Nz; k++){
+			for (j=1; j<Ny; j++){
+				for (i=1; i<Nx; i++){
+					rho_mu_fluctuation_local[ion] += (Rho(i,j,k)*Rho(i,j,k) - rho_mu_avg_global[ion]);
+					rho_psi_fluctuation_local[ion] += (Rho(i,j,k)*ElectricalPotential(i,j,k) - rho_psi_avg_global[ion]);
+				}
+			}
+		}
+		rho_mu_fluctuation_global[ion]=sumReduce( Dm->Comm, rho_mu_fluctuation_local[ion]);
+		rho_psi_fluctuation_global[ion]=sumReduce( Dm->Comm, rho_psi_fluctuation_local[ion]);
+	}
+	
+	if (Dm->rank()==0){	
+		fprintf(TIMELOG,"%i ",timestep); 
+		for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
+		}
+		fflush(TIMELOG);
+	}
+/*	else{
+		fprintf(TIMELOG,"%i ",timestep); 
+		for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
+		}
+		fflush(TIMELOG);
+	} */
+}
+
+void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep){
+	
+	auto vis_db =  input_db->getDatabase( "Visualization" );
+    char VisName[40];
+
+    IO::initialize("","silo","false");
+    // Create the MeshDataStruct    
+    visData.resize(1);
+
+    visData[0].meshName = "domain";
+    visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+    auto ElectricPotential = std::make_shared<IO::Variable>();
+    std::vector<shared_ptr<IO::Variable>> IonConcentration;
+    for (int ion=0; ion<Ion.number_ion_species; ion++){
+        IonConcentration.push_back(std::make_shared<IO::Variable>());
+    }
+    auto VxVar = std::make_shared<IO::Variable>();
+    auto VyVar = std::make_shared<IO::Variable>();
+    auto VzVar = std::make_shared<IO::Variable>();
+    
+    if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
+    	ElectricPotential->name = "ElectricPotential";
+    	ElectricPotential->type = IO::VariableType::VolumeVariable;
+    	ElectricPotential->dim = 1;
+    	ElectricPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricPotential);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_concentration", true )){
+    	for (int ion=0; ion<Ion.number_ion_species; ion++){
+    		sprintf(VisName,"IonConcentration_%i",ion+1);
+    		IonConcentration[ion]->name = VisName;
+    		IonConcentration[ion]->type = IO::VariableType::VolumeVariable;
+    		IonConcentration[ion]->dim = 1;
+    		IonConcentration[ion]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonConcentration[ion]);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim = 1;
+        VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VxVar);
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim = 1;
+        VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VyVar);
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim = 1;
+        VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VzVar);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
+    	ASSERT(visData[0].vars[0]->name=="ElectricPotential");
+    	Poisson.getElectricPotential(ElectricalPotential);
+    	Array<double>& ElectricPotentialData = visData[0].vars[0]->data;
+    	fillData.copy(ElectricalPotential,ElectricPotentialData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_concentration", true )){
+    	for (int ion=0; ion<Ion.number_ion_species; ion++){
+    		sprintf(VisName,"IonConcentration_%i",ion+1);
+    		IonConcentration[ion]->name = VisName;
+    		ASSERT(visData[0].vars[1+ion]->name==VisName);
+    		Array<double>& IonConcentrationData = visData[0].vars[1+ion]->data;
+    		Ion.getIonConcentration(Rho,ion);
+    		fillData.copy(Rho,IonConcentrationData);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+0]->name=="Velocity_x");
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+1]->name=="Velocity_y");
+    	ASSERT(visData[0].vars[1+Ion.number_ion_species+2]->name=="Velocity_z");
+    	Stokes.getVelocity(Vel_x,Vel_y,Vel_z);
+    	Array<double>& VelxData = visData[0].vars[1+Ion.number_ion_species+0]->data;
+    	Array<double>& VelyData = visData[0].vars[1+Ion.number_ion_species+1]->data;
+    	Array<double>& VelzData = visData[0].vars[1+Ion.number_ion_species+2]->data;
+    	fillData.copy(Vel_x,VelxData);
+    	fillData.copy(Vel_y,VelyData);
+    	fillData.copy(Vel_z,VelzData);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "write_silo", true ))
+    	IO::writeData( timestep, visData, comm );
+
+/*    if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
+    	char CurrentIDFilename[40];
+    	sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
+    	Averages.AggregateLabels(CurrentIDFilename);
+    }
+*/	
+}

analysis/ElectroChemistry.h

@@ -0,0 +1,59 @@
+/*
+ *  averaging tools for electrochemistry
+ */
+
+#ifndef ElectroChem_INC
+#define ElectroChem_INC
+
+#include <vector>
+#include "common/Domain.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "analysis/distance.h"
+#include "analysis/Minkowski.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+#include "models/IonModel.h"
+#include "models/PoissonSolver.h"
+#include "models/StokesModel.h"
+
+class ElectroChemistryAnalyzer{
+public:
+    MPI_Comm comm;
+    int tag;
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+
+    //...........................................................................
+    int Nx,Ny,Nz;
+	DoubleArray Rho;	           // density field 
+	DoubleArray ChemicalPotential;	   // density field 
+	DoubleArray ElectricalPotential;	// density field 
+	DoubleArray Pressure; 	// pressure field
+	DoubleArray Vel_x;		// velocity field
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+	DoubleArray SDs;
+
+	ElectroChemistryAnalyzer(std::shared_ptr <Domain> Dm);
+	~ElectroChemistryAnalyzer();
+	
+	void SetParams();
+	void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep);
+	void WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep);
+
+private:
+    std::vector<IO::MeshDataStruct> visData;
+    fillHalo<double> fillData;
+	FILE *TIMELOG;
+};
+#endif
+

analysis/GreyPhase.cpp

@@ -0,0 +1,259 @@
+#include "analysis/GreyPhase.h"
+
+// Constructor
+GreyPhaseAnalysis::GreyPhaseAnalysis(std::shared_ptr <Domain> dm):
+	Dm(dm)
+{
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	// Global arrays
+    SDs.resize(Nx,Ny,Nz);           SDs.fill(0);
+	Porosity.resize(Nx,Ny,Nz);      Porosity.fill(0);
+	//PhaseID.resize(Nx,Ny,Nz);       PhaseID.fill(0);
+	Rho_n.resize(Nx,Ny,Nz);       	Rho_n.fill(0);
+	Rho_w.resize(Nx,Ny,Nz);       	Rho_w.fill(0);
+	Pressure.resize(Nx,Ny,Nz);      Pressure.fill(0);
+	//Phi.resize(Nx,Ny,Nz);         	Phi.fill(0);
+	//DelPhi.resize(Nx,Ny,Nz);        DelPhi.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	//.........................................
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("timelog.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("timelog.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");				
+		}
+	}
+}
+
+
+// Destructor
+GreyPhaseAnalysis::~GreyPhaseAnalysis()
+{
+
+}
+
+void GreyPhaseAnalysis::Write(int timestep)
+{
+
+}
+
+void GreyPhaseAnalysis::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B, double GreyPorosity)
+{
+	Fx = force_x;
+	Fy = force_y;
+	Fz = force_z;
+	rho_n = rhoA;
+	rho_w = rhoB;
+	nu_n = (tauA-0.5)/3.f;
+	nu_w = (tauB-0.5)/3.f;
+	gamma_wn = 6.0*alpha;
+	beta = B;
+    grey_porosity = GreyPorosity;
+}
+
+void GreyPhaseAnalysis::Basic(){
+	int i,j,k,n,imin,jmin,kmin,kmax;
+
+	// If external boundary conditions are set, do not average over the inlet
+	kmin=1; kmax=Nz-1;
+	imin=jmin=1;
+	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
+	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
+
+	Water_local.reset();
+	Oil_local.reset();
+	double count_w = 0.0;
+	double count_n = 0.0;
+	for (k=kmin; k<kmax; k++){
+		for (j=jmin; j<Ny-1; j++){
+			for (i=imin; i<Nx-1; i++){
+				n = k*Nx*Ny + j*Nx + i;
+				// Compute volume averages
+				if ( Dm->id[n] > 0 ){
+					// compute density
+					double nA = Rho_n(n);
+					double nB = Rho_w(n);				
+					double phi = (nA-nB)/(nA+nB);					
+					double porosity = Porosity(n);
+					Water_local.M += rho_w*nB*porosity;
+					Water_local.Px += porosity*rho_w*nB*Vel_x(n);
+					Water_local.Py += porosity*rho_w*nB*Vel_y(n);
+					Water_local.Pz += porosity*rho_w*nB*Vel_z(n);
+					Oil_local.M += rho_n*nA*porosity;
+					Oil_local.Px += porosity*rho_n*nA*Vel_x(n);
+					Oil_local.Py += porosity*rho_n*nA*Vel_y(n);
+					Oil_local.Pz += porosity*rho_n*nA*Vel_z(n);
+
+					if ( phi > 0.99 ){
+						Oil_local.p += Pressure(n);
+					    //Oil_local.p += pressure*(rho_n*nA)/(rho_n*nA+rho_w*nB);
+						count_n += 1.0;
+					}
+					else if ( phi < -0.99 ){
+						Water_local.p += Pressure(n);
+					    //Water_local.p += pressure*(rho_w*nB)/(rho_n*nA+rho_w*nB);
+						count_w += 1.0;
+					}
+				}
+			}
+		}
+	}
+	Oil.M=sumReduce( Dm->Comm, Oil_local.M);
+	Oil.Px=sumReduce( Dm->Comm, Oil_local.Px);
+	Oil.Py=sumReduce( Dm->Comm, Oil_local.Py);
+	Oil.Pz=sumReduce( Dm->Comm, Oil_local.Pz);
+
+	Water.M=sumReduce( Dm->Comm, Water_local.M);
+	Water.Px=sumReduce( Dm->Comm, Water_local.Px);
+	Water.Py=sumReduce( Dm->Comm, Water_local.Py);
+	Water.Pz=sumReduce( Dm->Comm, Water_local.Pz);
+
+
+	//Oil.p /= Oil.M;	
+	//Water.p /= Water.M;
+	count_w=sumReduce( Dm->Comm, count_w);
+	count_n=sumReduce( Dm->Comm, count_n);
+	if (count_w > 0.0)
+		Water.p=sumReduce( Dm->Comm, Water_local.p) / count_w;
+	else 
+		Water.p = 0.0;
+	if (count_n > 0.0)
+		Oil.p=sumReduce( Dm->Comm, Oil_local.p) / count_n;
+	else 
+		Oil.p = 0.0;
+
+	// check for NaN
+	bool err=false;
+	if (Water.M != Water.M) err=true;
+	if (Water.p != Water.p) err=true;
+	if (Water.Px != Water.Px) err=true;
+	if (Water.Py != Water.Py) err=true;
+	if (Water.Pz != Water.Pz) err=true;
+
+	if (Oil.M != Oil.M) err=true;
+	if (Oil.p != Oil.p) err=true;
+	if (Oil.Px != Oil.Px) err=true;
+	if (Oil.Py != Oil.Py) err=true;
+	if (Oil.Pz != Oil.Pz) err=true;
+	
+	if (Dm->rank() == 0){
+		double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+		double dir_x = 0.0;
+		double dir_y = 0.0;
+		double dir_z = 0.0;
+		if (force_mag > 0.0){
+			dir_x = Fx/force_mag;
+			dir_y = Fy/force_mag;
+			dir_z = Fz/force_mag;
+		}
+		else {
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+		}
+		if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){
+			// compute the pressure drop
+			double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0;
+			double length = ((Nz-2)*Dm->nprocz());
+			force_mag -= pressure_drop/length;
+		}
+		if (force_mag == 0.0){
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+			force_mag = 1.0;
+		}
+		saturation=Water.M/(Water.M + Oil.M); // assume constant density
+		water_flow_rate=grey_porosity*saturation*(Water.Px*dir_x + Water.Py*dir_y + Water.Pz*dir_z)/Water.M;
+		oil_flow_rate  =grey_porosity*(1.0-saturation)*(Oil.Px*dir_x + Oil.Py*dir_y + Oil.Pz*dir_z)/Oil.M;
+
+		double h = Dm->voxel_length;		
+        //TODO check if need greyporosity or domain porosity ? - compare to analytical solution
+		double krn = h*h*nu_n*oil_flow_rate / force_mag ;
+		double krw = h*h*nu_w*water_flow_rate / force_mag;
+		//printf("   water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*oil_flow_rate, Water.p, Oil.p); 
+		fflush(TIMELOG); 
+	}
+
+	if (err==true){
+		// exception if simulation produceds NaN
+		printf("GreyPhaseAnalysis.cpp: NaN encountered, may need to check simulation parameters \n");
+	}	
+	ASSERT(err==false);
+}
+/*
+inline void InterfaceTransportMeasures( double beta, double rA, double rB, double nA, double nB, 
+		double nx, double ny, double nz, double ux, double uy, double uz, interface &I){
+	
+	double A1,A2,A3,A4,A5,A6;
+	double B1,B2,B3,B4,B5,B6;
+	double nAB,delta;
+	// Instantiate mass transport distributions
+	// Stationary value - distribution 0
+	nAB = 1.0/(nA+nB);
+	//...............................................
+	// q = 0,2,4
+	// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nx;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
+	B1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
+	A2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
+	B2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+
+	//...............................................
+	// Cq = {0,1,0}
+	delta = beta*nA*nB*nAB*0.1111111111111111*ny;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A3 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
+	B3 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
+	A4 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
+	B4 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+
+	//...............................................
+	// q = 4
+	// Cq = {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nz;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A5 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
+	B5 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
+	A6 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
+	B6 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+	
+	double unx = (A1-A2);
+	double uny = (A3-A4);
+	double unz = (A5-A6);
+	double uwx = (B1-B2);
+	double uwy = (B3-B4);
+	double uwz = (B5-B6);
+	
+	I.Mn += rA*nA;
+	I.Mw += rB*nB;
+	I.Pnx += rA*nA*unx;
+	I.Pny += rA*nA*uny;
+	I.Pnz += rA*nA*unz;
+	I.Pwx += rB*nB*uwx;
+	I.Pwy += rB*nB*uwy;
+	I.Pwz += rB*nB*uwz;
+	I.Kn += rA*nA*(unx*unx + uny*uny + unz*unz);
+	I.Kw += rB*nB*(uwx*uwx + uwy*uwy + uwz*uwz);
+
+}
+*/

analysis/GreyPhase.h

@@ -0,0 +1,71 @@
+/*
+ * Sub-phase averaging tools
+ */
+
+#ifndef GreyPhase_INC
+#define GreyPhase_INC
+
+#include <vector>
+#include "common/ScaLBL.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+
+class GreyPhase{
+ public:
+  double p;
+  double M,Px,Py,Pz;
+  void reset(){
+    p=M=Px=Py=Pz=0.0;
+  }
+
+ private:
+};
+
+class GreyPhaseAnalysis{
+public:
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+    double grey_porosity;
+	// outputs
+	double saturation,water_flow_rate, oil_flow_rate;
+
+	//simulation outputs (averaged values)
+	GreyPhase Water, Oil;
+	GreyPhase Water_local, Oil_local;
+	//...........................................................................
+    int Nx,Ny,Nz;
+	//IntArray PhaseID;		// Phase ID array 
+	DoubleArray SDs;		// contains porosity map 
+	DoubleArray Porosity;		// contains porosity map 
+	DoubleArray Rho_n;	// density field 
+	DoubleArray Rho_w;	// density field 
+	//DoubleArray Phi;		// phase indicator field
+	//DoubleArray DelPhi;		// Magnitude of Gradient of the phase indicator field
+	DoubleArray Pressure; 	// pressure field
+	DoubleArray Vel_x;		// velocity field
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+
+	GreyPhaseAnalysis(std::shared_ptr <Domain> Dm);
+	~GreyPhaseAnalysis();
+	
+	void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta, double GreyPorosity);
+	void Basic();
+	void Write(int time);
+
+private:
+	FILE *TIMELOG;
+};
+
+#endif
+

analysis/Minkowski.cpp

@@ -15,11 +15,9 @@
 */
 #include "analysis/Minkowski.h"
 #include "analysis/pmmc.h"
+#include "analysis/analysis.h"
 #include "common/Domain.h"
 #include "common/Communication.h"
-#include "analysis/analysis.h"
-
-#include "shared_ptr.h"
 #include "common/Utilities.h"
 #include "common/MPI_Helpers.h"
 #include "IO/MeshDatabase.h"
@@ -28,6 +26,8 @@
 
 #include "ProfilerApp.h"
 
+#include <memory>
+
 
 #define PI 3.14159265359
 
@@ -39,6 +39,10 @@ Minkowski::Minkowski(std::shared_ptr <Domain> dm):
 	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
 	Volume=double((Nx-2)*(Ny-2)*(Nz-2))*double(Dm->nprocx()*Dm->nprocy()*Dm->nprocz());
 	
+	id.resize(Nx,Ny,Nz);         	id.fill(0);
+	label.resize(Nx,Ny,Nz);         label.fill(0);
+	distance.resize(Nx,Ny,Nz);      distance.fill(0);
+
 	if (Dm->rank()==0){
 		LOGFILE = fopen("minkowski.csv","a+");
 		if (fseek(LOGFILE,0,SEEK_SET) == fseek(LOGFILE,0,SEEK_CUR))
@@ -57,22 +61,6 @@ Minkowski::~Minkowski()
     if ( LOGFILE!=NULL ) { fclose(LOGFILE); }
 }
 
-double Minkowski::V(){
-	return Vi_global;
-}
-
-double Minkowski::A(){
-	return Ai_global;
-}
-
-double Minkowski::J(){
-	return Ji_global;
-}
-
-double Minkowski::X(){
-	return Xi_global;
-}
-
 void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
 {
     PROFILE_START("ComputeScalar");
@@ -118,7 +106,21 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
 					Xi -= 0.5;
 				}
 				// Euler characteristic -- each vertex shared by four cubes
-				Xi += 0.25*double(object.VertexCount);
+				//Xi += 0.25*double(object.VertexCount);
+				// check if vertices are at corners
+				for (int idx=0; idx<object.VertexCount; idx++){
+				  /*auto P1 = object.vertex.coords(idx);
+					if ( remainder(P1.x,1.0)==0.0 && remainder(P1.y,1.0)==0.0  && remainder(P1.z,1.0)==0.0 ){
+					  Xi += 0.125;
+					}
+					else
+				  */
+					Xi += 0.25;
+				}
+				/*double nside_extern = double(npts);
+				double nside_intern = double(npts)-3.0;
+				EulerChar=0.0;
+				if (npts > 0)	EulerChar = (0.25*nvert - nside_intern - 0.5*nside_extern + nface); */
 			}
 		}
 	}
@@ -133,6 +135,9 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
 			}
 		}
 	}
+	// convert X for 2D manifold to 3D object
+	Xi *= 0.5;
+	
 	MPI_Barrier(Dm->Comm);
 	// Phase averages
 	MPI_Allreduce(&Vi,&Vi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
@@ -143,6 +148,156 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
     PROFILE_STOP("ComputeScalar");
 }
 
+
+void Minkowski::MeasureObject(){
+	/*
+	 *  compute the distance to an object 
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 *    0 - labels the object
+	 *    1 - labels the rest of the 
+	 */
+        //DoubleArray smooth_distance(Nx,Ny,Nz);
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				distance(i,j,k) =2.0*double(id(i,j,k))-1.0;
+			}
+		}
+	}	
+	CalcDist(distance,id,*Dm);
+	//Mean3D(distance,smooth_distance);
+	//Eikonal(distance, id, *Dm, 20, {true, true, true});
+	ComputeScalar(distance,0.0);
+}
+
+
+void Minkowski::MeasureObject(double factor, const DoubleArray &Phi){
+	/*
+	 *  compute the distance to an object 
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 *    0 - labels the object
+	 *    1 - labels the rest of the 
+	 */
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				distance(i,j,k) =2.0*double(id(i,j,k))-1.0;
+			}
+		}
+	}	
+	CalcDist(distance,id,*Dm);
+	
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				double value = Phi(i,j,k);
+				double dist_value = distance(i,j,k);
+				if (dist_value < 2.5 &&  dist_value > -2.5) {
+					double new_distance = factor*log((1.0+value)/(1.0-value));
+					if (dist_value*new_distance < 0.0 )
+						new_distance = (-1.0)*new_distance;
+					distance(i,j,k) = new_distance;					
+				}
+			}
+		}
+	}
+	
+	ComputeScalar(distance,0.0);
+
+}
+
+
+int Minkowski::MeasureConnectedPathway(){
+	/*
+	 * compute the connected pathway for object with LABEL in id field
+	 * compute the labels for connected components
+	 * compute the distance to the connected pathway
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 */
+	
+	char LABEL = 0;
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if (id(i,j,k) == LABEL){
+					distance(i,j,k) = 1.0;
+				}
+				else
+					distance(i,j,k) = -1.0;
+			}
+		}
+	}
+	
+	// Extract only the connected part of NWP
+	double vF=0.0; 
+	n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
+//	int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
+	MPI_Barrier(Dm->Comm);
+	
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if ( label(i,j,k) == 0){
+					id(i,j,k) = 0;
+				}
+				else{
+					id(i,j,k) = 1;
+				}
+			}
+		}
+	}
+	MeasureObject();
+	return n_connected_components; 
+}
+
+int Minkowski::MeasureConnectedPathway(double factor, const DoubleArray &Phi){
+	/*
+	 * compute the connected pathway for object with LABEL in id field
+	 * compute the labels for connected components
+	 * compute the distance to the connected pathway
+	 * 
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 */
+	
+	char LABEL = 0;
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if (id(i,j,k) == LABEL){
+					distance(i,j,k) = 1.0;
+				}
+				else
+					distance(i,j,k) = -1.0;
+			}
+		}
+	}
+	
+	// Extract only the connected part of NWP
+	double vF=0.0; 
+	n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
+//	int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
+	MPI_Barrier(Dm->Comm);
+	
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if ( label(i,j,k) == 0){
+					id(i,j,k) = 0;
+				}
+				else{
+					id(i,j,k) = 1;
+				}
+			}
+		}
+	}
+	MeasureObject(factor,Phi);
+	return n_connected_components; 
+}
+
+
 void Minkowski::PrintAll()
 {
 	if (Dm->rank()==0){

analysis/Minkowski.h

@@ -17,14 +17,16 @@
 #ifndef Minkowski_INC
 #define Minkowski_INC
 
+#include <memory>
 #include <vector>
 
 #include "analysis/dcel.h"
 #include "common/Domain.h"
 #include "common/Communication.h"
 #include "analysis/analysis.h"
+#include "analysis/distance.h"
+#include "analysis/filters.h"
 
-#include "shared_ptr.h"
 #include "common/Utilities.h"
 #include "common/MPI_Helpers.h"
 #include "IO/MeshDatabase.h"
@@ -45,6 +47,9 @@ class Minkowski{
 public:
 	//...........................................................................
 	std::shared_ptr <Domain> Dm;
+	Array <char> id;
+	Array <int> label;
+	Array <double> distance;
 	//...........................................................................
 	// Averaging variables
 	//...........................................................................
@@ -52,19 +57,32 @@ public:
 	double Ai,Ji,Xi,Vi;
 	// Global averages (all processes)
 	double Ai_global,Ji_global,Xi_global,Vi_global;
-
+	int n_connected_components;
 	//...........................................................................
 	int Nx,Ny,Nz;
-	double V();
-	double A();
-	double J();
-	double X();
-	
+	double V(){
+		return Vi;
+	}
+	double A(){
+		return Ai;
+	}
+	double H(){
+		return Ji;
+	}
+	double X(){
+		return Xi;
+	}
+		
 	//..........................................................................
 	Minkowski(){};//NULL CONSTRUCTOR
 	Minkowski(std::shared_ptr <Domain> Dm);
 	~Minkowski();
+	void MeasureObject();
+	void MeasureObject(double factor, const DoubleArray &Phi);
+	int MeasureConnectedPathway();
+	int MeasureConnectedPathway(double factor, const DoubleArray &Phi);
 	void ComputeScalar(const DoubleArray& Field, const double isovalue);
+
 	void PrintAll();
 
 };

analysis/SubPhase.cpp

@@ -0,0 +1,730 @@
+#include "analysis/SubPhase.h"
+
+// Constructor
+SubPhase::SubPhase(std::shared_ptr <Domain> dm):
+	Dm(dm)
+{
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	morph_w = std::shared_ptr<Minkowski>(new Minkowski(Dm));
+	morph_n = std::shared_ptr<Minkowski>(new Minkowski(Dm));
+	morph_i = std::shared_ptr<Minkowski>(new Minkowski(Dm));
+
+	// Global arrays
+	PhaseID.resize(Nx,Ny,Nz);       PhaseID.fill(0);
+	Label_WP.resize(Nx,Ny,Nz);      Label_WP.fill(0);
+	Label_NWP.resize(Nx,Ny,Nz);     Label_NWP.fill(0);
+	Rho_n.resize(Nx,Ny,Nz);       	Rho_n.fill(0);
+	Rho_w.resize(Nx,Ny,Nz);       	Rho_w.fill(0);
+	Pressure.resize(Nx,Ny,Nz);      Pressure.fill(0);
+	Phi.resize(Nx,Ny,Nz);         	Phi.fill(0);
+	DelPhi.resize(Nx,Ny,Nz);        DelPhi.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
+	//.........................................
+
+	//.........................................
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		SUBPHASE = fopen("subphase.csv","r");
+		if (SUBPHASE != NULL)
+			fclose(SUBPHASE);
+		else
+			WriteHeader=true;
+
+		SUBPHASE = fopen("subphase.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
+			fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");				
+			fprintf(SUBPHASE,"pwc pwd pnc pnd ");						// pressures 
+			fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni ");				// mass 
+			fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x ");	// momentum 
+			fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y ");			
+			fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z ");			
+			fprintf(SUBPHASE,"Kwc Kwd Kwi Knc Knd Kni ");				// kinetic energy
+			fprintf(SUBPHASE,"Vwc Awc Hwc Xwc ");					 	// wc region 
+			fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd ");					// wd region
+			fprintf(SUBPHASE,"Vnc Anc Hnc Xnc ");					 	// nc region
+			fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd ");					// nd region
+			fprintf(SUBPHASE,"Vi Ai Hi Xi ");					 		// interface region 
+			fprintf(SUBPHASE,"Vic Aic Hic Xic Nic\n");					// interface region 
+
+			// stress tensor?
+		}
+
+	}
+	else{
+		char LocalRankString[8];
+		sprintf(LocalRankString,"%05d",Dm->rank());
+		char LocalRankFilename[40];
+		sprintf(LocalRankFilename,"%s%s","subphase.csv.",LocalRankString);
+		SUBPHASE = fopen(LocalRankFilename,"a+");
+		//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
+		fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");				
+		fprintf(SUBPHASE,"pwc pwd pnc pnd ");						// pressures 
+		fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni ");				// mass 
+		fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x ");	// momentum 
+		fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y ");			
+		fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z ");			
+		fprintf(SUBPHASE,"Kwc Kwd Kwi Knc Knd Kni ");				// kinetic energy
+		fprintf(SUBPHASE,"Vwc Awc Hwc Xwc ");					 	// wc region 
+		fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd ");					// wd region
+		fprintf(SUBPHASE,"Vnc Anc Hnc Xnc ");					 	// nc region
+		fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd ");					// nd region
+		fprintf(SUBPHASE,"Vi Ai Hi Xi ");					 		// interface region 
+		fprintf(SUBPHASE,"Vic Aic Hic Xic Nic\n");					// interface region 
+	}
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("timelog.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("timelog.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");				
+		}
+	}
+}
+
+
+// Destructor
+SubPhase::~SubPhase()
+{
+    if ( SUBPHASE!=NULL ) { fclose(SUBPHASE); }
+
+}
+
+void SubPhase::Write(int timestep)
+{
+	if (Dm->rank()==0){
+		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn); 
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Py, gwd.Py, giwn.Pwy, gnc.Py, gnd.Py, giwn.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Pz, gwd.Pz, giwn.Pwz, gnc.Pz, gnd.Pz, giwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.K, gwd.K, giwn.Kw, gnc.K, gnd.K, giwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.V, gwc.A, gwc.H, gwc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",gwd.V, gwd.A, gwd.H, gwd.X, gwd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gnc.V, gnc.A, gnc.H, gnc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",gnd.V, gnd.A, gnd.H, gnd.X, gnd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",giwn.V, giwn.A, giwn.H, giwn.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i\n",giwnc.V, giwnc.A, giwnc.H, giwnc.X, giwnc.Nc);
+		fflush(SUBPHASE);
+	}
+	else{
+		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Py, wd.Py, iwn.Pwy, nc.Py, nd.Py, iwn.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Pz, wd.Pz, iwn.Pwz, nc.Pz, nd.Pz, iwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.K, wd.K, iwn.Kw, nc.K, nd.K, iwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.V, wc.A, wc.H, wc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",wd.V, wd.A, wd.H, wd.X, wd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",nc.V, nc.A, nc.H, nc.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",nd.V, nd.A, nd.H, nd.X, nd.Nc);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",iwn.V, iwn.A, iwn.H, iwn.X);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g\n",iwnc.V, iwnc.A, iwnc.H, iwnc.X);
+	}
+
+}
+
+void SubPhase::SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double B)
+{
+	Fx = force_x;
+	Fy = force_y;
+	Fz = force_z;
+	rho_n = rhoA;
+	rho_w = rhoB;
+	nu_n = (tauA-0.5)/3.f;
+	nu_w = (tauB-0.5)/3.f;
+	gamma_wn = 5.796*alpha;
+	beta = B;
+}
+
+void SubPhase::Basic(){
+	int i,j,k,n,imin,jmin,kmin,kmax;
+
+	// If external boundary conditions are set, do not average over the inlet
+	kmin=1; kmax=Nz-1;
+	imin=jmin=1;
+	/*// If inlet/outlet layers exist use these as default
+	if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
+	if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
+	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
+	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
+	*/
+	nb.reset(); wb.reset();
+
+	double count_w = 0.0;
+	double count_n = 0.0;
+	
+	for (k=0; k<Nz; k++){
+		for (j=0; j<Ny; j++){
+			for (i=0; i<Nx; i++){
+				n = k*Nx*Ny + j*Nx + i;
+				// Compute volume averages
+				if ( Dm->id[n] > 0 ){
+					// compute density
+					double nA = Rho_n(n);
+					double nB = Rho_w(n);
+					double phi = (nA-nB)/(nA+nB);
+					Phi(n) = phi;
+				}
+			}
+		}
+	}
+	
+	for (k=kmin; k<kmax; k++){
+		for (j=jmin; j<Ny-1; j++){
+			for (i=imin; i<Nx-1; i++){
+				n = k*Nx*Ny + j*Nx + i;
+				// Compute volume averages
+				if ( Dm->id[n] > 0 ){
+					// compute density
+					double nA = Rho_n(n);
+					double nB = Rho_w(n);
+					double phi = (nA-nB)/(nA+nB);					
+					if ( phi > 0.0 ){
+						nA = 1.0;
+						nb.V += 1.0;
+						nb.M += nA*rho_n;						
+						// velocity
+						nb.Px += rho_n*nA*Vel_x(n);
+						nb.Py += rho_n*nA*Vel_y(n);
+						nb.Pz += rho_n*nA*Vel_z(n);
+					}
+					else{
+						nB = 1.0;
+						wb.M += nB*rho_w;
+						wb.V += 1.0;
+
+						// velocity
+						wb.Px += rho_w*nB*Vel_x(n);
+						wb.Py += rho_w*nB*Vel_y(n);
+						wb.Pz += rho_w*nB*Vel_z(n);
+					}
+					if ( phi > 0.99 ){
+						nb.p += Pressure(n);
+						count_n += 1.0;
+					}
+					else if ( phi < -0.99 ){
+						wb.p += Pressure(n);
+						count_w += 1.0;
+					}
+				}
+			}
+		}
+	}
+	gwb.V=sumReduce( Dm->Comm, wb.V);
+	gnb.V=sumReduce( Dm->Comm, nb.V);
+	gwb.M=sumReduce( Dm->Comm, wb.M);
+	gnb.M=sumReduce( Dm->Comm, nb.M);
+	gwb.Px=sumReduce( Dm->Comm, wb.Px);
+	gwb.Py=sumReduce( Dm->Comm, wb.Py);
+	gwb.Pz=sumReduce( Dm->Comm, wb.Pz);
+	gnb.Px=sumReduce( Dm->Comm, nb.Px);
+	gnb.Py=sumReduce( Dm->Comm, nb.Py);
+	gnb.Pz=sumReduce( Dm->Comm, nb.Pz);
+	
+	count_w=sumReduce( Dm->Comm, count_w);
+	count_n=sumReduce( Dm->Comm, count_n);
+	if (count_w > 0.0)
+		gwb.p=sumReduce( Dm->Comm, wb.p) / count_w;
+	else 
+		gwb.p = 0.0;
+	if (count_n > 0.0)
+		gnb.p=sumReduce( Dm->Comm, nb.p) / count_n;
+	else 
+		gnb.p = 0.0;
+
+	// check for NaN
+	bool err=false;
+	if (gwb.V != gwb.V) err=true;
+	if (gnb.V != gnb.V) err=true;
+	if (gwb.p != gwb.p) err=true;
+	if (gnb.p != gnb.p) err=true;
+	if (gwb.Px != gwb.Px) err=true;
+	if (gwb.Py != gwb.Py) err=true;
+	if (gwb.Pz != gwb.Pz) err=true;
+	if (gnb.Px != gnb.Px) err=true;
+	if (gnb.Py != gnb.Py) err=true;
+	if (gnb.Pz != gnb.Pz) err=true;	
+	
+	if (Dm->rank() == 0){
+		double force_mag = sqrt(Fx*Fx+Fy*Fy+Fz*Fz);
+		double dir_x = 0.0;
+		double dir_y = 0.0;
+		double dir_z = 0.0;
+		if (force_mag > 0.0){
+			dir_x = Fx/force_mag;
+			dir_y = Fy/force_mag;
+			dir_z = Fz/force_mag;
+		}
+		else {
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+		}
+		if (Dm->BoundaryCondition == 1 || Dm->BoundaryCondition == 2 || Dm->BoundaryCondition == 3 || Dm->BoundaryCondition == 4 ){
+			// compute the pressure drop
+			double pressure_drop = (Pressure(Nx*Ny + Nx + 1) - 1.0) / 3.0;
+			double length = ((Nz-2)*Dm->nprocz());
+			force_mag -= pressure_drop/length;
+		}
+		if (force_mag == 0.0){
+			// default to z direction
+			dir_x = 0.0;
+			dir_y = 0.0;
+			dir_z = 1.0;
+			force_mag = 1.0;
+		}
+		double saturation=gwb.V/(gwb.V + gnb.V);
+		double water_flow_rate=gwb.V*(gwb.Px*dir_x + gwb.Py*dir_y + gwb.Pz*dir_z)/gwb.M / Dm->Volume;
+		double not_water_flow_rate=gnb.V*(gnb.Px*dir_x + gnb.Py*dir_y + gnb.Pz*dir_z)/gnb.M/ Dm->Volume;
+		//double total_flow_rate = water_flow_rate + not_water_flow_rate;
+		//double fractional_flow = water_flow_rate / total_flow_rate;
+
+		double h = Dm->voxel_length;		
+		double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
+		double krw = h*h*nu_w*water_flow_rate / force_mag;
+		//printf("   water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p); 
+		fflush(TIMELOG);
+	}
+	if (err==true){
+		// exception if simulation produceds NaN
+		printf("SubPhase.cpp: NaN encountered, may need to check simulation parameters \n");
+	}	
+	ASSERT(err==false);
+}
+
+inline void InterfaceTransportMeasures( double beta, double rA, double rB, double nA, double nB, 
+		double nx, double ny, double nz, double ux, double uy, double uz, interface &I){
+	
+	double A1,A2,A3,A4,A5,A6;
+	double B1,B2,B3,B4,B5,B6;
+	double nAB,delta;
+	// Instantiate mass transport distributions
+	// Stationary value - distribution 0
+	nAB = 1.0/(nA+nB);
+	//...............................................
+	// q = 0,2,4
+	// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nx;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
+	B1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
+	A2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
+	B2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+
+	//...............................................
+	// Cq = {0,1,0}
+	delta = beta*nA*nB*nAB*0.1111111111111111*ny;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A3 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
+	B3 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
+	A4 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
+	B4 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+
+	//...............................................
+	// q = 4
+	// Cq = {0,0,1}
+	delta = beta*nA*nB*nAB*0.1111111111111111*nz;
+	if (!(nA*nB*nAB>0)) delta=0;
+	A5 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
+	B5 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
+	A6 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
+	B6 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+	
+	double unx = (A1-A2);
+	double uny = (A3-A4);
+	double unz = (A5-A6);
+	double uwx = (B1-B2);
+	double uwy = (B3-B4);
+	double uwz = (B5-B6);
+	
+	I.Mn += rA*nA;
+	I.Mw += rB*nB;
+	I.Pnx += rA*nA*unx;
+	I.Pny += rA*nA*uny;
+	I.Pnz += rA*nA*unz;
+	I.Pwx += rB*nB*uwx;
+	I.Pwy += rB*nB*uwy;
+	I.Pwz += rB*nB*uwz;
+	I.Kn += rA*nA*(unx*unx + uny*uny + unz*unz);
+	I.Kw += rB*nB*(uwx*uwx + uwy*uwy + uwz*uwz);
+
+}
+
+void SubPhase::Full(){
+	int i,j,k,n,imin,jmin,kmin,kmax;
+
+	// If external boundary conditions are set, do not average over the inlet
+	kmin=1; kmax=Nz-1;
+	/*if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == 0) kmin=4;
+	if (Dm->BoundaryCondition > 0 && Dm->BoundaryCondition != 5 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
+	*/
+	imin=jmin=1;
+	/*// If inlet layers exist use these as default
+	 * NOTE -- excluding inlet / outlet will screw up topological averages!!!
+	if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
+	if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
+	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
+	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
+	*/
+	nd.reset();	nc.reset(); wd.reset();	wc.reset();	iwn.reset();	iwnc.reset();
+
+ 	Dm->CommunicateMeshHalo(Phi);
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				// Compute all of the derivatives using finite differences
+				double fx = 0.5*(Phi(i+1,j,k) - Phi(i-1,j,k));
+				double fy = 0.5*(Phi(i,j+1,k) - Phi(i,j-1,k));
+				double fz = 0.5*(Phi(i,j,k+1) - Phi(i,j,k-1));
+				DelPhi(i,j,k) = sqrt(fx*fx+fy*fy+fz*fz);
+			}
+		}
+	}
+ 	Dm->CommunicateMeshHalo(DelPhi);
+
+ 	/*  Set up geometric analysis of each region */
+	
+ 	// non-wetting
+	for (k=0; k<Nz; k++){
+		for (j=0; j<Ny; j++){
+			for (i=0; i<Nx; i++){
+				n = k*Nx*Ny+j*Nx+i;
+				if (!(Dm->id[n] > 0)){
+					// Solid phase
+					morph_n->id(i,j,k) = 1;
+
+				}
+				else if (Phi(n) > 0.0){
+					// non-wetting phase
+					morph_n->id(i,j,k) = 0;
+				}
+				else {
+					// wetting phase
+					morph_n->id(i,j,k) = 1;
+				}
+			}
+		}
+	}
+	// measure the whole object
+	morph_n->MeasureObject();//0.5/beta,Phi);
+	nd.V = morph_n->V(); 
+	nd.A = morph_n->A(); 
+	nd.H = morph_n->H(); 
+	nd.X = morph_n->X(); 
+	// measure only the connected part
+	nd.Nc = morph_n->MeasureConnectedPathway();//0.5/beta,Phi);
+	nc.V = morph_n->V(); 
+	nc.A = morph_n->A(); 
+	nc.H = morph_n->H(); 
+	nc.X = morph_n->X(); 
+	// update disconnected part
+	nd.V -= nc.V;
+	nd.A -= nc.A;
+	nd.H -= nc.H;
+	nd.X -= nc.X;
+
+	// compute global entities
+	gnc.V=sumReduce( Dm->Comm, nc.V);
+	gnc.A=sumReduce( Dm->Comm, nc.A);
+	gnc.H=sumReduce( Dm->Comm, nc.H);
+	gnc.X=sumReduce( Dm->Comm, nc.X);
+	gnd.V=sumReduce( Dm->Comm, nd.V);
+	gnd.A=sumReduce( Dm->Comm, nd.A);
+	gnd.H=sumReduce( Dm->Comm, nd.H);
+	gnd.X=sumReduce( Dm->Comm, nd.X);
+	gnd.Nc = nd.Nc;
+ 	// wetting
+	for (k=0; k<Nz; k++){
+		for (j=0; j<Ny; j++){
+			for (i=0; i<Nx; i++){
+				n = k*Nx*Ny+j*Nx+i;
+				if (!(Dm->id[n] > 0)){
+					// Solid phase
+					morph_w->id(i,j,k) = 1;
+
+				}
+				else if (Phi(n) < 0.0){
+					// wetting phase
+					morph_w->id(i,j,k) = 0;
+				}
+				else {
+					// non-wetting phase
+					morph_w->id(i,j,k) = 1;
+				}
+			}
+		}
+	}	
+	morph_w->MeasureObject();//-0.5/beta,Phi);
+	wd.V = morph_w->V(); 
+	wd.A = morph_w->A(); 
+	wd.H = morph_w->H(); 
+	wd.X = morph_w->X(); 
+	// measure only the connected part
+	wd.Nc = morph_w->MeasureConnectedPathway();//-0.5/beta,Phi);
+	wc.V = morph_w->V(); 
+	wc.A = morph_w->A(); 
+	wc.H = morph_w->H(); 
+	wc.X = morph_w->X(); 
+	// update disconnected part
+	wd.V -= wc.V;
+	wd.A -= wc.A;
+	wd.H -= wc.H;
+	wd.X -= wc.X;
+	// compute global entities
+	gwc.V=sumReduce( Dm->Comm, wc.V);
+	gwc.A=sumReduce( Dm->Comm, wc.A);
+	gwc.H=sumReduce( Dm->Comm, wc.H);
+	gwc.X=sumReduce( Dm->Comm, wc.X);
+	gwd.V=sumReduce( Dm->Comm, wd.V);
+	gwd.A=sumReduce( Dm->Comm, wd.A);
+	gwd.H=sumReduce( Dm->Comm, wd.H);
+	gwd.X=sumReduce( Dm->Comm, wd.X);
+	gwd.Nc = wd.Nc;
+	
+ 	/*  Set up geometric analysis of interface region */
+	for (k=0; k<Nz; k++){
+		for (j=0; j<Ny; j++){
+			for (i=0; i<Nx; i++){
+				n = k*Nx*Ny+j*Nx+i;
+				if (!(Dm->id[n] > 0)){
+					// Solid phase
+					morph_i->id(i,j,k) = 1;
+				}
+				else if (DelPhi(n) > 1e-4){
+					// interface
+					morph_i->id(i,j,k) = 0;
+				}
+				else {
+					// not interface
+					morph_i->id(i,j,k) = 1;
+				}
+			}
+		}
+	}	
+	morph_i->MeasureObject();
+	iwn.V = morph_i->V(); 
+	iwn.A = morph_i->A(); 
+	iwn.H = morph_i->H(); 
+	iwn.X = morph_i->X(); 
+	giwn.V=sumReduce( Dm->Comm, iwn.V);
+	giwn.A=sumReduce( Dm->Comm, iwn.A);
+	giwn.H=sumReduce( Dm->Comm, iwn.H);
+	giwn.X=sumReduce( Dm->Comm, iwn.X);
+	// measure only the connected part
+	iwnc.Nc = morph_i->MeasureConnectedPathway();
+	iwnc.V = morph_i->V(); 
+	iwnc.A = morph_i->A(); 
+	iwnc.H = morph_i->H(); 
+	iwnc.X = morph_i->X(); 
+	giwnc.V=sumReduce( Dm->Comm, iwnc.V);
+	giwnc.A=sumReduce( Dm->Comm, iwnc.A);
+	giwnc.H=sumReduce( Dm->Comm, iwnc.H);
+	giwnc.X=sumReduce( Dm->Comm, iwnc.X);
+	giwnc.Nc = iwnc.Nc;
+
+	double vol_nc_bulk = 0.0;
+	double vol_wc_bulk = 0.0;
+	double vol_nd_bulk = 0.0;
+	double vol_wd_bulk = 0.0;
+	for (k=kmin; k<kmax; k++){
+		for (j=jmin; j<Ny-1; j++){
+			for (i=imin; i<Nx-1; i++){
+				n = k*Nx*Ny + j*Nx + i;
+				// Compute volume averages
+				if ( Dm->id[n] > 0 ){
+					// compute density
+					double nA = Rho_n(n);
+					double nB = Rho_w(n);
+					double phi = (nA-nB)/(nA+nB);
+					double ux = Vel_x(n);
+					double uy = Vel_y(n);
+					double uz = Vel_z(n);
+					
+					if (DelPhi(n) > 1e-3){
+						// interface region
+						double nx = 0.5*(Phi(i+1,j,k)-Phi(i-1,j,k));
+						double ny = 0.5*(Phi(i,j+1,k)-Phi(i,j-1,k));
+						double nz = 0.5*(Phi(i,j,k+1)-Phi(i,j,k-1));
+						InterfaceTransportMeasures(  beta,  rho_w,  rho_n,  nA, nB, nx, ny, nz, ux, uy, uz, iwn);
+					}
+					else if ( phi > 0.0){
+						if (morph_n->label(i,j,k) > 0 ){
+							vol_nd_bulk += 1.0;
+							nd.p += Pressure(n);
+						}
+						else{
+							vol_nc_bulk += 1.0;
+							nc.p += Pressure(n);
+						}
+					}
+					else{
+						// water region
+						if (morph_w->label(i,j,k) > 0 ){
+							vol_wd_bulk += 1.0;
+							wd.p += Pressure(n);
+						}
+						else{
+							vol_wc_bulk += 1.0;
+							wc.p += Pressure(n);
+						}
+					}
+					if ( phi > 0.0){
+						if (morph_n->label(i,j,k) > 0 ){
+							nA = 1.0;
+							nd.M += nA*rho_n;						
+							nd.Px += nA*rho_n*ux;
+							nd.Py += nA*rho_n*uy;
+							nd.Pz += nA*rho_n*uz;
+							nd.K += nA*rho_n*(ux*ux + uy*uy + uz*uz);
+						}
+						else{
+							nA = 1.0;
+							nc.M += nA*rho_n;						
+							nc.Px += nA*rho_n*ux;
+							nc.Py += nA*rho_n*uy;
+							nc.Pz += nA*rho_n*uz;
+							nc.K += nA*rho_n*(ux*ux + uy*uy + uz*uz);
+						}
+					}
+					else{
+						// water region
+						if (morph_w->label(i,j,k) > 0 ){
+							nB = 1.0;
+							wd.M += nB*rho_w;						
+							wd.Px += nB*rho_w*ux;
+							wd.Py += nB*rho_w*uy;
+							wd.Pz += nB*rho_w*uz;
+							wd.K += nB*rho_w*(ux*ux + uy*uy + uz*uz);
+						}
+						else{
+							nB = 1.0;
+							wc.M += nB*rho_w;						
+							wc.Px += nB*rho_w*ux;
+							wc.Py += nB*rho_w*uy;
+							wc.Pz += nB*rho_w*uz;
+							wc.K += nB*rho_w*(ux*ux + uy*uy + uz*uz);
+						}
+					}
+				}
+			}
+		}
+	}
+
+	gnd.M=sumReduce( Dm->Comm, nd.M);
+	gnd.Px=sumReduce( Dm->Comm, nd.Px);
+	gnd.Py=sumReduce( Dm->Comm, nd.Py);
+	gnd.Pz=sumReduce( Dm->Comm, nd.Pz);
+	gnd.K=sumReduce( Dm->Comm, nd.K);
+
+	gwd.M=sumReduce( Dm->Comm, wd.M);
+	gwd.Px=sumReduce( Dm->Comm, wd.Px);
+	gwd.Py=sumReduce( Dm->Comm, wd.Py);
+	gwd.Pz=sumReduce( Dm->Comm, wd.Pz);
+	gwd.K=sumReduce( Dm->Comm, wd.K);
+	
+	gnc.M=sumReduce( Dm->Comm, nc.M);
+	gnc.Px=sumReduce( Dm->Comm, nc.Px);
+	gnc.Py=sumReduce( Dm->Comm, nc.Py);
+	gnc.Pz=sumReduce( Dm->Comm, nc.Pz);
+	gnc.K=sumReduce( Dm->Comm, nc.K);
+
+	gwc.M=sumReduce( Dm->Comm, wc.M);
+	gwc.Px=sumReduce( Dm->Comm, wc.Px);
+	gwc.Py=sumReduce( Dm->Comm, wc.Py);
+	gwc.Pz=sumReduce( Dm->Comm, wc.Pz);
+	gwc.K=sumReduce( Dm->Comm, wc.K);
+	
+	giwn.Mn=sumReduce( Dm->Comm, iwn.Mn);
+	giwn.Pnx=sumReduce( Dm->Comm, iwn.Pnx);
+	giwn.Pny=sumReduce( Dm->Comm, iwn.Pny);
+	giwn.Pnz=sumReduce( Dm->Comm, iwn.Pnz);
+	giwn.Kn=sumReduce( Dm->Comm, iwn.Kn);
+	giwn.Mw=sumReduce( Dm->Comm, iwn.Mw);
+	giwn.Pwx=sumReduce( Dm->Comm, iwn.Pwx);
+	giwn.Pwy=sumReduce( Dm->Comm, iwn.Pwy);
+	giwn.Pwz=sumReduce( Dm->Comm, iwn.Pwz);
+	giwn.Kw=sumReduce( Dm->Comm, iwn.Kw);
+	
+	// pressure averaging
+	gnc.p=sumReduce( Dm->Comm, nc.p);
+	gnd.p=sumReduce( Dm->Comm, nd.p);
+	gwc.p=sumReduce( Dm->Comm, wc.p);
+	gwd.p=sumReduce( Dm->Comm, wd.p);
+
+	if (vol_wc_bulk > 0.0)
+		wc.p = wc.p /vol_wc_bulk;
+	if (vol_nc_bulk > 0.0)
+		nc.p = nc.p /vol_nc_bulk;
+	if (vol_wd_bulk > 0.0)
+		wd.p = wd.p /vol_wd_bulk;
+	if (vol_nd_bulk > 0.0)
+		nd.p = nd.p /vol_nd_bulk;
+
+	vol_wc_bulk=sumReduce( Dm->Comm, vol_wc_bulk);
+	vol_wd_bulk=sumReduce( Dm->Comm, vol_wd_bulk);
+	vol_nc_bulk=sumReduce( Dm->Comm, vol_nc_bulk);
+	vol_nd_bulk=sumReduce( Dm->Comm, vol_nd_bulk);
+	
+	if (vol_wc_bulk > 0.0)
+		gwc.p = gwc.p /vol_wc_bulk;
+	if (vol_nc_bulk > 0.0)
+		gnc.p = gnc.p /vol_nc_bulk;
+	if (vol_wd_bulk > 0.0)
+		gwd.p = gwd.p /vol_wd_bulk;
+	if (vol_nd_bulk > 0.0)
+		gnd.p = gnd.p /vol_nd_bulk;
+}
+
+
+void SubPhase::AggregateLabels( const std::string& filename )
+{
+	
+	int nx = Dm->Nx;
+	int ny = Dm->Ny;
+	int nz = Dm->Nz;
+		
+	//printf("aggregate labels: local size=%i, global size = %i",local_size, full_size);
+	// assign the ID from the phase indicator field
+	for (int k=0; k<nz; k++){
+		for (int j=0; j<ny; j++){
+			for (int i=0; i<nx; i++){
+				int n = k*nx*ny+j*nx+i;
+				signed char local_id_val = Dm->id[n]; 
+				if (local_id_val > 0){
+					double value = Phi(i,j,k);
+					if (value > 0.0)	local_id_val = 1;
+					else 				local_id_val = 2;
+				}
+				 Dm->id[n] = local_id_val;
+			}
+		}
+	}
+	MPI_Barrier(Dm->Comm);
+
+	Dm->AggregateLabels( filename );
+
+}
+
+
+

analysis/SubPhase.h

@@ -0,0 +1,112 @@
+/*
+ * Sub-phase averaging tools
+ */
+
+#ifndef SubPhase_INC
+#define SubPhase_INC
+
+#include <vector>
+#include "common/Domain.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "analysis/distance.h"
+#include "analysis/Minkowski.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+
+
+class phase{
+public:
+	int Nc;
+	double p;
+	double M,Px,Py,Pz,K;
+	double V,A,H,X;
+	void reset(){
+		p=M=Px=Py=Pz=K=0.0;
+		V=A=H=X=0.0;
+		Nc=1;
+	}
+
+private:
+};
+
+class interface{
+public:
+	int Nc;
+	double M,Px,Py,Pz,K;
+	double Mw,Mn,Pnx,Pny,Pnz,Pwx,Pwy,Pwz,Kw,Kn;
+	double V,A,H,X;
+	void reset(){
+		Nc = 0;
+		M=Px=Py=Pz=K=0.0;
+		V=A=H=X=0.0;
+		Mw=Mn=Pnx=Pny=Pnz=Pwx=Pwy=Pwz=Kw=Kn=0.0;
+	}
+
+private:
+};
+
+class SubPhase{
+public:
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+	/*
+	 * indices 
+	 * 	    w - water phase
+	 * 	    n - not water phase
+	 * 		c - connected part
+	 * 		d - disconnected part
+	 * 		i - interface region
+	 * 		b - bulk (total)
+	 */
+	// local entities
+	phase wc,wd,wb,nc,nd,nb;
+	interface iwn,iwnc;
+	
+	// global entities
+	phase gwc,gwd,gwb,gnc,gnd,gnb;
+	interface giwn,giwnc;
+	//...........................................................................
+    int Nx,Ny,Nz;
+	IntArray PhaseID;		// Phase ID array (solid=0, non-wetting=1, wetting=2)
+	BlobIDArray Label_WP;   // Wetting phase label
+	BlobIDArray Label_NWP;  // Non-wetting phase label index (0:nblobs-1)
+	std::vector<BlobIDType> Label_NWP_map;  // Non-wetting phase label for each index
+	DoubleArray Rho_n;	// density field 
+	DoubleArray Rho_w;	// density field 
+	DoubleArray Phi;		// phase indicator field
+	DoubleArray DelPhi;		// Magnitude of Gradient of the phase indicator field
+	DoubleArray Pressure; 	// pressure field
+	DoubleArray Vel_x;		// velocity field
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+	DoubleArray SDs;
+
+	std::shared_ptr<Minkowski> morph_w;
+	std::shared_ptr<Minkowski> morph_n;
+	std::shared_ptr<Minkowski> morph_i;
+
+	SubPhase(std::shared_ptr <Domain> Dm);
+	~SubPhase();
+	
+	void SetParams(double rhoA, double rhoB, double tauA, double tauB, double force_x, double force_y, double force_z, double alpha, double beta);
+	void Basic();
+	void Full();
+	void Write(int time);
+    void AggregateLabels( const std::string& filename );
+
+private:
+	FILE *TIMELOG;
+	FILE *SUBPHASE;
+};
+
+#endif
+

analysis/TwoPhase.cpp

@@ -31,17 +31,18 @@
 #include "analysis/TwoPhase.h"
 
 #include "analysis/pmmc.h"
+#include "analysis/analysis.h"
 #include "common/Domain.h"
 #include "common/Communication.h"
-#include "analysis/analysis.h"
-
-#include "shared_ptr.h"
 #include "common/Utilities.h"
 #include "common/MPI_Helpers.h"
 #include "IO/MeshDatabase.h"
 #include "IO/Reader.h"
 #include "IO/Writer.h"
 
+#include <memory>
+
+
 #define BLOB_AVG_COUNT 35
 
 // Array access for averages defined by the following
@@ -233,6 +234,7 @@ TwoPhase::~TwoPhase()
 
 void TwoPhase::ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData)
 {
+    NULL_USE( Beta );
   /*double factor,temp,value;
 		factor=0.5/Beta;
 	// Initialize to -1,1 (segmentation)
@@ -459,11 +461,10 @@ void TwoPhase::UpdateMeshValues()
 			}
 		}
 	}
-
 }
 void TwoPhase::ComputeLocal()
 {
-	int i,j,k,n,kmin,kmax;
+	int i,j,k,n,imin,jmin,kmin,kmax;
 	int cube[8][3] = {{0,0,0},{1,0,0},{0,1,0},{1,1,0},{0,0,1},{1,0,1},{0,1,1},{1,1,1}};
 
 	// If external boundary conditions are set, do not average over the inlet
@@ -471,9 +472,15 @@ void TwoPhase::ComputeLocal()
 	if (Dm->BoundaryCondition > 0 && Dm->kproc() == 0) kmin=4;
 	if (Dm->BoundaryCondition > 0 && Dm->kproc() == Dm->nprocz()-1) kmax=Nz-4;
 
+	imin=jmin=1;
+	// If inlet layers exist use these as default
+	if (Dm->inlet_layers_x > 0) imin = Dm->inlet_layers_x;
+	if (Dm->inlet_layers_y > 0) jmin = Dm->inlet_layers_y;
+	if (Dm->inlet_layers_z > 0) kmin = Dm->inlet_layers_z;
+		
 	for (k=kmin; k<kmax; k++){
-		for (j=1; j<Ny-1; j++){
-			for (i=1; i<Nx-1; i++){
+		for (j=jmin; j<Ny-1; j++){
+			for (i=imin; i<Nx-1; i++){
 				//...........................................................................
 				n_nw_pts=n_ns_pts=n_ws_pts=n_nws_pts=n_local_sol_pts=n_local_nws_pts=0;
 				n_nw_tris=n_ns_tris=n_ws_tris=n_nws_seg=n_local_sol_tris=0;
@@ -651,8 +658,8 @@ void TwoPhase::ComputeLocal()
 
 void TwoPhase::AssignComponentLabels()
 {
-	int LabelNWP=1;
-	int LabelWP=2;
+	//int LabelNWP=1;
+	//int LabelWP=2;
 	// NOTE: labeling the wetting phase components is tricky! One sandstone media had over 800,000 components
 	// NumberComponents_WP = ComputeGlobalPhaseComponent(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->rank_info,PhaseID,LabelWP,Label_WP);
 	// treat all wetting phase is connected
@@ -1196,6 +1203,8 @@ void TwoPhase::Reduce()
 
 void TwoPhase::NonDimensionalize(double D, double viscosity, double IFT)
 {
+    NULL_USE( viscosity );
+    NULL_USE( IFT );
 	awn_global *= D;
 	ans_global *= D;
 	ans_global *= D;
@@ -1224,8 +1233,8 @@ void TwoPhase::PrintAll(int timestep)
 				Gws_global(0),Gws_global(1),Gws_global(2),Gws_global(3),Gws_global(4),Gws_global(5));	// orientation of ws interface
 		fprintf(TIMELOG,"%.5g %.5g %.5g ",trawn_global, trJwn_global, trRwn_global);		// Trimmed curvature
 		fprintf(TIMELOG,"%.5g %.5g %.5g ",wwndnw_global, wwnsdnwn_global, Jwnwwndnw_global);		// kinematic quantities
-		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g ",wet_morph->V(), wet_morph->A(), wet_morph->J(), wet_morph->X());
-		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g\n",nonwet_morph->V(), nonwet_morph->A(), nonwet_morph->J(), nonwet_morph->X());
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g ",wet_morph->V(), wet_morph->A(), wet_morph->H(), wet_morph->X());
+		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g\n",nonwet_morph->V(), nonwet_morph->A(), nonwet_morph->H(), nonwet_morph->X());
 //		fprintf(TIMELOG,"%.5g %.5g %.5g %.5g\n",euler_global, Kn_global, Jn_global, An_global);			// minkowski measures
 		fflush(TIMELOG);
 	}

analysis/TwoPhase.h

@@ -17,16 +17,15 @@
 #ifndef TwoPhase_INC
 #define TwoPhase_INC
 
+#include <memory>
 #include <vector>
 
 #include "analysis/pmmc.h"
-#include "common/Domain.h"
-#include "common/Communication.h"
 #include "analysis/analysis.h"
 #include "analysis/distance.h"
 #include "analysis/Minkowski.h"
-
-#include "shared_ptr.h"
+#include "common/Domain.h"
+#include "common/Communication.h"
 #include "common/Utilities.h"
 #include "common/MPI_Helpers.h"
 #include "IO/MeshDatabase.h"

analysis/dcel.cpp

@@ -1,7 +1,5 @@
 #include "analysis/dcel.h"
 
-
-
 DECL::DECL(){
 }
 
@@ -15,6 +13,25 @@ int DECL::Face(int index){
 	return FaceData[index];
 }
 
+void DECL::Write(){
+	int e1,e2,e3;
+	FILE *TRIANGLES;
+	TRIANGLES = fopen("triangles.stl","w");
+	fprintf(TRIANGLES,"solid \n");
+	for (int idx=0; idx<TriangleCount; idx++){
+		e1 = Face(idx); 
+		e2 = halfedge.next(e1);
+		e3 = halfedge.next(e2);
+		auto P1 = vertex.coords(halfedge.v1(e1));
+		auto P2 = vertex.coords(halfedge.v1(e2));
+		auto P3 = vertex.coords(halfedge.v1(e3));
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P1.x,P1.y,P1.z);
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P2.x,P2.y,P2.z);
+		fprintf(TRIANGLES,"vertex %f %f %f\n",P3.x,P3.y,P3.z);
+	}
+	fclose(TRIANGLES);
+}
+
 void DECL::LocalIsosurface(const DoubleArray& A, double value, const int i, const int j, const int k){
 	Point P,Q;
 	Point PlaceHolder;
@@ -312,6 +329,7 @@ double DECL::EdgeAngle(int edge)
 		}
 
 		if (dotprod > 1.f) dotprod=1.f;
+		if (dotprod < -1.f) dotprod=-1.f;
 		angle = acos(dotprod);
 		/* project onto plane of cube face also works
 		W = U - dotprod*V;
@@ -344,245 +362,48 @@ double DECL::EdgeAngle(int edge)
 		// concave
 		angle = -angle;
 	}
+	if (angle != angle) angle = 0.0;
 	//printf("angle=%f,dot=%f (Edge=%i, twin=%i): P={%f, %f, %f}, Q={%f, %f, %f} U={%f, %f, %f}, V={%f, %f, %f}\n",angle,dotprod,edge,halfedge.twin(edge),P.x,P.y,P.z,Q.x,Q.y,Q.z,U.x,U.y,U.z,V.x,V.y,V.z);
 	return angle;
 }
 
-void Isosurface(DoubleArray &A, const double &v)
+void iso_surface(const Array<double>&Field, const double isovalue)
 {
-	Point P,Q;
-	Point PlaceHolder;
-	Point C0,C1,C2,C3,C4,C5,C6,C7;
-
-	int TriangleCount;
-	int VertexCount;
-	int CubeIndex;
-
-	Point VertexList[12];
-	Point NewVertexList[12];
-	int LocalRemap[12];
-
-	Point cellvertices[20];
-    std::array<std::array<int,3>,20> Triangles;
-    Triangles.fill( { 0 } );
-
-	// Values from array 'A' at the cube corners
-	double CubeValues[8];
-
-	int Nx = A.size(0);
-	int Ny = A.size(1);
-	int Nz = A.size(2);
-
-	// Points corresponding to cube corners
-	C0.x = 0.0; C0.y = 0.0; C0.z = 0.0;
-	C1.x = 1.0; C1.y = 0.0; C1.z = 0.0;
-	C2.x = 1.0; C2.y = 1.0; C2.z = 0.0;
-	C3.x = 0.0; C3.y = 1.0; C3.z = 0.0;
-	C4.x = 0.0; C4.y = 0.0; C4.z = 1.0;
-	C5.x = 1.0; C5.y = 0.0; C5.z = 1.0;
-	C6.x = 1.0; C6.y = 1.0; C6.z = 1.0;
-	C7.x = 0.0; C7.y = 1.0; C7.z = 1.0;							
-
-	std::vector<std::array<int,6>> HalfEdge;
-	for (int k=1; k<Nz-1; k++){		
-		for (int j=1; j<Ny-1; j++){
-			for (int i=1; i<Nx-1; i++){
-				// Set the corner values for this cube
-				CubeValues[0] = A(i,j,k);
-				CubeValues[1] = A(i+1,j,k);
-				CubeValues[2] = A(i+1,j+1,k);
-				CubeValues[3] = A(i,j+1,k);
-				CubeValues[4] = A(i,j,k+1);
-				CubeValues[5] = A(i+1,j,k+1);
-				CubeValues[6] = A(i+1,j+1,k+1);
-				CubeValues[7] = A(i,j+1,k+1);
-
-				//Determine the index into the edge table which
-				//tells us which vertices are inside of the surface
-				CubeIndex = 0;
-				if (CubeValues[0] < 0.0f) CubeIndex |= 1;
-				if (CubeValues[1] < 0.0f) CubeIndex |= 2;
-				if (CubeValues[2] < 0.0f) CubeIndex |= 4;
-				if (CubeValues[3] < 0.0f) CubeIndex |= 8;
-				if (CubeValues[4] < 0.0f) CubeIndex |= 16;
-				if (CubeValues[5] < 0.0f) CubeIndex |= 32;
-				if (CubeValues[6] < 0.0f) CubeIndex |= 64;
-				if (CubeValues[7] < 0.0f) CubeIndex |= 128;
-
-				//Find the vertices where the surface intersects the cube
-				if (edgeTable[CubeIndex] & 1){
-					P = VertexInterp(C0,C1,CubeValues[0],CubeValues[1]);
-					VertexList[0] = P;
-					Q = C0;
-				}
-				if (edgeTable[CubeIndex] & 2){
-					P = VertexInterp(C1,C2,CubeValues[1],CubeValues[2]);
-					VertexList[1] = P;
-					Q = C1;
-				}
-				if (edgeTable[CubeIndex] & 4){
-					P = VertexInterp(C2,C3,CubeValues[2],CubeValues[3]);
-					VertexList[2] =	P;
-					Q = C2;
-				}
-				if (edgeTable[CubeIndex] & 8){
-					P = VertexInterp(C3,C0,CubeValues[3],CubeValues[0]);
-					VertexList[3] =	P;
-					Q = C3;
-				}
-				if (edgeTable[CubeIndex] & 16){
-					P = VertexInterp(C4,C5,CubeValues[4],CubeValues[5]);
-					VertexList[4] =	P;
-					Q = C4;
-				}
-				if (edgeTable[CubeIndex] & 32){
-					P = VertexInterp(C5,C6,CubeValues[5],CubeValues[6]);
-					VertexList[5] = P;
-					Q = C5;
-				}
-				if (edgeTable[CubeIndex] & 64){ 
-					P = VertexInterp(C6,C7,CubeValues[6],CubeValues[7]);
-					VertexList[6] = P;
-					Q = C6;
-				}
-				if (edgeTable[CubeIndex] & 128){
-					P = VertexInterp(C7,C4,CubeValues[7],CubeValues[4]);
-					VertexList[7] =	P;
-					Q = C7;
-				}
-				if (edgeTable[CubeIndex] & 256){
-					P = VertexInterp(C0,C4,CubeValues[0],CubeValues[4]);
-					VertexList[8] =	P;
-					Q = C0;
-				}
-				if (edgeTable[CubeIndex] & 512){
-					P = VertexInterp(C1,C5,CubeValues[1],CubeValues[5]);
-					VertexList[9] =	P;
-					Q = C1;
-				}
-				if (edgeTable[CubeIndex] & 1024){
-					P = VertexInterp(C2,C6,CubeValues[2],CubeValues[6]);
-					VertexList[10] = P;
-					Q = C2;
-				}
-				if (edgeTable[CubeIndex] & 2048){
-					P = VertexInterp(C3,C7,CubeValues[3],CubeValues[7]);
-					VertexList[11] = P;
-					Q = C3;
-				}
-
-				VertexCount=0;
-				for (int idx=0;idx<12;idx++)
-					LocalRemap[idx] = -1;
-
-				for (int idx=0;triTable[CubeIndex][idx]!=-1;idx++)
-				{
-					if(LocalRemap[triTable[CubeIndex][idx]] == -1)
-					{
-						NewVertexList[VertexCount] = VertexList[triTable[CubeIndex][idx]];
-						LocalRemap[triTable[CubeIndex][idx]] = VertexCount;
-						VertexCount++;
-					}
-				}
-
-				for (int idx=0;idx<VertexCount;idx++) {
-					P = NewVertexList[idx];
-					//P.x  += i;
-					//P.y  += j;
-					//P.z  += k;
-					cellvertices[idx] = P;
-				}
-
-				TriangleCount = 0;
-				for (int idx=0;triTable[CubeIndex][idx]!=-1;idx+=3) {
-					Triangles[TriangleCount][0] = LocalRemap[triTable[CubeIndex][idx+0]];
-					Triangles[TriangleCount][1] = LocalRemap[triTable[CubeIndex][idx+1]];
-					Triangles[TriangleCount][2] = LocalRemap[triTable[CubeIndex][idx+2]];
-					TriangleCount++;
-				}
-				int nTris = TriangleCount;
-
-				// Now add the local values to the DECL data structure
-				HalfEdge.resize(nTris*3);
-				int idx_edge=0;
-				for (int idx=0; idx<TriangleCount; idx++){
-					int V1 = Triangles[idx][0];
-					int V2 = Triangles[idx][1];
-					int V3 = Triangles[idx][2];
-					// first edge: V1->V2 
-					HalfEdge[idx_edge][0] = V1;  // first vertex
-					HalfEdge[idx_edge][1] = V2;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge+2;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge+1;  // next edge
-					idx_edge++;
-					// second edge: V2->V3
-					HalfEdge[idx_edge][0] = V2;  // first vertex
-					HalfEdge[idx_edge][1] = V3;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge-1;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge+1;  // next edge
-					idx_edge++;
-					// third edge: V3->V1
-					HalfEdge[idx_edge][0] = V3;  // first vertex
-					HalfEdge[idx_edge][1] = V1;  // second vertex
-					HalfEdge[idx_edge][2] = idx; // triangle
-					HalfEdge[idx_edge][3] = -1;  // twin
-					HalfEdge[idx_edge][4] = idx_edge-1;  // previous edge
-					HalfEdge[idx_edge][5] = idx_edge-2;  // next edge
-					idx_edge++;
-				}
-				int EdgeCount=idx_edge;
-				for (int idx=0; idx<EdgeCount; idx++){
-					int V1=HalfEdge[idx][0];
-					int V2=HalfEdge[idx][1];
-					// Find all the twins within the cube
-					for (int jdx=0; idx<EdgeCount; jdx++){
-						if (HalfEdge[jdx][1] == V1 && HalfEdge[jdx][0] == V2){
-							// this is the pair
-							HalfEdge[idx][3] = jdx;
-							HalfEdge[jdx][3] = idx;
-						}
-						if (HalfEdge[jdx][1] == V2 && HalfEdge[jdx][0] == V1 && !(idx==jdx)){
-							std::printf("WARNING: half edges with identical orientation! \n");
-						}
-					}	
-					// Use "ghost" twins if edge is on a cube face
-					P = cellvertices[V1];
-					Q = cellvertices[V2];
-					if (P.x == 0.0 && Q.x == 0.0) HalfEdge[idx_edge][3] = -1;  // ghost twin for x=0 face
-					if (P.x == 1.0 && Q.x == 1.0) HalfEdge[idx_edge][3] = -2;  // ghost twin for x=1 face
-					if (P.y == 0.0 && Q.y == 0.0) HalfEdge[idx_edge][3] = -3;  // ghost twin for y=0 face
-					if (P.y == 1.0 && Q.y == 1.0) HalfEdge[idx_edge][3] = -4;  // ghost twin for y=1 face
-					if (P.z == 0.0 && Q.z == 0.0) HalfEdge[idx_edge][3] = -5;  // ghost twin for z=0 face
-					if (P.z == 1.0 && Q.z == 1.0) HalfEdge[idx_edge][3] = -6;  // ghost twin for z=1 face
-				}
-				// Find all the angles
-				for (int idx=0; idx<EdgeCount; idx++){
-					int V1=HalfEdge[idx][0];
-					int V2=HalfEdge[idx][1];
-					int T1= HalfEdge[idx_edge][2];
-					int twin=HalfEdge[idx_edge][3];
-					if (twin == -1){
-						
-					}
-				}
-
-				// Map vertices to global coordinates
-				for (int idx=0;idx<VertexCount;idx++) {
-					P = cellvertices[idx];
-					P.x  += i;
-					P.y  += j;
-					P.z  += k;
-					cellvertices[idx] = P;
-				}
-			}
-		}
-	}
+  DECL object;
+  int e1,e2,e3;
+  FILE *TRIANGLES;
+  TRIANGLES = fopen("isosurface.stl","w");
+  fprintf(TRIANGLES,"solid isosurface\n");
+  int Nx = Field.size(0);
+  int Ny = Field.size(1);
+  int Nz = Field.size(2);
+  for (int k=1; k<Nz-1; k++){
+    for (int j=1; j<Ny-1; j++){
+      for (int i=1; i<Nx-1; i++){
+        object.LocalIsosurface(Field,isovalue,i,j,k);
+        for (int idx=0; idx<object.TriangleCount; idx++){
+          e1 = object.Face(idx);
+          e2 = object.halfedge.next(e1);
+          e3 = object.halfedge.next(e2);
+          auto P1 = object.vertex.coords(object.halfedge.v1(e1));
+          auto P2 = object.vertex.coords(object.halfedge.v1(e2));
+          auto P3 = object.vertex.coords(object.halfedge.v1(e3));
+	  auto Normal  = object.TriNormal(e1);
+	  // P1.x += 1.0*i; P1.y += 1.0*j; P1.z +=1.0*k;
+          //P2.x += 1.0*i; P2.y += 1.0*j; P2.z +=1.0*k;
+          //P3.x += 1.0*i; P3.y += 1.0*j; P3.z +=1.0*k;
+          fprintf(TRIANGLES,"facet normal %f %f %f\n",Normal.x,Normal.y,Normal.z);
+	  fprintf(TRIANGLES,"   outer loop\n");
+	  fprintf(TRIANGLES,"      vertex %f %f %f\n",P1.x,P1.y,P1.z);
+          fprintf(TRIANGLES,"      vertex %f %f %f\n",P2.x,P2.y,P2.z);
+          fprintf(TRIANGLES,"      vertex %f %f %f\n",P3.x,P3.y,P3.z);
+	  fprintf(TRIANGLES,"   endloop\n");
+	  fprintf(TRIANGLES,"endfacet\n");
 
+        }
+      }
+    }
+  }
+  fprintf(TRIANGLES,"endsolid isosurface\n");
+  fclose(TRIANGLES);
 }
-
-
-

analysis/dcel.h

@@ -1,3 +1,6 @@
+#ifndef DCEL_INC
+#define DCEL_INC
+
 #include <vector>
 #include "analysis/pmmc.h"
 
@@ -67,6 +70,7 @@ public:
 	Vertex vertex;
 	Halfedge halfedge;
 	void LocalIsosurface(const DoubleArray& A, double value, int i, int j, int k);
+	void Write();
 	int Face(int index);
 	
 	double origin(int edge);
@@ -78,3 +82,7 @@ public:
 private:
 	std::vector<int> FaceData;
 };
+
+void iso_surface(const Array<double>&Field, const double isovalue);
+
+#endif

analysis/distance.cpp

@@ -197,6 +197,148 @@ void CalcVecDist( Array<Vec> &d, const Array<int> &ID0, const Domain &Dm,
     }
 }
 
+double Eikonal(DoubleArray &Distance, const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic){
+
+  /*
+   * This routine converts the data in the Distance array to a signed distance
+   * by solving the equation df/dt = sign(1-|grad f|), where Distance provides
+   * the values of f on the mesh associated with domain Dm
+   * It has been tested with segmented data initialized to values [-1,1]
+   * and will converge toward the signed distance to the surface bounding the associated phases
+   *
+   * Reference:
+   * Min C (2010) On reinitializing level set functions, Journal of Computational Physics229
+   */
+
+  int i,j,k;
+  double dt=0.1;
+  double Dx,Dy,Dz;
+  double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
+  double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
+  double sign,norm;
+  double LocalVar,GlobalVar,LocalMax,GlobalMax;
+
+  int xdim,ydim,zdim;
+  xdim=Dm.Nx-2;
+  ydim=Dm.Ny-2;
+  zdim=Dm.Nz-2;
+  //fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
+  fillHalo<double> fillData(  Dm.Comm, Dm.rank_info, {xdim, ydim, zdim}, {1,1,1}, 50, 1, {true,true,true}, periodic );
+
+  // Arrays to store the second derivatives
+  DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
+  DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
+  DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
+
+  int count = 0;
+  while (count < timesteps){
+
+    // Communicate the halo of values
+    fillData.fill(Distance);
+
+    // Compute second order derivatives
+    for (k=1;k<Dm.Nz-1;k++){
+      for (j=1;j<Dm.Ny-1;j++){
+	for (i=1;i<Dm.Nx-1;i++){
+	  Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
+	  Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
+	  Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
+	}
+      }
+    }
+    fillData.fill(Dxx);
+    fillData.fill(Dyy);
+    fillData.fill(Dzz);
+
+    LocalMax=LocalVar=0.0;
+    // Execute the next timestep
+    for (k=1;k<Dm.Nz-1;k++){
+      for (j=1;j<Dm.Ny-1;j++){
+	for (i=1;i<Dm.Nx-1;i++){
+
+	  int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
+
+	  sign = -1;
+	  if (ID(i,j,k) == 1) sign = 1;
+
+	  // local second derivative terms
+	  Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
+	  Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
+	  Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
+	  Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
+	  Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
+	  Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
+
+	  /* //............Compute upwind derivatives ...................
+                    Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
+                    Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
+                    Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
+                    Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+                    Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+                    Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+	  */
+	  Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
+	  Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
+	  Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
+
+	  Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+	  Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+	  Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+
+	  // Compute upwind derivatives for Godunov Hamiltonian
+	  if (sign < 0.0){
+	    if (Dxp + Dxm > 0.f)  Dx = Dxp*Dxp;
+	    else Dx = Dxm*Dxm;
+
+	    if (Dyp + Dym > 0.f)  Dy = Dyp*Dyp;
+	    else Dy = Dym*Dym;
+
+	    if (Dzp + Dzm > 0.f)  Dz = Dzp*Dzp;
+	    else Dz = Dzm*Dzm;
+	  }
+	  else{
+
+	    if (Dxp + Dxm < 0.f)  Dx = Dxp*Dxp;
+	    else Dx = Dxm*Dxm;
+
+	    if (Dyp + Dym < 0.f)  Dy = Dyp*Dyp;
+	    else Dy = Dym*Dym;
+
+	    if (Dzp + Dzm < 0.f)  Dz = Dzp*Dzp;
+	    else Dz = Dzm*Dzm;
+	  }
+
+	  //Dx = max(Dxp*Dxp,Dxm*Dxm);
+	  //Dy = max(Dyp*Dyp,Dym*Dym);
+	  //Dz = max(Dzp*Dzp,Dzm*Dzm);
+
+	  norm=sqrt(Dx + Dy + Dz);
+	  if (norm > 1.0) norm=1.0;
+
+	  Distance(i,j,k) += dt*sign*(1.0 - norm);
+	  LocalVar +=  dt*sign*(1.0 - norm);
+
+	  if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
+	    LocalMax = fabs(dt*sign*(1.0 - norm));
+	}
+      }
+    }
+
+    MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
+    MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
+    GlobalVar /= Dm.Volume;
+    count++;
+
+    if (count%50 == 0 && Dm.rank()==0 )
+      printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
+
+    if (fabs(GlobalMax) < 1e-5){
+      if (Dm.rank()==0) printf("Exiting with max tolerance of 1e-5 \n");
+      count=timesteps;
+    }
+  }
+  return GlobalVar;
+}
 
 // Explicit instantiations
 template void CalcDist<float>( Array<float>&, const Array<char>&, const Domain&, const std::array<bool,3>&, const std::array<double,3>& );

analysis/distance.h

@@ -31,6 +31,16 @@ struct Vec {
 };
 inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; }
 
+inline double minmod(double &a, double &b){
+
+  double value;
+
+  value = a;
+  if ( a*b < 0.0)    value=0.0;
+  else if (fabs(a) > fabs(b)) value = b;
+
+  return value;
+}
 
 /*!
  * @brief  Calculate the distance using a simple method
@@ -55,4 +65,16 @@ void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
 void CalcVecDist( Array<Vec> &Distance, const Array<int> &ID, const Domain &Dm,
     const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );
 
+
+/*!
+ * @brief  Calculate the distance based on solution of Eikonal equation
+ * @details  This routine calculates the signed distance to the nearest domain surface.
+ * @param[out] Distance     Distance function
+ * @param[in] ID            Domain id
+ * @param[in] Dm            Domain information
+ * @param[in] timesteps      number of timesteps to run for Eikonal solver
+ * @param[in] periodic      Directions that are periodic 
+ */
+double Eikonal(DoubleArray &Distance,  const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic);
+
 #endif

analysis/filters.cpp

@@ -17,6 +17,33 @@
 #include "math.h"
 #include "ProfilerApp.h"
 
+void Mean3D( const Array<double> &Input, Array<double> &Output )
+{
+	PROFILE_START("Mean3D");
+	// Perform a 3D Mean filter on Input array
+	int i,j,k;
+
+	int Nx = int(Input.size(0));
+	int Ny = int(Input.size(1));
+	int Nz = int(Input.size(2));
+
+	for (k=1; k<Nz-1; k++){
+		for (j=1; j<Ny-1; j++){
+			for (i=1; i<Nx-1; i++){
+			  double MeanValue = Input(i,j,k);
+			  // next neighbors
+			  MeanValue += Input(i+1,j,k)+Input(i,j+1,k)+Input(i,j,k+1)+Input(i-1,j,k)+Input(i,j-1,k)+Input(i,j,k-1);
+			  MeanValue += Input(i+1,j+1,k)+Input(i-1,j+1,k)+Input(i+1,j-1,k)+Input(i-1,j-1,k);
+			  MeanValue += Input(i+1,j,k+1)+Input(i-1,j,k+1)+Input(i+1,j,k-1)+Input(i-1,j,k-1);
+			  MeanValue += Input(i,j+1,k+1)+Input(i,j-1,k+1)+Input(i,j+1,k-1)+Input(i,j-1,k-1);
+			  MeanValue += Input(i+1,j+1,k+1)+Input(i-1,j+1,k+1)+Input(i+1,j-1,k+1)+Input(i-1,j-1,k+1);
+			  MeanValue += Input(i+1,j+1,k-1)+Input(i-1,j+1,k-1)+Input(i+1,j-1,k-1)+Input(i-1,j-1,k-1);
+			  Output(i,j,k) = MeanValue/27.0;
+			}
+		}
+	}
+	PROFILE_STOP("Mean3D");
+}
 
 void Med3D( const Array<float> &Input, Array<float> &Output )
 {

analysis/filters.h

@@ -19,6 +19,13 @@
 
 #include "common/Array.h"
 
+/*!
+ * @brief  Filter image
+ * @details  This routine performs a mean filter
+ * @param[in] Input     Input image
+ * @param[out] Output   Output image
+ */
+void Mean3D( const Array<double> &Input, Array<double> &Output );
 
 /*!
  * @brief  Filter image
@@ -28,7 +35,6 @@
  */
 void Med3D( const Array<float> &Input, Array<float> &Output );
 
-
 /*!
  * @brief  Filter image
  * @details  This routine performs a non-linear local means filter

analysis/morphology.cpp

@@ -0,0 +1,782 @@
+#include <analysis/morphology.h>
+// Implementation of morphological opening routine
+
+inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){
+	// Fill in the phase ID values from neighboring processors
+	// This packs up the values that need to be sent from one processor to another
+	int idx,n;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		sendbuf[idx] = ID[n];
+	}
+}
+//***************************************************************************************
+
+inline void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID){
+	// Fill in the phase ID values from neighboring processors
+	// This unpacks the values once they have been recieved from neighbors
+	int idx,n;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		ID[n] = recvbuf[idx];
+	}
+}
+
+//***************************************************************************************
+double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction, signed char ErodeLabel, signed char NewLabel){
+	// SignDist is the distance to the object that you want to constaing the morphological opening
+	// VoidFraction is the the empty space where the object inst
+	// id is a labeled map
+	// Dm contains information about the domain structure
+	
+	int nx = Dm->Nx;
+	int ny = Dm->Ny;
+	int nz = Dm->Nz;
+	int nprocx = Dm->nprocx();
+	int nprocy = Dm->nprocy();
+	int nprocz = Dm->nprocz();
+	int rank = Dm->rank();
+
+	int n;
+	double final_void_fraction;
+	double count,countGlobal,totalGlobal;
+	count = 0.f;
+	double maxdist=-200.f;
+	double maxdistGlobal;
+	for (int k=1; k<nz-1; k++){
+		for (int j=1; j<ny-1; j++){
+			for (int i=1; i<nx-1; i++){
+				n = k*nx*ny+j*nx+i;
+				// extract maximum distance for critical radius
+				if ( SignDist(i,j,k) > maxdist) maxdist=SignDist(i,j,k);
+				if ( id[n] == ErodeLabel){
+					count += 1.0;
+					//id[n]  = 2;
+				}
+			}
+		}
+	}
+	MPI_Barrier(Dm->Comm);
+	
+	// total Global is the number of nodes in the pore-space
+	MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+	MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
+	double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
+	double volume_fraction=totalGlobal/volume;
+	if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
+	if (rank==0) printf("Maximum pore size: %f \n",maxdistGlobal);
+	final_void_fraction = volume_fraction; //initialize
+
+	// Communication buffers
+	signed char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
+	signed char *sendID_xy, *sendID_yz, *sendID_xz, *sendID_Xy, *sendID_Yz, *sendID_xZ;
+	signed char *sendID_xY, *sendID_yZ, *sendID_Xz, *sendID_XY, *sendID_YZ, *sendID_XZ;
+	signed char *recvID_x, *recvID_y, *recvID_z, *recvID_X, *recvID_Y, *recvID_Z;
+	signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
+	signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
+	// send buffers
+	sendID_x = new signed char [Dm->sendCount_x];
+	sendID_y = new signed char [Dm->sendCount_y];
+	sendID_z = new signed char [Dm->sendCount_z];
+	sendID_X = new signed char [Dm->sendCount_X];
+	sendID_Y = new signed char [Dm->sendCount_Y];
+	sendID_Z = new signed char [Dm->sendCount_Z];
+	sendID_xy = new signed char [Dm->sendCount_xy];
+	sendID_yz = new signed char [Dm->sendCount_yz];
+	sendID_xz = new signed char [Dm->sendCount_xz];
+	sendID_Xy = new signed char [Dm->sendCount_Xy];
+	sendID_Yz = new signed char [Dm->sendCount_Yz];
+	sendID_xZ = new signed char [Dm->sendCount_xZ];
+	sendID_xY = new signed char [Dm->sendCount_xY];
+	sendID_yZ = new signed char [Dm->sendCount_yZ];
+	sendID_Xz = new signed char [Dm->sendCount_Xz];
+	sendID_XY = new signed char [Dm->sendCount_XY];
+	sendID_YZ = new signed char [Dm->sendCount_YZ];
+	sendID_XZ = new signed char [Dm->sendCount_XZ];
+	//......................................................................................
+	// recv buffers
+	recvID_x = new signed char [Dm->recvCount_x];
+	recvID_y = new signed char [Dm->recvCount_y];
+	recvID_z = new signed char [Dm->recvCount_z];
+	recvID_X = new signed char [Dm->recvCount_X];
+	recvID_Y = new signed char [Dm->recvCount_Y];
+	recvID_Z = new signed char [Dm->recvCount_Z];
+	recvID_xy = new signed char [Dm->recvCount_xy];
+	recvID_yz = new signed char [Dm->recvCount_yz];
+	recvID_xz = new signed char [Dm->recvCount_xz];
+	recvID_Xy = new signed char [Dm->recvCount_Xy];
+	recvID_xZ = new signed char [Dm->recvCount_xZ];
+	recvID_xY = new signed char [Dm->recvCount_xY];
+	recvID_yZ = new signed char [Dm->recvCount_yZ];
+	recvID_Yz = new signed char [Dm->recvCount_Yz];
+	recvID_Xz = new signed char [Dm->recvCount_Xz];
+	recvID_XY = new signed char [Dm->recvCount_XY];
+	recvID_YZ = new signed char [Dm->recvCount_YZ];
+	recvID_XZ = new signed char [Dm->recvCount_XZ];
+	//......................................................................................
+	int sendtag,recvtag;
+	sendtag = recvtag = 7;
+
+	int ii,jj,kk;
+	int Nx = nx;
+	int Ny = ny;
+	int Nz = nz;
+
+	double void_fraction_old=1.0;
+	double void_fraction_new=1.0; 
+	double void_fraction_diff_old = 1.0;
+	double void_fraction_diff_new = 1.0;
+
+	// Increase the critical radius until the target saturation is met
+	double deltaR=0.05; // amount to change the radius in voxel units
+	double Rcrit_old=0.0;
+
+	double GlobalNumber = 1.f;
+	int imin,jmin,kmin,imax,jmax,kmax;
+
+	if (ErodeLabel == 1){
+		VoidFraction = 1.0 - VoidFraction;
+	}
+
+	double Rcrit_new = maxdistGlobal;
+
+	while (void_fraction_new > VoidFraction)
+	{
+		void_fraction_diff_old = void_fraction_diff_new;
+		void_fraction_old = void_fraction_new;
+		Rcrit_old = Rcrit_new;
+		Rcrit_new -= deltaR*Rcrit_old;
+		int Window=round(Rcrit_new);
+		if (Window == 0) Window = 1; // If Window = 0 at the begining, after the following process will have sw=1.0
+		// and sw<Sw will be immediately broken
+		double LocalNumber=0.f;
+		for(int k=0; k<Nz; k++){
+			for(int j=0; j<Ny; j++){
+				for(int i=0; i<Nx; i++){
+					n = k*nx*ny + j*nx+i;
+					if (SignDist(i,j,k) > Rcrit_new){
+						// loop over the window and update
+						imin=max(1,i-Window);
+						jmin=max(1,j-Window);
+						kmin=max(1,k-Window);
+						imax=min(Nx-1,i+Window);
+						jmax=min(Ny-1,j+Window);
+						kmax=min(Nz-1,k+Window);
+						for (kk=kmin; kk<kmax; kk++){
+							for (jj=jmin; jj<jmax; jj++){
+								for (ii=imin; ii<imax; ii++){
+									int nn = kk*nx*ny+jj*nx+ii;
+									double dsq = double((ii-i)*(ii-i)+(jj-j)*(jj-j)+(kk-k)*(kk-k));
+									if (id[nn] == ErodeLabel && dsq <= Rcrit_new*Rcrit_new){
+										LocalNumber+=1.0;
+										id[nn]=NewLabel;
+									}
+								}
+							}
+						}
+
+					}
+					// move on
+				}
+			}
+		}
+		// Pack and send the updated ID values
+		PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
+		PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
+		PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
+		PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
+		PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
+		PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
+		PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
+		PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
+		PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
+		PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
+		PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
+		PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
+		PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
+		PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
+		PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
+		PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
+		PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
+		PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
+		//......................................................................................
+		MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
+				recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
+				recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
+				recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
+				recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
+				recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
+				recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
+				recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
+				recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
+				recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
+				recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
+				recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
+				recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
+				recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
+				recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
+				recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
+				recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
+				recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
+				recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		//......................................................................................
+		UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
+		UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
+		UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
+		UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
+		UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
+		UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
+		UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
+		UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
+		UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
+		UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
+		UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
+		UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
+		UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
+		UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
+		UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
+		UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
+		UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
+		UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
+		//......................................................................................
+
+		MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+
+		count = 0.f;
+		for (int k=1; k<Nz-1; k++){
+			for (int j=1; j<Ny-1; j++){
+				for (int i=1; i<Nx-1; i++){
+					n=k*Nx*Ny+j*Nx+i;
+					if (id[n] == ErodeLabel){
+						count+=1.0;
+					}
+				}
+			}
+		}
+		MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+		void_fraction_new = countGlobal/totalGlobal;
+		void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
+	/*	if (rank==0){
+			printf("     %f ",void_fraction_new);
+			printf("     %f\n",Rcrit_new);
+		} */
+	}
+
+	if (void_fraction_diff_new<void_fraction_diff_old){
+		final_void_fraction=void_fraction_new;
+		if (rank==0){
+			printf("Final void fraction =%f\n",void_fraction_new);
+			printf("Final critical radius=%f\n",Rcrit_new);
+		}
+	}
+	else{
+		final_void_fraction=void_fraction_old;
+		if (rank==0){
+			printf("Final void fraction=%f\n",void_fraction_old);
+			printf("Final critical radius=%f\n",Rcrit_old);
+		}
+	}
+	return final_void_fraction;
+}
+/*
+double morph_open()
+{
+
+	fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
+
+
+	MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+
+	count = 0.f;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				n=k*Nx*Ny+j*Nx+i;
+				if (id[n] == 2){
+					count+=1.0;
+				}
+			}
+		}
+	}
+	MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+	return countGlobal;
+}
+*/
+
+//***************************************************************************************
+double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction){
+	// SignDist is the distance to the object that you want to constaing the morphological opening
+	// VoidFraction is the the empty space where the object inst
+	// id is a labeled map
+	// Dm contains information about the domain structure
+	
+	int nx = Dm->Nx;
+	int ny = Dm->Ny;
+	int nz = Dm->Nz;
+	int nprocx = Dm->nprocx();
+	int nprocy = Dm->nprocy();
+	int nprocz = Dm->nprocz();
+	int rank = Dm->rank();
+	
+	DoubleArray phase(nx,ny,nz);
+	IntArray phase_label(nx,ny,nz);
+
+	int n;
+	double final_void_fraction;
+	double count,countGlobal,totalGlobal;
+	count = 0.f;
+	double maxdist=-200.f;
+	double maxdistGlobal;
+	for (int k=1; k<nz-1; k++){
+		for (int j=1; j<ny-1; j++){
+			for (int i=1; i<nx-1; i++){
+				n = k*nx*ny+j*nx+i;
+				// extract maximum distance for critical radius
+				if ( SignDist(i,j,k) > maxdist) maxdist=SignDist(i,j,k);
+				if ( SignDist(i,j,k) > 0.0 ){
+					count += 1.0;
+					id[n]  = 2;
+				}
+			}
+		}
+	}
+
+	MPI_Barrier(Dm->Comm);
+	
+	// total Global is the number of nodes in the pore-space
+	MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+	MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
+	double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
+	double volume_fraction=totalGlobal/volume;
+	if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
+	if (rank==0) printf("Maximum pore size: %f \n",maxdistGlobal);
+
+	// Communication buffers
+	signed char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
+	signed char *sendID_xy, *sendID_yz, *sendID_xz, *sendID_Xy, *sendID_Yz, *sendID_xZ;
+	signed char *sendID_xY, *sendID_yZ, *sendID_Xz, *sendID_XY, *sendID_YZ, *sendID_XZ;
+	signed char *recvID_x, *recvID_y, *recvID_z, *recvID_X, *recvID_Y, *recvID_Z;
+	signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
+	signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
+	// send buffers
+	sendID_x = new signed char [Dm->sendCount_x];
+	sendID_y = new signed char [Dm->sendCount_y];
+	sendID_z = new signed char [Dm->sendCount_z];
+	sendID_X = new signed char [Dm->sendCount_X];
+	sendID_Y = new signed char [Dm->sendCount_Y];
+	sendID_Z = new signed char [Dm->sendCount_Z];
+	sendID_xy = new signed char [Dm->sendCount_xy];
+	sendID_yz = new signed char [Dm->sendCount_yz];
+	sendID_xz = new signed char [Dm->sendCount_xz];
+	sendID_Xy = new signed char [Dm->sendCount_Xy];
+	sendID_Yz = new signed char [Dm->sendCount_Yz];
+	sendID_xZ = new signed char [Dm->sendCount_xZ];
+	sendID_xY = new signed char [Dm->sendCount_xY];
+	sendID_yZ = new signed char [Dm->sendCount_yZ];
+	sendID_Xz = new signed char [Dm->sendCount_Xz];
+	sendID_XY = new signed char [Dm->sendCount_XY];
+	sendID_YZ = new signed char [Dm->sendCount_YZ];
+	sendID_XZ = new signed char [Dm->sendCount_XZ];
+	//......................................................................................
+	// recv buffers
+	recvID_x = new signed char [Dm->recvCount_x];
+	recvID_y = new signed char [Dm->recvCount_y];
+	recvID_z = new signed char [Dm->recvCount_z];
+	recvID_X = new signed char [Dm->recvCount_X];
+	recvID_Y = new signed char [Dm->recvCount_Y];
+	recvID_Z = new signed char [Dm->recvCount_Z];
+	recvID_xy = new signed char [Dm->recvCount_xy];
+	recvID_yz = new signed char [Dm->recvCount_yz];
+	recvID_xz = new signed char [Dm->recvCount_xz];
+	recvID_Xy = new signed char [Dm->recvCount_Xy];
+	recvID_xZ = new signed char [Dm->recvCount_xZ];
+	recvID_xY = new signed char [Dm->recvCount_xY];
+	recvID_yZ = new signed char [Dm->recvCount_yZ];
+	recvID_Yz = new signed char [Dm->recvCount_Yz];
+	recvID_Xz = new signed char [Dm->recvCount_Xz];
+	recvID_XY = new signed char [Dm->recvCount_XY];
+	recvID_YZ = new signed char [Dm->recvCount_YZ];
+	recvID_XZ = new signed char [Dm->recvCount_XZ];
+	//......................................................................................
+	int sendtag,recvtag;
+	sendtag = recvtag = 7;
+
+	int ii,jj,kk;
+	int Nx = nx;
+	int Ny = ny;
+	int Nz = nz;
+
+	double void_fraction_old=1.0;
+	double void_fraction_new=1.0; 
+	double void_fraction_diff_old = 1.0;
+	double void_fraction_diff_new = 1.0;
+
+	// Increase the critical radius until the target saturation is met
+	double deltaR=0.05; // amount to change the radius in voxel units
+	double Rcrit_old;
+
+	double GlobalNumber = 1.f;
+	int imin,jmin,kmin,imax,jmax,kmax;
+
+	double Rcrit_new = maxdistGlobal;
+	//if (argc>2){
+	//	Rcrit_new = strtod(argv[2],NULL);
+	//	if (rank==0) printf("Max. distance =%f, Initial critical radius = %f \n",maxdistGlobal,Rcrit_new);
+	//}
+	MPI_Barrier(Dm->Comm);
+
+	
+	FILE *DRAIN = fopen("morphdrain.csv","w");
+	fprintf(DRAIN,"sw radius\n");				
+
+	while (void_fraction_new > VoidFraction && Rcrit_new > 0.5)
+	{
+		void_fraction_diff_old = void_fraction_diff_new;
+		void_fraction_old = void_fraction_new;
+		Rcrit_old = Rcrit_new;
+		Rcrit_new -= deltaR*Rcrit_old;
+		int Window=round(Rcrit_new);
+		if (Window == 0) Window = 1; // If Window = 0 at the begining, after the following process will have sw=1.0
+		// and sw<Sw will be immediately broken
+		double LocalNumber=0.f;
+		for(int k=1; k<Nz-1; k++){
+			for(int j=1; j<Ny-1; j++){
+				for(int i=1; i<Nx-1; i++){
+					n = k*nx*ny + j*nx+i;
+					if (SignDist(i,j,k) > Rcrit_new){
+						// loop over the window and update
+						imin=max(1,i-Window);
+						jmin=max(1,j-Window);
+						kmin=max(1,k-Window);
+						imax=min(Nx-1,i+Window);
+						jmax=min(Ny-1,j+Window);
+						kmax=min(Nz-1,k+Window);
+						for (kk=kmin; kk<kmax; kk++){
+							for (jj=jmin; jj<jmax; jj++){
+								for (ii=imin; ii<imax; ii++){
+									int nn = kk*nx*ny+jj*nx+ii;
+									double dsq = double((ii-i)*(ii-i)+(jj-j)*(jj-j)+(kk-k)*(kk-k));
+									if (id[nn] == 2 && dsq <= (Rcrit_new+1)*(Rcrit_new+1)){
+										LocalNumber+=1.0;
+										id[nn]=1;
+									}
+								}
+							}
+						}
+
+					}
+					// move on
+				}
+			}
+		}
+		// Pack and send the updated ID values
+		PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
+		PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
+		PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
+		PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
+		PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
+		PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
+		PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
+		PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
+		PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
+		PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
+		PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
+		PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
+		PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
+		PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
+		PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
+		PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
+		PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
+		PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
+		//......................................................................................
+		MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
+				recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
+				recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
+				recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
+				recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
+				recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
+				recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
+				recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
+				recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
+				recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
+				recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
+				recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
+				recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
+				recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
+				recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
+				recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
+				recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
+				recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
+				recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
+		//......................................................................................
+		UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
+		UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
+		UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
+		UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
+		UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
+		UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
+		UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
+		UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
+		UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
+		UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
+		UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
+		UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
+		UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
+		UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
+		UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
+		UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
+		UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
+		UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
+		//......................................................................................
+		MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+		
+		for (int k=0; k<nz; k++){
+			for (int j=0; j<ny; j++){
+				for (int i=0; i<nx; i++){
+					n=k*nx*ny+j*nx+i;
+					if (id[n] == 1){
+						phase(i,j,k) = 1.0;
+					}
+					else
+						phase(i,j,k) = -1.0;
+				}
+			}
+		}
+		
+		// Extract only the connected part of NWP
+		BlobIDstruct new_index;
+		double vF=0.0; double vS=0.0;
+		ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
+		MPI_Barrier(Dm->Comm);
+		
+		for (int k=0; k<nz; k++){
+			for (int j=0; j<ny; j++){
+				for (int i=0; i<nx; i++){
+					n=k*nx*ny+j*nx+i;
+					if (id[n] == 1 && phase_label(i,j,k) > 1){
+						id[n] = 2;
+					}
+				}
+			}
+		}
+		
+		
+		/*
+		* Extract only the connected part of NWP
+		for (int k=1; k<nz-1; k++){
+			for (int j=1; j<ny-1; j++){
+				for (int i=1; i<nx-1; i++){
+					n=k*nx*ny+j*nx+i;
+					if (id[n] == 2){
+						phase(i,j,k) = 1.0;
+					}
+					else if (id[n] == 1){
+						// nwp
+						phase(i,j,k) = -1.0;
+					}
+					else{
+						// treat solid as WP since films can connect 
+						phase(i,j,k) = 1.0;
+					}
+				}
+			}
+		}
+		
+		ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
+		MPI_Barrier(Dm->Comm);
+		
+		for (int k=1; k<nz-1; k++){
+			for (int j=1; j<ny-1; j++){
+				for (int i=1; i<nx-1; i++){
+					n=k*nx*ny+j*nx+i;
+					if (id[n] == 2 && phase_label(i,j,k) > 1){
+						id[n] = 20;
+					}
+				}
+			}
+		}
+		// done
+		*/
+
+		count = 0.f;
+		for (int k=1; k<nz-1; k++){
+			for (int j=1; j<ny-1; j++){
+				for (int i=1; i<nx-1; i++){
+					n=k*nx*ny+j*nx+i;
+					if (id[n] > 1){
+						count+=1.0;
+					}
+				}
+			}
+		}
+		MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
+		void_fraction_new = countGlobal/totalGlobal;
+		void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
+		if (rank==0){
+			fprintf(DRAIN,"%f ",void_fraction_new);
+			fprintf(DRAIN,"%f\n",Rcrit_new);
+			printf("     %f ",void_fraction_new);
+			printf("     %f\n",Rcrit_new);
+		}
+	}
+
+	if (void_fraction_diff_new<void_fraction_diff_old){
+		final_void_fraction=void_fraction_new;
+		if (rank==0){
+			printf("Final void fraction =%f\n",void_fraction_new);
+			printf("Final critical radius=%f\n",Rcrit_new);
+		}
+	}
+	else{
+		final_void_fraction=void_fraction_old;
+		if (rank==0){
+			printf("Final void fraction=%f\n",void_fraction_old);
+			printf("Final critical radius=%f\n",Rcrit_old);
+		}
+	}
+	// label all WP components as 2
+	for (int k=1; k<nz-1; k++){
+		for (int j=1; j<ny-1; j++){
+			for (int i=1; i<nx-1; i++){
+				n=k*nx*ny+j*nx+i;
+				if (id[n] > 1){
+					id[n] = 2;
+				}
+			}
+		}
+	}
+	
+	return final_void_fraction;
+}
+
+double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth, double WallFactor)
+{
+	int Nx = Dm->Nx;
+	int Ny = Dm->Ny;
+	int Nz = Dm->Nz;
+	int rank = Dm->rank();
+		
+	double count=0.0;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				if (Dist(i,j,k) < 0.0){
+					count+=1.0;
+				}
+			}
+		}
+	}
+	double count_original=sumReduce( Dm->Comm, count);
+
+	// Estimate morph_delta
+	double morph_delta = 0.0;
+	if (TargetGrowth > 0.0) morph_delta = 0.1;
+	else					morph_delta = -0.1;
+	double morph_delta_previous = 0.0;
+	double GrowthEstimate = 0.0;
+	double GrowthPrevious = 0.0;
+	int COUNT_FOR_LOOP = 0;
+	double ERROR = 100.0;
+	if (rank == 0) printf("Estimate delta for growth=%f \n",TargetGrowth);
+	while ( ERROR > 0.01 && COUNT_FOR_LOOP < 10 ){
+		COUNT_FOR_LOOP++;
+		count = 0.0;
+		double MAX_DISPLACEMENT = 0.0;
+		for (int k=1; k<Nz-1; k++){
+			for (int j=1; j<Ny-1; j++){
+				for (int i=1; i<Nx-1; i++){
+					double walldist=BoundaryDist(i,j,k);
+					double wallweight = WallFactor/ (1+exp(-5.f*(walldist-1.f))); 
+					if (fabs(wallweight*morph_delta) > MAX_DISPLACEMENT) MAX_DISPLACEMENT= fabs(wallweight*morph_delta);
+					
+					if (Dist(i,j,k) - wallweight*morph_delta < 0.0){
+						count+=1.0;
+					}
+				}
+			}
+		}
+		count=sumReduce( Dm->Comm, count);
+		MAX_DISPLACEMENT = maxReduce( Dm->Comm, MAX_DISPLACEMENT);
+		GrowthEstimate = count - count_original;
+		ERROR = fabs((GrowthEstimate-TargetGrowth) /TargetGrowth);
+
+		if (rank == 0) printf("     delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT);
+		// Now adjust morph_delta
+		double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
+		GrowthPrevious = GrowthEstimate;
+		morph_delta_previous = morph_delta;
+		morph_delta += step_size;
+		if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0;
+				
+		//MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25);
+		
+		if (morph_delta > 0.0 ){
+			// object is growing
+			if (MAX_DISPLACEMENT > 3.0 ){
+				morph_delta = 3.0;
+				COUNT_FOR_LOOP = 100; // exit loop if displacement is too large
+			}
+		}
+		else{
+			// object is shrinking
+			if (MAX_DISPLACEMENT > 1.0 ){
+				morph_delta = -1.0;
+				COUNT_FOR_LOOP = 100; // exit loop if displacement is too large
+			}
+		}
+	}
+	if (rank == 0) printf("Final delta=%f \n",morph_delta);
+
+	count = 0.0;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				double walldist=BoundaryDist(i,j,k);
+				double wallweight = WallFactor / (1+exp(-5.f*(walldist-1.f))); 
+				//wallweight = 1.0;
+				Dist(i,j,k) -= wallweight*morph_delta;
+				if (Dist(i,j,k) < 0.0)	count+=1.0;
+			}
+		}
+	}
+	count=sumReduce( Dm->Comm, count);
+
+	return count;
+}
+

analysis/morphology.h

@@ -0,0 +1,8 @@
+// Morphological opening routine
+#include "common/Array.h"
+#include "common/Domain.h"
+#include "analysis/runAnalysis.h"
+
+double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction, signed char ErodeLabel, signed char ReplaceLabel);
+double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction);
+double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetVol, double WallFactor);

analysis/runAnalysis.cpp

@@ -180,28 +180,34 @@ public:
     ~WriteVisWorkItem() { }
     virtual void run() {
         PROFILE_START("Save Vis",1);
+        
         ASSERT(visData[0].vars[0]->name=="phase");
+        Array<double>& PhaseData = visData[0].vars[0]->data;
+        fillData.copy(Averages.SDn,PhaseData);
+
+        ASSERT(visData[0].vars[5]->name=="SignDist");
+        Array<double>& SignData  = visData[0].vars[5]->data;
+        fillData.copy(Averages.SDs,SignData);
+
         ASSERT(visData[0].vars[1]->name=="Pressure");
+        Array<double>& PressData = visData[0].vars[1]->data;
+        fillData.copy(Averages.Press,PressData);
+
         ASSERT(visData[0].vars[2]->name=="Velocity_x");
         ASSERT(visData[0].vars[3]->name=="Velocity_y");
         ASSERT(visData[0].vars[4]->name=="Velocity_z");
-        ASSERT(visData[0].vars[5]->name=="SignDist");
-        ASSERT(visData[0].vars[6]->name=="BlobID");
-        Array<double>& PhaseData = visData[0].vars[0]->data;
-        Array<double>& PressData = visData[0].vars[1]->data;
         Array<double>& VelxData = visData[0].vars[2]->data;
         Array<double>& VelyData = visData[0].vars[3]->data;
         Array<double>& VelzData = visData[0].vars[4]->data;
-        Array<double>& SignData  = visData[0].vars[5]->data;
-        Array<double>& BlobData  = visData[0].vars[6]->data;
-        fillData.copy(Averages.SDn,PhaseData);
-        fillData.copy(Averages.Press,PressData);
-        fillData.copy(Averages.SDs,SignData);
         fillData.copy(Averages.Vel_x,VelxData);
         fillData.copy(Averages.Vel_y,VelyData);
         fillData.copy(Averages.Vel_z,VelzData);
+        
+        ASSERT(visData[0].vars[6]->name=="BlobID");
+        Array<double>& BlobData  = visData[0].vars[6]->data;        
         fillData.copy(Averages.Label_NWP,BlobData);
         IO::writeData( timestep, visData, comm.comm );
+        
         PROFILE_STOP("Save Vis",1);
     };
 private:
@@ -213,6 +219,80 @@ private:
     runAnalysis::commWrapper comm;
 };
 
+// Helper class to write the vis file from a thread
+class IOWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+	IOWorkItem(int timestep_, std::shared_ptr<Database> input_db_, std::vector<IO::MeshDataStruct>& visData_,
+        SubPhase& Averages_, fillHalo<double>& fillData_, runAnalysis::commWrapper&& comm_ ):
+        timestep(timestep_), input_db(input_db_), visData(visData_), Averages(Averages_), fillData(fillData_), comm(std::move(comm_))
+        {
+        }
+    ~IOWorkItem() { }
+    virtual void run() {
+    	auto color_db =  input_db->getDatabase( "Color" );
+    	auto vis_db =  input_db->getDatabase( "Visualization" );
+       // int timestep = color_db->getWithDefault<int>( "timestep", 0 );
+
+        PROFILE_START("Save Vis",1);
+
+        if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
+        	ASSERT(visData[0].vars[0]->name=="phase");
+        	Array<double>& PhaseData = visData[0].vars[0]->data;
+        	fillData.copy(Averages.Phi,PhaseData);
+        }
+
+        if (vis_db->getWithDefault<bool>( "save_pressure", false )){
+        	ASSERT(visData[0].vars[1]->name=="Pressure");
+        	Array<double>& PressData = visData[0].vars[1]->data;
+        	fillData.copy(Averages.Pressure,PressData);
+        }
+
+        if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+        	ASSERT(visData[0].vars[2]->name=="Velocity_x");
+        	ASSERT(visData[0].vars[3]->name=="Velocity_y");
+        	ASSERT(visData[0].vars[4]->name=="Velocity_z");
+        	Array<double>& VelxData = visData[0].vars[2]->data;
+        	Array<double>& VelyData = visData[0].vars[3]->data;
+        	Array<double>& VelzData = visData[0].vars[4]->data;
+        	fillData.copy(Averages.Vel_x,VelxData);
+        	fillData.copy(Averages.Vel_y,VelyData);
+        	fillData.copy(Averages.Vel_z,VelzData);
+        }
+
+        if (vis_db->getWithDefault<bool>( "save_distance", false )){
+        	ASSERT(visData[0].vars[5]->name=="SignDist");
+        	Array<double>& SignData  = visData[0].vars[5]->data;
+        	fillData.copy(Averages.SDs,SignData);
+        }
+
+        if (vis_db->getWithDefault<bool>( "save_connected_components", false )){
+        	ASSERT(visData[0].vars[6]->name=="BlobID");
+        	Array<double>& BlobData  = visData[0].vars[6]->data;
+        	fillData.copy(Averages.morph_n->label,BlobData);
+        }
+        
+        if (vis_db->getWithDefault<bool>( "write_silo", true ))
+        	IO::writeData( timestep, visData, comm.comm );
+
+        if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
+        	char CurrentIDFilename[40];
+        	sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
+        	Averages.AggregateLabels(CurrentIDFilename);
+        }
+
+        PROFILE_STOP("Save Vis",1);
+    };
+private:
+    IOWorkItem();
+    int timestep;
+    std::shared_ptr<Database> input_db;
+    std::vector<IO::MeshDataStruct>& visData;
+    SubPhase& Averages;
+    fillHalo<double>& fillData;
+    runAnalysis::commWrapper comm;
+};
+
 
 // Helper class to run the analysis from within a thread
 // Note: Averages will be modified after the constructor is called
@@ -262,6 +342,138 @@ private:
 };
 
 
+class TCATWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+	TCATWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_, 
+            BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
+                type(type_), timestep(timestep_), Averages(Averages_), 
+                blob_ids(ids), id_list(id_list_), beta(beta_) { }
+    ~TCATWorkItem() { }
+    virtual void run() {
+        Averages.NumberComponents_NWP = blob_ids->first;
+        Averages.Label_NWP = blob_ids->second;
+        Averages.Label_NWP_map = *id_list;
+        Averages.NumberComponents_WP = 1;
+        Averages.Label_WP.fill(0.0);
+        if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
+            // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
+        }
+        if ( matches(type,AnalysisType::ComputeAverages) ) {
+            PROFILE_START("Compute TCAT",1);
+            Averages.Initialize();
+            Averages.ComputeDelPhi();
+            Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.SDn);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tminus,Averages.Phase_tminus);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tplus,Averages.Phase_tplus);
+            Averages.UpdateMeshValues();
+            Averages.ComputeLocal();
+            Averages.Reduce();
+            Averages.PrintAll(timestep);
+            PROFILE_STOP("Compute TCAT",1);
+        }
+    }
+private:
+    TCATWorkItem();
+    AnalysisType type;
+    int timestep;
+    TwoPhase& Averages;
+    BlobIDstruct blob_ids;
+    BlobIDList id_list;
+    double beta;
+};
+
+
+class GanglionTrackingWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+	GanglionTrackingWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_, 
+            BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
+                type(type_), timestep(timestep_), Averages(Averages_), 
+                blob_ids(ids), id_list(id_list_), beta(beta_) { }
+    ~GanglionTrackingWorkItem() { }
+    virtual void run() {
+        Averages.NumberComponents_NWP = blob_ids->first;
+        Averages.Label_NWP = blob_ids->second;
+        Averages.Label_NWP_map = *id_list;
+        Averages.NumberComponents_WP = 1;
+        Averages.Label_WP.fill(0.0);
+        if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
+            // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
+        }
+        if ( matches(type,AnalysisType::ComputeAverages) ) {
+            PROFILE_START("Compute ganglion",1);
+            Averages.Initialize();
+            Averages.ComputeDelPhi();
+            Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.SDn);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tminus,Averages.Phase_tminus);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tplus,Averages.Phase_tplus);
+            Averages.UpdateMeshValues();
+            Averages.ComponentAverages();
+            Averages.SortBlobs();
+            Averages.PrintComponents(timestep);
+            PROFILE_STOP("Compute ganglion",1);
+        }
+    }
+private:
+    GanglionTrackingWorkItem();
+    AnalysisType type;
+    int timestep;
+    TwoPhase& Averages;
+    BlobIDstruct blob_ids;
+    BlobIDList id_list;
+    double beta;
+};
+
+
+class BasicWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+	BasicWorkItem( AnalysisType type_, int timestep_, SubPhase& Averages_ ):
+                type(type_), timestep(timestep_), Averages(Averages_){ }
+    ~BasicWorkItem() { }
+    virtual void run() {
+
+        if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
+            // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
+        }
+        if ( matches(type,AnalysisType::ComputeAverages) ) {
+            PROFILE_START("Compute basic averages",1);
+            Averages.Basic();
+            PROFILE_STOP("Compute basic averages",1);
+        }
+    }
+private:
+    BasicWorkItem();
+    AnalysisType type;
+    int timestep;
+    SubPhase& Averages;
+    double beta;
+};
+
+class SubphaseWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+	SubphaseWorkItem( AnalysisType type_, int timestep_, SubPhase& Averages_ ):
+                type(type_), timestep(timestep_), Averages(Averages_){ }
+    ~SubphaseWorkItem() { }
+    virtual void run() {
+
+    	PROFILE_START("Compute subphase",1);
+    	Averages.Full();
+    	Averages.Write(timestep);
+    	PROFILE_STOP("Compute subphase",1);
+    }
+private:
+    SubphaseWorkItem();
+    AnalysisType type;
+    int timestep;
+    SubPhase& Averages;
+    double beta;
+};
+
+
+
 /******************************************************************
  *  MPI comm wrapper for use with analysis                         *
  ******************************************************************/
@@ -310,11 +522,9 @@ runAnalysis::commWrapper runAnalysis::getComm( )
 /******************************************************************
  *  Constructor/Destructors                                        *
  ******************************************************************/
-runAnalysis::runAnalysis( std::shared_ptr<Database> db,
-        const RankInfoStruct& rank_info, std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> Dm,
-        int Np, bool Regular, double beta, IntArray Map ):
+runAnalysis::runAnalysis(std::shared_ptr<Database> input_db, const RankInfoStruct& rank_info, std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> Dm,
+        int Np, bool Regular, IntArray Map ):
             d_Np( Np ),
-            d_beta( beta ),
             d_regular ( Regular),
             d_rank_info( rank_info ),
             d_Map( Map ),
@@ -322,29 +532,48 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> db,
             d_ScaLBL_Comm( ScaLBL_Comm)
 {
 
+	auto db = input_db->getDatabase( "Analysis" );
+	auto vis_db = input_db->getDatabase( "Visualization" );
+
     // Ids of work items to use for dependencies
     ThreadPool::thread_id_t d_wait_blobID;
     ThreadPool::thread_id_t d_wait_analysis;
     ThreadPool::thread_id_t d_wait_vis;
     ThreadPool::thread_id_t d_wait_restart;
+    ThreadPool::thread_id_t d_wait_subphase;
 
     char rankString[20];
     sprintf(rankString,"%05d",Dm->rank());
     d_N[0] = Dm->Nx;
     d_N[1] = Dm->Ny;
     d_N[2] = Dm->Nz;
+    
     d_restart_interval = db->getScalar<int>( "restart_interval" );
-    d_analysis_interval = db->getScalar<int>( "analysis_interval" );
-    d_blobid_interval = db->getScalar<int>( "blobid_interval" );
-    d_visualization_interval = db->getScalar<int>( "visualization_interval" );
+    d_analysis_interval = db->getScalar<int>( "analysis_interval" );    
+    d_subphase_analysis_interval = INT_MAX;
+	d_visualization_interval = INT_MAX;
+	d_blobid_interval = INT_MAX;
+	if (db->keyExists( "blobid_interval" )){
+	    d_blobid_interval = db->getScalar<int>( "blobid_interval" );
+	}
+	if (db->keyExists( "visualization_interval" )){
+	    d_visualization_interval = db->getScalar<int>( "visualization_interval" );
+	}
+	if (db->keyExists( "subphase_analysis_interval" )){
+		d_subphase_analysis_interval = db->getScalar<int>( "subphase_analysis_interval" );
+	}
+	
     auto restart_file = db->getScalar<std::string>( "restart_file" );
     d_restartFile = restart_file + "." + rankString;
+    
+    
     d_rank = MPI_WORLD_RANK();
     writeIDMap(ID_map_struct(),0,id_map_filename);
     // Initialize IO for silo
     IO::initialize("","silo","false");
     // Create the MeshDataStruct    
     d_meshData.resize(1);
+
     d_meshData[0].meshName = "domain";
     d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
     auto PhaseVar = std::make_shared<IO::Variable>();
@@ -355,43 +584,57 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> db,
     auto SignDistVar = std::make_shared<IO::Variable>();
     auto BlobIDVar = std::make_shared<IO::Variable>();
     
-    PhaseVar->name = "phase";
-    PhaseVar->type = IO::VariableType::VolumeVariable;
-    PhaseVar->dim = 1;
-    PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(PhaseVar);
-    PressVar->name = "Pressure";
-    PressVar->type = IO::VariableType::VolumeVariable;
-    PressVar->dim = 1;
-    PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(PressVar);
+    if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
+        PhaseVar->name = "phase";
+        PhaseVar->type = IO::VariableType::VolumeVariable;
+        PhaseVar->dim = 1;
+        PhaseVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(PhaseVar);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_pressure", false )){
+        PressVar->name = "Pressure";
+        PressVar->type = IO::VariableType::VolumeVariable;
+        PressVar->dim = 1;
+        PressVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(PressVar);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim = 1;
+        VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(VxVar);
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim = 1;
+        VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(VyVar);
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim = 1;
+        VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(VzVar);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_distance", false )){
+        SignDistVar->name = "SignDist";
+        SignDistVar->type = IO::VariableType::VolumeVariable;
+        SignDistVar->dim = 1;
+        SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(SignDistVar);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_connected_components", false )){
+        BlobIDVar->name = "BlobID";
+        BlobIDVar->type = IO::VariableType::VolumeVariable;
+        BlobIDVar->dim = 1;
+        BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        d_meshData[0].vars.push_back(BlobIDVar);
+    }
     
-    VxVar->name = "Velocity_x";
-    VxVar->type = IO::VariableType::VolumeVariable;
-    VxVar->dim = 1;
-    VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(VxVar);
-    VyVar->name = "Velocity_y";
-    VyVar->type = IO::VariableType::VolumeVariable;
-    VyVar->dim = 1;
-    VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(VyVar);
-    VzVar->name = "Velocity_z";
-    VzVar->type = IO::VariableType::VolumeVariable;
-    VzVar->dim = 1;
-    VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(VzVar);
-    
-    SignDistVar->name = "SignDist";
-    SignDistVar->type = IO::VariableType::VolumeVariable;
-    SignDistVar->dim = 1;
-    SignDistVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(SignDistVar);
-    BlobIDVar->name = "BlobID";
-    BlobIDVar->type = IO::VariableType::VolumeVariable;
-    BlobIDVar->dim = 1;
-    BlobIDVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-    d_meshData[0].vars.push_back(BlobIDVar);
+
     // Initialize the comms
     MPI_Comm_dup(MPI_COMM_WORLD,&d_comm);
     for (int i=0; i<1024; i++) {
@@ -423,6 +666,7 @@ void runAnalysis::finish( )
     d_wait_blobID.reset();
     d_wait_analysis.reset();
     d_wait_vis.reset();
+    d_wait_subphase.reset();
     d_wait_restart.reset();
     // Syncronize
     MPI_Barrier( d_comm );
@@ -534,10 +778,15 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep )
 /******************************************************************
  *  Run the analysis                                               *
  ******************************************************************/
-void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
+void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase& Averages, const double *Phi,
         double *Pressure, double *Velocity, double *fq, double *Den)
 {
     int N = d_N[0]*d_N[1]*d_N[2];
+    NULL_USE( N );
+    NULL_USE( Phi );
+    
+	auto db = input_db->getDatabase( "Analysis" );
+    //int timestep = db->getWithDefault<int>( "timestep", 0 );
 
     // Check which analysis steps we need to perform
     auto type = computeAnalysisType( timestep );
@@ -673,13 +922,14 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
     // Spawn a thread to write the restart file
     //    if ( matches(type,AnalysisType::CreateRestart) ) {
     if (timestep%d_restart_interval==0){
-
-        if (d_rank==0) {
-            FILE *Rst = fopen("Restart.txt","w");
-            fprintf(Rst,"%i\n",timestep+4);
-            fclose(Rst);
-        }
-        // Write the restart file (using a seperate thread)
+    	auto Restart_db = input_db->cloneDatabase();
+	//	Restart_db->putScalar<bool>( "Restart", true );
+    	if (d_rank==0) {
+	  //	std::ofstream OutStream("Restart.db");
+	  //	Restart_db->print(OutStream, "");
+	  //	OutStream.close();
+    	}
+    	// Write the restart file (using a seperate thread)
         auto work = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np);
         work->add_dependency(d_wait_restart);
         d_wait_restart = d_tpool.add_work(work);
@@ -699,4 +949,138 @@ void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
 }
 
 
+/******************************************************************
+ *  Run the analysis                                               *
+ ******************************************************************/
+void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
+{
+    // Check which analysis steps we need to perform
+	auto color_db =  input_db->getDatabase( "Color" );
+	auto vis_db =  input_db->getDatabase( "Visualization" );
 
+    //int timestep = color_db->getWithDefault<int>( "timestep", 0 );
+    auto type = computeAnalysisType( timestep );
+    if ( type == AnalysisType::AnalyzeNone )
+        return;
+
+    // Check how may queued items we have
+    if ( d_tpool.N_queued() > 20 ) {
+        std::cerr << "Analysis queue is getting behind, waiting ...\n";
+        finish();
+    }
+
+    PROFILE_START("basic");
+
+    // Copy the appropriate variables to the host (so we can spawn new threads)
+    ScaLBL_DeviceBarrier();
+    PROFILE_START("Copy data to host",1);
+
+    //if ( matches(type,AnalysisType::CopySimState) ) {
+    if ( timestep%d_analysis_interval == 0 ) {
+        finish(); // can't copy if threads are still working on data
+        // Copy the members of Averages to the cpu (phase was copied above)
+        PROFILE_START("Copy-Pressure",1);
+        ScaLBL_D3Q19_Pressure(fq,Pressure,d_Np);
+        //ScaLBL_D3Q19_Momentum(fq,Velocity,d_Np);
+        ScaLBL_DeviceBarrier();
+        PROFILE_STOP("Copy-Pressure",1);
+        PROFILE_START("Copy-Wait",1);
+        PROFILE_STOP("Copy-Wait",1);
+        PROFILE_START("Copy-State",1);
+        // copy other variables
+        d_ScaLBL_Comm->RegularLayout(d_Map,Pressure,Averages.Pressure);
+        d_ScaLBL_Comm->RegularLayout(d_Map,&Den[0],Averages.Rho_n);
+        d_ScaLBL_Comm->RegularLayout(d_Map,&Den[d_Np],Averages.Rho_w);
+        d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[0],Averages.Vel_x);
+        d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y);
+        d_ScaLBL_Comm->RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z);
+        PROFILE_STOP("Copy-State",1);
+    }
+    PROFILE_STOP("Copy data to host");
+
+    // Spawn threads to do the analysis work
+    //if (timestep%d_restart_interval==0){
+    // if ( matches(type,AnalysisType::ComputeAverages) ) {
+    if ( timestep%d_analysis_interval == 0 ) {
+        auto work = new BasicWorkItem(type,timestep,Averages);
+        work->add_dependency(d_wait_subphase);    // Make sure we are done using analysis before modifying
+        work->add_dependency(d_wait_analysis);  
+        work->add_dependency(d_wait_vis);
+        d_wait_analysis = d_tpool.add_work(work);
+    }
+    
+    if ( timestep%d_subphase_analysis_interval == 0 ) {
+        auto work = new SubphaseWorkItem(type,timestep,Averages);
+        work->add_dependency(d_wait_subphase);    // Make sure we are done using analysis before modifying
+        work->add_dependency(d_wait_analysis);  
+        work->add_dependency(d_wait_vis);
+        d_wait_subphase = d_tpool.add_work(work);
+    }
+
+    if (timestep%d_restart_interval==0){
+    	std::shared_ptr<double> cfq,cDen;
+    	// Copy restart data to the CPU
+    	cDen = std::shared_ptr<double>(new double[2*d_Np],DeleteArray<double>);
+    	cfq = std::shared_ptr<double>(new double[19*d_Np],DeleteArray<double>);
+    	ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double));
+    	ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double));
+    	// clone the input database to avoid modifying shared data
+    	auto Restart_db = input_db->cloneDatabase();
+    	auto tmp_color_db =  Restart_db->getDatabase( "Color" );
+    	tmp_color_db->putScalar<int>("timestep",timestep);    		
+    	tmp_color_db->putScalar<bool>( "Restart", true );
+    	Restart_db->putDatabase("Color", tmp_color_db);
+    	if (d_rank==0) {
+    		std::ofstream OutStream("Restart.db");
+    		Restart_db->print(OutStream, "");
+    		OutStream.close();
+    	}
+    	// Write the restart file (using a seperate thread)
+    	auto work1 = new WriteRestartWorkItem(d_restartFile.c_str(),cDen,cfq,d_Np);
+    	work1->add_dependency(d_wait_restart);
+    	d_wait_restart = d_tpool.add_work(work1);
+    }
+    
+    if (timestep%d_visualization_interval==0){
+        // Write the vis files
+         auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, d_fillData, getComm() );
+        work->add_dependency(d_wait_analysis);
+        work->add_dependency(d_wait_subphase);
+        work->add_dependency(d_wait_vis);
+        d_wait_vis = d_tpool.add_work(work);
+    }
+
+    PROFILE_STOP("basic");
+}
+
+void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
+{
+	auto color_db =  input_db->getDatabase( "Color" );
+	auto vis_db =  input_db->getDatabase( "Visualization" );
+    //int timestep = color_db->getWithDefault<int>( "timestep", 0 );
+
+    // Check which analysis steps we need to perform
+    auto type = computeAnalysisType( timestep );
+    if ( type == AnalysisType::AnalyzeNone )
+        return;
+
+    // Check how may queued items we have
+    if ( d_tpool.N_queued() > 20 ) {
+        std::cerr << "Analysis queue is getting behind, waiting ...\n";
+        finish();
+    }
+
+    // Copy the appropriate variables to the host (so we can spawn new threads)
+    ScaLBL_DeviceBarrier();
+
+    PROFILE_START("write vis",1);
+
+    // if (Averages.WriteVis == true){
+    auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, d_fillData, getComm() );
+    work2->add_dependency(d_wait_vis);
+    d_wait_vis = d_tpool.add_work(work2);
+
+    //Averages.WriteVis = false;
+    
+    PROFILE_STOP("write vis");
+}

analysis/runAnalysis.h

@@ -18,10 +18,11 @@
 
 #include "analysis/analysis.h"
 #include "analysis/TwoPhase.h"
+#include "analysis/SubPhase.h"
 #include "common/Communication.h"
 #include "common/ScaLBL.h"
 #include "threadpool/thread_pool.h"
-
+#include <limits.h>
 
 typedef std::shared_ptr<std::pair<int,IntArray>> BlobIDstruct;
 typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
@@ -29,7 +30,7 @@ typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
 
 // Types of analysis
 enum class AnalysisType : uint64_t { AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02, 
-    CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20 };
+    CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20, ComputeSubphase=0x40 };
 
 
 //! Class to run the analysis in multiple threads
@@ -38,15 +39,18 @@ class runAnalysis
 public:
 
     //! Constructor
-    runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct& rank_info,
-    		std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool Regular, double beta, IntArray Map );
+    runAnalysis(std::shared_ptr<Database> db, const RankInfoStruct& rank_info,
+    		std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool Regular, IntArray Map );
 
     //! Destructor
     ~runAnalysis();
 
     //! Run the next analysis
-    void run( int timestep, TwoPhase &Averages, const double *Phi,
+    void run(int timestep, std::shared_ptr<Database> db,  TwoPhase &Averages, const double *Phi,
         double *Pressure, double *Velocity, double *fq, double *Den );
+    
+    void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
+    void WriteVisData(int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den);
 
     //! Finish all active analysis
     void finish();
@@ -99,6 +103,7 @@ private:
     int d_Np;
     int d_rank;
     int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
+    int d_subphase_analysis_interval;
     double d_beta;
     bool d_regular;
     ThreadPool d_tpool;
@@ -118,6 +123,7 @@ private:
     // Ids of work items to use for dependencies
     ThreadPool::thread_id_t d_wait_blobID;
     ThreadPool::thread_id_t d_wait_analysis;
+    ThreadPool::thread_id_t d_wait_subphase;
     ThreadPool::thread_id_t d_wait_vis;
     ThreadPool::thread_id_t d_wait_restart;
 

analysis/uCT.cpp

@@ -172,6 +172,7 @@ void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
     //	int depth = 5;
     //	float sigsq=0.1;
 	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+    NULL_USE( nlm_count );
 	fillFloat.fill(NonLocalMean);
 }
 
@@ -216,6 +217,7 @@ void refine( const Array<float>& Dist_coarse,
 //	int depth = 3;
 //	float sigsq = 0.1;
 	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+    NULL_USE( nlm_count );
 	fillFloat.fill(NonLocalMean);
     segment( NonLocalMean, ID, 0.001 );
     for (size_t i=0; i<ID.length(); i++) {

cmake/SharedPtr.cmake

@@ -1,170 +0,0 @@
-# Create a shared_ptr.h file in the include directory that contains 
-#    a shared_ptr class (hopefully typedef to a compiler basic)
-# Arguements:
-#    INSTALL_DIR - Directory to install shared_ptr.h
-#    NAMESPACE - Namespace to contain the shared_ptr class (may be empty)
-INCLUDE( CheckCXXSourceCompiles )
-FUNCTION( CONFIGURE_SHARED_PTR INSTALL_DIR NAMESPACE )
-    SET( CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS} )
-    CHECK_CXX_SOURCE_COMPILES(
-	    "   #include <memory>
-            namespace ${NAMESPACE} { using std::shared_ptr; }
-	        int main() {
-	            ${NAMESPACE}::shared_ptr<int> ptr;
-	            return 0;
-	        }
-	    "
-	    MEMORY_SHARED_PTR )
-    CHECK_CXX_SOURCE_COMPILES(
-	    "   #include <memory>
-            namespace ${NAMESPACE} { using std::tr1::shared_ptr; }
-	        int main() {
-	            ${NAMESPACE}::shared_ptr<int> ptr;
-	            return 0;
-	        }
-	    "
-	    MEMORY_TR1_SHARED_PTR )
-    CHECK_CXX_SOURCE_COMPILES(
-	    "   #include <tr1/memory>
-            namespace  ${NAMESPACE} { using std::tr1::shared_ptr; }
-	        int main() {
-	            ${NAMESPACE}::shared_ptr<int> ptr;
-	            return 0;
-	        }
-	    "
-	    TR1_MEMORY_TR1_SHARED_PTR )
-    GET_DIRECTORY_PROPERTY( dirs INCLUDE_DIRECTORIES )
-    SET( CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS}" )
-    SET( CMAKE_REQUIRED_INCLUDES ${dirs} "${BOOST_INCLUDE}" )
-    CHECK_CXX_SOURCE_COMPILES(
-	    "   #include \"boost/shared_ptr.hpp\"
-            namespace  ${NAMESPACE} { using boost::shared_ptr; }
-	        int main() {
-	            ${NAMESPACE}::shared_ptr<int> ptr;
-	            return 0;
-	        }
-	    "
-	    BOOST_SHARED_PTR )
-    WRITE_DUMMY_SHARED_PTR( "${NAMESPACE}" "${CMAKE_CURRENT_BINARY_DIR}/tmp/dummy_shared_ptr.h" )
-    CHECK_CXX_SOURCE_COMPILES(
-	    "   #include <iostream>
-	        #include \"${CMAKE_CURRENT_BINARY_DIR}/tmp/dummy_shared_ptr.h\"
-	        int main() {
-	            ${NAMESPACE}::shared_ptr<int> ptr;
-	            return 0;
-	        }
-	    "
-	    DUMMY_SHARED_PTR )
-    IF ( NOT NAMESPACE )
-        SET( NAMESPACE " " )
-    ENDIF()
-    IF ( BOOST_SHARED_PTR )
-        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/shared_ptr.hpp\"\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/weak_ptr.hpp\"\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/enable_shared_from_this.hpp\"\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::shared_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::dynamic_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::const_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::weak_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::enable_shared_from_this; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
-    ELSEIF ( MEMORY_SHARED_PTR )
-        IF ( ${NAMESPACE} STREQUAL "std" )
-            FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
-        ELSE()
-            FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::shared_ptr; \n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::dynamic_pointer_cast; \n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::const_pointer_cast; \n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::weak_ptr; \n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::enable_shared_from_this; \n")
-            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
-        ENDIF()
-    ELSEIF ( MEMORY_TR1_SHARED_PTR )
-        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::shared_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::dynamic_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::const_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::weak_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::enable_shared_from_this; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
-    ELSEIF ( TR1_MEMORY_TR1_SHARED_PTR )
-        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <tr1/memory>\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::shared_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::dynamic_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::const_pointer_cast; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::weak_ptr; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::enable_shared_from_this; \n")
-        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
-    ELSEIF ( DUMMY_SHARED_PTR ) 
-        MESSAGE("Warning: No valid shared_ptr found, using dummy shared_ptr" )
-        WRITE_DUMMY_SHARED_PTR( "${NAMESPACE}" "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" )
-    ELSE()
-        MESSAGE(FATAL_ERROR "No shared_ptr availible")
-    ENDIF()
-    EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different 
-        "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "${INSTALL_DIR}/shared_ptr.h" )
-ENDFUNCTION()
-
-
-FUNCTION( WRITE_DUMMY_SHARED_PTR NAMESPACE FILENAME )
-    FILE(WRITE  "${FILENAME}" "#ifndef DUMMY_SHARED_PTR_INC\n")
-    FILE(APPEND "${FILENAME}" "#define DUMMY_SHARED_PTR_INC\n")
-    FILE(APPEND "${FILENAME}" "namespace dummy {\n\n")
-    FILE(APPEND "${FILENAME}" "template<class T> void DefaultDeleter(T* p) {delete p;}\n\n")
-    FILE(APPEND "${FILENAME}" "template<class T> class shared_ptr {\n")
-    FILE(APPEND "${FILENAME}" "public:\n")
-    FILE(APPEND "${FILENAME}" "    typedef void (*D)(T*);\n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr( ): obj(NULL), deleter(DefaultDeleter<T>), count(NULL) {}\n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr( T *ptr, void (*D)(T*)=DefaultDeleter<T>):\n")
-    FILE(APPEND "${FILENAME}" "        obj(ptr), deleter(D), count(NULL) { if (ptr) { count = new int; (*count)=1; } } \n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr( const shared_ptr<T>& rhs ):  \n")
-    FILE(APPEND "${FILENAME}" "        obj(rhs.get()), deleter(reinterpret_cast<D>(rhs.deleter)), count(rhs.count) { if ( count!=NULL ) { ++(*count); } } \n")
-    FILE(APPEND "${FILENAME}" "    template<class U> shared_ptr( const shared_ptr<U>& rhs ):  \n")
-    FILE(APPEND "${FILENAME}" "        obj(rhs.get()), deleter(reinterpret_cast<D>(rhs.deleter)), count(rhs.count) { if ( count!=NULL ) { ++(*count); } } \n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr& operator=( const shared_ptr<T>& rhs )\n")
-    FILE(APPEND "${FILENAME}" "        { if (this==&rhs) { return *this;} reset(); obj=rhs.obj; deleter=reinterpret_cast<D>(rhs.deleter); count=rhs.count; ++(*count); return *this; } \n")
-    FILE(APPEND "${FILENAME}" "    ~shared_ptr( ) { reset(); }\n")
-    FILE(APPEND "${FILENAME}" "    void reset( T *ptr ) { reset(); obj=ptr; count=new int; (*count)=1; }\n")
-    FILE(APPEND "${FILENAME}" "    void reset( void ) { \n")
-    FILE(APPEND "${FILENAME}" "        if ( count!=NULL) { int tmp=--(*count); if ( tmp==0 ) { deleter(obj); delete count; } } \n")
-    FILE(APPEND "${FILENAME}" "        obj=NULL; count=NULL; \n")
-    FILE(APPEND "${FILENAME}" "    }\n")
-    FILE(APPEND "${FILENAME}" "    T* get( ) const { return obj; } \n")
-    FILE(APPEND "${FILENAME}" "    T* operator->( ) const { return obj; } \n")
-    FILE(APPEND "${FILENAME}" "    const T& operator*( ) const { return *obj; } \n")
-    FILE(APPEND "${FILENAME}" "    bool operator==( const T * rhs ) const { return obj==rhs; } \n")
-    FILE(APPEND "${FILENAME}" "    bool operator!=( const T * rhs ) const { return obj!=rhs; } \n")
-    FILE(APPEND "${FILENAME}" "protected:\n")
-    FILE(APPEND "${FILENAME}" "    T *obj;\n")
-    FILE(APPEND "${FILENAME}" "    void (*deleter)(T*);\n")
-    FILE(APPEND "${FILENAME}" "    volatile int *count;\n")
-    FILE(APPEND "${FILENAME}" "template<class T1, class U> friend shared_ptr<T1> dynamic_pointer_cast( shared_ptr<U> const & );\n")
-    FILE(APPEND "${FILENAME}" "template<class T1, class U> friend shared_ptr<T1> const_pointer_cast( shared_ptr<U> const & );\n")
-    FILE(APPEND "${FILENAME}" "template<class Y> friend class shared_ptr;\n")
-    FILE(APPEND "${FILENAME}" "};\n\n")
-    FILE(APPEND "${FILENAME}" "template<class T, class U> shared_ptr<T> dynamic_pointer_cast( shared_ptr<U> const & rhs ) {\n")
-    FILE(APPEND "${FILENAME}" "    T* obj = dynamic_cast<T*>(rhs.obj);\n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr<T> ptr;\n")
-    FILE(APPEND "${FILENAME}" "    if ( obj!=NULL ) { ptr.obj = obj; ptr.count=rhs.count; ++(*ptr.count); }\n")
-    FILE(APPEND "${FILENAME}" "    return ptr;\n}\n")
-    FILE(APPEND "${FILENAME}" "template<class T, class U> shared_ptr<T> const_pointer_cast( shared_ptr<U> const & rhs ) {\n")
-    FILE(APPEND "${FILENAME}" "    T* obj = const_cast<T*>(rhs.obj);\n")
-    FILE(APPEND "${FILENAME}" "    shared_ptr<T> ptr;\n")
-    FILE(APPEND "${FILENAME}" "    if ( obj!=NULL ) { ptr.obj = obj; ptr.count=rhs.count; ++(*ptr.count); }\n")
-    FILE(APPEND "${FILENAME}" "    return ptr;\n}\n")
-    FILE(APPEND "${FILENAME}" "\n} // namespace dummy\n")
-    FILE(APPEND "${FILENAME}" "\n\n")
-    FILE(APPEND "${FILENAME}" "namespace ${NAMESPACE} {\n")
-    FILE(APPEND "${FILENAME}" "    using dummy::shared_ptr; \n")
-    FILE(APPEND "${FILENAME}" "    using dummy::dynamic_pointer_cast; \n")
-    FILE(APPEND "${FILENAME}" "    using dummy::const_pointer_cast; \n")
-    FILE(APPEND "${FILENAME}" "}\n\n")
-    FILE(APPEND "${FILENAME}" "#endif\n")
-ENDFUNCTION()
-
-

cmake/libraries.cmake

@@ -77,7 +77,7 @@ MACRO( CONFIGURE_MPI )
                 ENDIF ()
             ELSE ()
                 # Search for the MPI executable in the current directory
-                FIND_PROGRAM ( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
+                FIND_PROGRAM( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
                 IF ( NOT MPIEXEC )
                     MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
                 ENDIF()
@@ -305,4 +305,9 @@ MACRO ( CONFIGURE_LBPM )
         SET( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )
         SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}" "${LBPM_INSTALL_DIR}/lib" )
     ENDIF()
+    # Suppress some common warnings
+    IF ( USING_GCC )
+        SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder -Wno-unused-parameter")
+        SET( CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --compiler-options -Wno-reorder,-Wno-unused-parameter")
+    ENDIF()
 ENDMACRO ()

cmake/macros.cmake

@@ -848,7 +848,7 @@ FUNCTION( ADD_${PROJ}_TEST EXEFILE ${ARGN} )
     ADD_PROJ_PROVISIONAL_TEST( ${EXEFILE} )
     CREATE_TEST_NAME( ${EXEFILE} ${ARGN} )
     IF ( USE_MPI_FOR_SERIAL_TESTS )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
     ELSE()
         ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@@ -877,7 +877,7 @@ FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
     ELSEIF( ${PROCS} STREQUAL "1" )
         CREATE_TEST_NAME( "${EXEFILE}_WEEKLY" ${ARGN} )
         IF ( USE_MPI_FOR_SERIAL_TESTS )
-            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+            ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" 1 $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
             SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         ELSE()
             ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
@@ -886,7 +886,7 @@ FUNCTION( ADD_${PROJ}_WEEKLY_TEST EXEFILE PROCS ${ARGN} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ELSEIF( (USE_MPI OR USE_EXT_MPI) AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
         CREATE_TEST_NAME( "${EXEFILE}_${PROCS}procs_WEEKLY" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -909,7 +909,7 @@ FUNCTION( ADD_${PROJ}_TEST_PARALLEL EXEFILE PROCS ${ARGN} )
     ELSEIF ( ${PROCS} GREATER ${TEST_MAX_PROCS} )
         MESSAGE("Disabling test ${TESTNAME} (exceeds maximum number of processors ${TEST_MAX_PROCS})")
     ELSE()
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -930,7 +930,7 @@ MACRO( ADD_${PROJ}_TEST_THREAD_MPI EXEFILE PROCS THREADS ${ARGN} )
         SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
     ELSEIF ( USE_MPI OR USE_EXT_MPI )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
         SET_TESTS_PROPERTIES ( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${TOT_PROCS} )
         ADD_RESOURCE_LOCK( ${TESTNAME} ${EXEFILE} ${ARGN} )
@@ -966,7 +966,7 @@ FUNCTION( ADD_${PROJ}_EXAMPLE EXEFILE PROCS ${ARGN} )
         ADD_TEST( NAME ${TESTNAME} COMMAND $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
     ELSEIF ( USE_EXT_MPI AND NOT (${PROCS} GREATER ${TEST_MAX_PROCS}) )
         CREATE_TEST_NAME( "example--${EXEFILE}_${PROCS}procs" ${ARGN} )
-        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
+        ADD_TEST( NAME ${TESTNAME} COMMAND ${MPIEXEC} ${MPIFLAGS} "${MPIEXEC_NUMPROC_FLAG}" ${PROCS} $<TARGET_FILE:${LAST_TEST}> ${ARGN} )
         SET_PROPERTY( TEST ${TESTNAME} APPEND PROPERTY ENVIRONMENT OMPI_MCA_hwloc_base_binding_policy=none )
     ENDIF()
     SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )

common/Array.hpp

@@ -724,15 +724,15 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::repmat(
     for ( size_t i4 = 0, index = 0; i4 < N1[4]; i4++ ) {
         for ( size_t j4 = 0; j4 < Nr[4]; j4++ ) {
             for ( size_t i3 = 0; i3 < N1[3]; i3++ ) {
-                for ( size_t j4 = 0; j4 < Nr[3]; j4++ ) {
+                for ( size_t j3 = 0; j3 < Nr[3]; j3++ ) {
                     for ( size_t i2 = 0; i2 < N1[2]; i2++ ) {
-                        for ( size_t j4 = 0; j4 < Nr[2]; j4++ ) {
+                        for ( size_t j2 = 0; j2 < Nr[2]; j2++ ) {
                             for ( size_t i1 = 0; i1 < N1[1]; i1++ ) {
-                                for ( size_t j4 = 0; j4 < Nr[1]; j4++ ) {
+                                for ( size_t j1 = 0; j1 < Nr[1]; j1++ ) {
                                     for ( size_t i0 = 0; i0 < N1[0]; i0++ ) {
                                         size_t k = d_size.index( i0, i1, i2, i3, i4 );
                                         TYPE x   = d_data[k];
-                                        for ( size_t j4 = 0; j4 < Nr[0]; j4++, index++ )
+                                        for ( size_t j0 = 0; j0 < Nr[0]; j0++, index++ )
                                             y2[index] = x;
                                     }
                                 }
@@ -1147,9 +1147,8 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::vector<
  *  Interpolate                                          *
  ********************************************************/
 template<class T>
-struct is_compatible_double : std::integral_constant<bool,
-                                  std::is_floating_point<typename std::remove_cv<T>::type>::value ||
-                                      std::is_integral<typename std::remove_cv<T>::type>::value> {
+struct is_compatible_double
+    : std::integral_constant<bool, std::is_floating_point<T>::value || std::is_integral<T>::value> {
 };
 template<class TYPE>
 inline typename std::enable_if<is_compatible_double<TYPE>::value, TYPE>::type Array_interp_1D(

common/Communication.cpp

@@ -19,16 +19,28 @@
 /********************************************************
 *  Structure to store the rank info                     *
 ********************************************************/
-inline int getRankForBlock( int nprocx, int nprocy, int nprocz, int i, int j, int k )
+int RankInfoStruct::getRankForBlock( int i, int j, int k ) const
 {
-	int i2 = (i+nprocx)%nprocx;
-	int j2 = (j+nprocy)%nprocy;
-	int k2 = (k+nprocz)%nprocz;
-	return i2 + j2*nprocx + k2*nprocx*nprocy;
+	int i2 = (i+nx)%nx;
+	int j2 = (j+ny)%ny;
+	int k2 = (k+nz)%nz;
+	return i2 + j2*nx + k2*nx*ny;
 }
 RankInfoStruct::RankInfoStruct()
 {
-    memset(this,0,sizeof(RankInfoStruct));
+    nx = 0;
+    ny = 0;
+    nz = 0;
+    ix = -1;
+    jy = -1;
+    kz = -1;
+    for (int i=-1; i<=1; i++) {
+        for (int j=-1; j<=1; j++) {
+            for (int k=-1; k<=1; k++) {
+                rank[i+1][j+1][k+1] = -1;
+            }
+        }
+    }
 }
 RankInfoStruct::RankInfoStruct( int rank0, int nprocx, int nprocy, int nprocz )
 {
@@ -36,17 +48,30 @@ RankInfoStruct::RankInfoStruct( int rank0, int nprocx, int nprocy, int nprocz )
     nx = nprocx;
     ny = nprocy;
     nz = nprocz;
-	ix = rank0%nprocx;
-	jy = (rank0/nprocx)%nprocy;
-	kz = rank0/(nprocx*nprocy);
-    for (int i=-1; i<=1; i++) {
-        for (int j=-1; j<=1; j++) {
-            for (int k=-1; k<=1; k++) {
-                rank[i+1][j+1][k+1] = getRankForBlock(nprocx,nprocy,nprocz,ix+i,jy+j,kz+k);
+    if ( rank0 >= nprocx * nprocy * nprocz ) {
+	    ix = -1;
+	    jy = -1;
+	    kz = -1;
+        for (int i=-1; i<=1; i++) {
+            for (int j=-1; j<=1; j++) {
+                for (int k=-1; k<=1; k++) {
+                    rank[i+1][j+1][k+1] = -1;
+                }
             }
         }
+    } else {
+	    ix = rank0%nprocx;
+	    jy = (rank0/nprocx)%nprocy;
+	    kz = rank0/(nprocx*nprocy);
+        for (int i=-1; i<=1; i++) {
+            for (int j=-1; j<=1; j++) {
+                for (int k=-1; k<=1; k++) {
+                    rank[i+1][j+1][k+1] = getRankForBlock(ix+i,jy+j,kz+k);
+                }
+            }
+        }
+        ASSERT(rank[1][1][1]==rank0);
     }
-    ASSERT(rank[1][1][1]==rank0);
 }
 
 

common/Communication.h

@@ -46,9 +46,16 @@ struct RankInfoStruct {
     int rank[3][3][3];      //!<  The rank for the neighbor [i][j][k]
     RankInfoStruct();
     RankInfoStruct( int rank, int nprocx, int nprocy, int nprocz );
+    int getRankForBlock( int i, int j, int k ) const;
 };
 
 
+//! Redistribute domain data (dst may be smaller than the src)
+template<class TYPE>
+Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
+    const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm );
+
+
 /*!
  * @brief  Communicate halo
  * @details  Fill the halo cells in an array from the neighboring processes

common/Communication.hpp

@@ -38,7 +38,90 @@
 
 
 /********************************************************
-*  Structure to store the rank info                     *
+*  Redistribute data between two grids                  *
+********************************************************/
+template<class TYPE>
+Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
+    const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm )
+{
+#ifdef USE_MPI
+    // Get the src size
+    std::array<int,3> src_size;
+    int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) };
+    MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm );
+    if ( !src_data.empty() )
+        ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] );
+    // Check that dst_size matches on all ranks
+    MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm );
+    ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] );
+    // Function to get overlap range
+    auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) {
+        std::vector<size_t> index;
+        if ( i1[0] > j2[0] || i2[0] < j1[0] || i1[1] > j2[1] || i2[1] < j1[1] || i1[2] > j2[2] || i2[2] < j1[2] )
+            return index;
+        index.resize( 6 );
+        index[0] = std::max( j1[0] - i1[0], 0 );
+        index[1] = std::min( j2[0] - i1[0], i2[0] - i1[0] );
+        index[2] = std::max( j1[1] - i1[1], 0 );
+        index[3] = std::min( j2[1] - i1[1], i2[1] - i1[1] );
+        index[4] = std::max( j1[2] - i1[2], 0 );
+        index[5] = std::min( j2[2] - i1[2], i2[2] - i1[2] );
+        return index;
+    };
+    // Pack and send my data to the appropriate ranks (including myself)
+    std::vector<int> send_rank;
+    std::vector<Array<TYPE>> send_data;
+    if ( !src_data.empty() ) {
+        int i1[3] = { src_size[0] * src_rank.ix, src_size[1] * src_rank.jy, src_size[2] * src_rank.kz };
+        int i2[3] = { i1[0] + src_size[0] - 1, i1[1] + src_size[1] - 1, i1[2] + src_size[2] - 1 };
+        for ( int i=0; i<dst_rank.nx; i++ ) {
+            for ( int j=0; j<dst_rank.ny; j++ ) {
+                for ( int k=0; k<dst_rank.nz; k++ ) {
+                    int j1[3] = { i * dst_size[0], j * dst_size[1], k * dst_size[2] };
+                    int j2[3] = { j1[0] + dst_size[0] - 1, j1[1] + dst_size[1] - 1, j1[2] + dst_size[2] - 1 };
+                    auto index = calcOverlap( i1, i2, j1, j2 );
+                    if ( index.empty() )
+                        continue;
+                    send_rank.push_back( dst_rank.getRankForBlock(i,j,k) );
+                    send_data.push_back( src_data.subset( index ) );
+                }
+            }
+        }
+    }
+    std::vector<MPI_Request> send_request( send_rank.size() );
+    for (size_t i=0; i<send_rank.size(); i++)
+        MPI_Isend( send_data[i].data(), sizeof(TYPE)*send_data[i].length(), MPI_BYTE, send_rank[i], 5462, comm, &send_request[i]);
+    // Unpack data from the appropriate ranks (including myself)
+    Array<TYPE> dst_data( dst_size[0], dst_size[1], dst_size[2] );
+    int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz };
+    int i2[3] = { i1[0] + dst_size[0] - 1, i1[1] + dst_size[1] - 1, i1[2] + dst_size[2] - 1 };
+    for ( int i=0; i<src_rank.nx; i++ ) {
+        for ( int j=0; j<src_rank.ny; j++ ) {
+            for ( int k=0; k<src_rank.nz; k++ ) {
+                int j1[3] = { i * src_size[0], j * src_size[1], k * src_size[2] };
+                int j2[3] = { j1[0] + src_size[0] - 1, j1[1] + src_size[1] - 1, j1[2] + src_size[2] - 1 };
+                auto index = calcOverlap( i1, i2, j1, j2 );
+                if ( index.empty() )
+                    continue;
+                int rank  = src_rank.getRankForBlock(i,j,k);
+                Array<TYPE> data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 );
+                MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE );
+                dst_data.copySubset( index, data );
+            }
+        }
+    }
+    // Free data
+    MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE );
+    return dst_data;
+#else
+    return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 );
+#endif
+}
+
+
+
+/********************************************************
+*  Structure to fill halo cells                         *
 ********************************************************/
 template<class TYPE>
 fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,

common/Domain.h

@@ -31,7 +31,6 @@
 #include "common/Communication.h"
 #include "common/Database.h"
 
-
 class Domain;
 template<class TYPE> class PatchData;
 
@@ -122,8 +121,14 @@ private:
 
 public: // Public variables (need to create accessors instead)
 
-    double Lx,Ly,Lz,Volume;
+    std::shared_ptr<Database> database;
+    double Lx,Ly,Lz,Volume,voxel_length;
     int Nx,Ny,Nz,N;
+    int inlet_layers_x, inlet_layers_y, inlet_layers_z;
+    int outlet_layers_x, outlet_layers_y, outlet_layers_z;
+    int inlet_layers_phase; //as usual: 1->n, 2->w
+    int outlet_layers_phase;
+    double porosity;
     RankInfoStruct rank_info;
 
     MPI_Comm Comm;        // MPI Communicator for this domain
@@ -135,6 +140,7 @@ public: // Public variables (need to create accessors instead)
     //**********************************
     // MPI ranks for all 18 neighbors
     //**********************************
+    inline double Porosity() const { return porosity; }
     inline int iproc() const { return rank_info.ix; }
     inline int jproc() const { return rank_info.jy; }
     inline int kproc() const { return rank_info.kz; }
@@ -183,17 +189,21 @@ public: // Public variables (need to create accessors instead)
     int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
     //......................................................................................    
     // Solid indicator function
-    char *id;
+    signed char *id;
 
     void ReadIDs();
+    void Decomp( const std::string& filename );
+    void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
     void CommunicateMeshHalo(DoubleArray &Mesh);
     void CommInit(); 
     int PoreCount();
+    void AggregateLabels( const std::string& filename );
+    void AggregateLabels( const std::string& filename, DoubleArray &UserData );
 
 private:
 
-    void PackID(int *list, int count, char *sendbuf, char *ID);
-    void UnpackID(int *list, int count, char *recvbuf, char *ID);
+    void PackID(int *list, int count, signed char *sendbuf, signed char *ID);
+    void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID);
     void CommHaloIDs();
     
 	//......................................................................................
@@ -251,10 +261,13 @@ private:
 
 };
 
-void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np);
+//void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
+//void ReadFromFile(const std::string& Filename, DoubleArray &Mesh);
 
-void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np);
+void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np);
 
-void ReadBinaryFile(char *FILENAME, double *Data, int N);
+void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, size_t Np);
+
+void ReadBinaryFile(char *FILENAME, double *Data, size_t N);
 
 #endif

common/MPI_Helpers.cpp

@@ -51,7 +51,7 @@ template<> MPI_Datatype getMPItype<double>() {
 ********************************************************/
 // unsigned char
 template<>
-size_t packsize<unsigned char>( const unsigned char& rhs )
+size_t packsize<unsigned char>( const unsigned char& )
 {
     return sizeof(unsigned char);
 }
@@ -67,7 +67,7 @@ void unpack<unsigned char>( unsigned char& data, const char *buffer )
 }
 // char
 template<>
-size_t packsize<char>( const char& rhs )
+size_t packsize<char>( const char& )
 {
     return sizeof(char);
 }
@@ -83,7 +83,7 @@ void unpack<char>( char& data, const char *buffer )
 }
 // int
 template<>
-size_t packsize<int>( const int& rhs )
+size_t packsize<int>( const int& )
 {
     return sizeof(int);
 }
@@ -99,7 +99,7 @@ void unpack<int>( int& data, const char *buffer )
 }
 // unsigned int
 template<>
-size_t packsize<unsigned int>( const unsigned int& rhs )
+size_t packsize<unsigned int>( const unsigned int& )
 {
     return sizeof(unsigned int);
 }
@@ -115,7 +115,7 @@ void unpack<unsigned int>( unsigned int& data, const char *buffer )
 }
 // size_t
 template<>
-size_t packsize<size_t>( const size_t& rhs )
+size_t packsize<size_t>( const size_t& )
 {
     return sizeof(size_t);
 }

common/MPI_Helpers.h

@@ -203,6 +203,12 @@ inline int sumReduce( MPI_Comm comm, int x )
 	MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm);
     return y;
 }
+inline long long sumReduce( MPI_Comm comm, long long x )
+{
+    long long y = 0;
+	MPI_Allreduce(&x,&y,1,MPI_LONG_LONG,MPI_SUM,comm);
+    return y;
+}
 inline bool sumReduce( MPI_Comm comm, bool x )
 {
     int y = sumReduce( comm, x?1:0 );

common/ReadMicroCT.cpp

@@ -0,0 +1,118 @@
+#include "common/ReadMicroCT.h"
+#include "common/Utilities.h"
+
+#include "zlib.h"
+
+#include <cstring>
+
+
+// Read a file into memory
+std::vector<char> readFile( const std::string& filename )
+{
+    auto fid = fopen( filename.c_str(), "rb" );
+    INSIST( fid, "File does not exist: " + filename );
+    fseek( fid, 0, SEEK_END );
+    size_t bytes = ftell(fid);
+    fseek( fid, 0, SEEK_SET );
+    std::vector<char> data( bytes );
+    size_t bytes2 = fread( data.data(), 1, bytes, fid );
+    ASSERT( bytes == bytes2 );
+    fclose( fid );
+    return data;
+}
+
+
+// Decompress a gzip buffer
+std::vector<char> gunzip( const std::vector<char>& in )
+{
+    z_stream stream;
+    std::vector<char> out( 1000000 );
+    stream.next_in = (Bytef*) in.data();
+    stream.avail_in = in.size();
+    stream.total_in = 0;
+    stream.zalloc = Z_NULL;
+    stream.zfree = Z_NULL;
+    stream.opaque = Z_NULL;
+    stream.next_out = (Bytef*) out.data();
+    stream.avail_out = out.size();
+    stream.total_out = 0;
+    ASSERT( inflateInit2(&stream, 16+MAX_WBITS) == Z_OK );
+    bool finished = inflate(&stream, Z_SYNC_FLUSH) == Z_STREAM_END;
+    while ( !finished && stream.msg == Z_NULL ) {
+        out.resize( 2 * out.size() );
+        stream.next_out = (Bytef*) &out[stream.total_out];
+        stream.avail_out = out.size() - stream.total_out;
+        finished = inflate(&stream, Z_SYNC_FLUSH) == Z_STREAM_END;
+    }
+    ASSERT( stream.msg == Z_NULL );
+    out.resize( stream.total_out );
+    inflateEnd(&stream);
+    return out;
+}
+
+
+// Read the compressed micro CT data
+Array<uint8_t> readMicroCT( const std::string& filename )
+{
+    auto in  = readFile( filename );
+    auto out = gunzip( in );
+    ASSERT( out.size() == 1024*1024*1024 );
+    Array<uint8_t> data( 1024, 1024, 1024 );
+    memcpy( data.data(), out.data(), data.length() );
+    return data;
+}
+
+
+// Read the compressed micro CT data and distribute
+Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
+{
+    // Get the local problem info
+    auto n = domain.getVector<int>( "n" );
+    int rank = comm_rank(MPI_COMM_WORLD);
+    auto nproc = domain.getVector<int>( "nproc" );
+    RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );
+    
+    // Determine the largest file number to get
+    int Nfx = ( n[0] * rankInfo.nx + 1023 ) / 1024;
+    int Nfy = ( n[1] * rankInfo.ny + 1023 ) / 1024;
+    int Nfz = ( n[2] * rankInfo.nz + 1023 ) / 1024;
+
+    // Load one of the files if rank < largest file
+    Array<uint8_t> data;
+    RankInfoStruct srcRankInfo( rank, Nfx, Nfy, Nfz );
+    if ( srcRankInfo.ix >= 0 ) {
+    	auto filename = domain.getScalar<std::string>( "Filename" );
+    	char tmp[100];
+    	if ( filename.find( "0x_0y_0z.gbd.gz" ) != std::string::npos ) {
+    		sprintf( tmp, "%ix_%iy_%iz.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
+    		filename = filename.replace( filename.find( "0x_0y_0z.gbd.gz" ), 15, std::string( tmp ) );
+    	} else if ( filename.find( "x0_y0_z0.gbd.gz" ) != std::string::npos ) {
+    		sprintf( tmp, "x%i_y%i_z%i.gbd.gz", srcRankInfo.ix, srcRankInfo.jy, srcRankInfo.kz );
+    		filename = filename.replace( filename.find( "x0_y0_z0.gbd.gz" ), 15, std::string( tmp ) );
+    	} else {
+    		ERROR( "Invalid name for first file" );
+    	}
+    	data = readMicroCT( filename );
+    }
+
+    // Redistribute the data
+    data = redistribute( srcRankInfo, data, rankInfo, { n[0], n[1], n[2] }, comm );
+
+	// Relabel the data
+    auto ReadValues = domain.getVector<int>( "ReadValues" );
+    auto WriteValues = domain.getVector<int>( "WriteValues" );
+    ASSERT( ReadValues.size() == WriteValues.size() );
+    int readMaxValue = 0;
+    for ( auto v : ReadValues )
+        readMaxValue = std::max( data.max()+1, v );
+    std::vector<int> map( readMaxValue + 1, -1 );
+    for ( size_t i=0; i<ReadValues.size(); i++ )
+        map[ReadValues[i]] = WriteValues[i];
+    for ( size_t i=0; i<data.length(); i++ ) {
+        int t = data(i);
+        ASSERT( t >= 0 && t <= readMaxValue );
+        data(i) = map[t];
+    }
+
+    return data;
+}

common/ReadMicroCT.h

@@ -0,0 +1,15 @@
+#ifndef READMICROCT_H
+#define READMICROCT_H
+
+
+#include "common/Array.h"
+#include "common/Communication.h"
+#include "common/Database.h"
+
+
+Array<uint8_t> readMicroCT( const std::string& filename );
+
+Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm );
+
+
+#endif

common/ScaLBL.cpp

@@ -93,45 +93,46 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
 	nprocy = Dm->nprocy();
 	nprocz = Dm->nprocz();
 	BoundaryCondition = Dm->BoundaryCondition;
+	//BoundaryCondition = 0; // default to periodic BC
 	//......................................................................................
 
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 5*sendCount_x*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 5*sendCount_X*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 5*sendCount_y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 5*sendCount_Y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 5*sendCount_z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 5*sendCount_Z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, sendCount_xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, sendCount_xY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, sendCount_XY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, sendCount_xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, sendCount_yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, 2*5*sendCount_x*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, 2*5*sendCount_X*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, 2*5*sendCount_y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, 2*5*sendCount_Y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, 2*5*sendCount_z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, 2*5*sendCount_Z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, 2*sendCount_xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, 2*sendCount_xY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, 2*sendCount_Xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, 2*sendCount_XY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, 2*sendCount_xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, 2*sendCount_xZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, 2*sendCount_Xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, 2*sendCount_XZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, 2*sendCount_yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, 2*sendCount_yZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, 2*sendCount_Yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, 2*sendCount_YZ*sizeof(double));	// Allocate device memory
 	//......................................................................................
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 5*recvCount_x*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 5*recvCount_X*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 5*recvCount_y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 5*recvCount_Y*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 5*recvCount_z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 5*recvCount_Z*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, recvCount_xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, recvCount_xY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, recvCount_XY*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, recvCount_xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, recvCount_yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double));	// Allocate device memory
-	ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, 2*5*recvCount_x*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, 2*5*recvCount_X*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, 2*5*recvCount_y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, 2*5*recvCount_Y*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, 2*5*recvCount_z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, 2*5*recvCount_Z*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, 2*recvCount_xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, 2*recvCount_xY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, 2*recvCount_Xy*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, 2*recvCount_XY*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, 2*recvCount_xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, 2*recvCount_xZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, 2*recvCount_Xz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, 2*recvCount_XZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, 2*recvCount_yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, 2*recvCount_yZ*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, 2*recvCount_Yz*sizeof(double));	// Allocate device memory
+	ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, 2*recvCount_YZ*sizeof(double));	// Allocate device memory
 	//......................................................................................
 	ScaLBL_AllocateZeroCopy((void **) &dvcSendList_x, sendCount_x*sizeof(int));	// Allocate device memory
 	ScaLBL_AllocateZeroCopy((void **) &dvcSendList_X, sendCount_X*sizeof(int));	// Allocate device memory
@@ -367,7 +368,7 @@ void ScaLBL_Communicator::D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list,  i
 	delete [] ReturnDist;
 }
 
-int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, char *id, int Np){
+int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np){
 	/*
 	 * Generate a memory optimized layout
 	 *   id[n] == 0 implies that site n should be ignored (treat as a mask)
@@ -387,7 +388,11 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 	for (k=0;k<Nz;k++){
 		for (j=0;j<Ny;j++){
 			for (i=0;i<Nx;i++){
-				Map(i,j,k) = -2;
+				n = k*Nx*Ny + j*Nx + i;
+				if (id[n] > 0)
+					Map(i,j,k) = -2; // this label is for parallel communication sites
+				else
+					Map(i,j,k) = -1; // this label is for solid bounce-back sites
 			}
 		}
 	}
@@ -537,7 +542,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 			}
 		}
 	}
-
+	
 	//for (idx=0; idx<Np; idx++)	printf("%i: %i %i\n", idx, neighborList[Np],  neighborList[Np+idx]);
 	//.......................................................................
 	// Now map through  SendList and RecvList to update indices
@@ -864,6 +869,239 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 	return(Np);
 }
 
+
+void ScaLBL_Communicator::SetupBounceBackList(IntArray &Map, signed char *id, int Np)
+{
+
+    int idx,i,j,k;
+    int neighbor;    
+	// save list of bounce-back distributions and interaction sites
+	n_bb_d3q7 = 0; n_bb_d3q19 = 0;
+	
+	int local_count = 0;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					neighbor=Map(i-1,j,k);
+					if (neighbor==-1)	  local_count++;
+
+					neighbor=Map(i+1,j,k);
+					if (neighbor==-1)	  local_count++;
+
+					neighbor=Map(i,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j,k-1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i+1,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j+1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i+1,j-1,k);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j,k-1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i+1,j,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i-1,j,k+1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i+1,j,k-1);
+					if (neighbor==-1)     local_count++;
+
+					neighbor=Map(i,j-1,k-1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j-1,k+1);
+					if (neighbor==-1)	    local_count++;
+
+					neighbor=Map(i,j+1,k-1);
+					if (neighbor==-1)	    local_count++;
+				}
+			}
+		}
+	}
+	
+	int *bb_dist_tmp = new int [local_count];	
+	int *bb_interactions_tmp = new int [local_count];	
+	ScaLBL_AllocateDeviceMemory((void **) &bb_dist, sizeof(int)*local_count);
+	ScaLBL_AllocateDeviceMemory((void **) &bb_interactions, sizeof(int)*local_count);
+	
+	local_count=0;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					int neighbor;    // cycle through the neighbors of lattice site idx
+					neighbor=Map(i-1,j,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 2*Np;
+					}
+
+					neighbor=Map(i+1,j,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++] = idx + 1*Np;
+					}
+
+					neighbor=Map(i,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 4*Np;
+					}
+
+					neighbor=Map(i,j+1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 3*Np;
+					}
+
+					neighbor=Map(i,j,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 6*Np;
+					}
+
+					neighbor=Map(i,j,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 5*Np;
+					}
+				}
+			}
+		}
+	}
+	n_bb_d3q7 = local_count;
+	for (k=1;k<Nz-1;k++){
+		for (j=1;j<Ny-1;j++){
+			for (i=1;i<Nx-1;i++){
+				n=k*Nx*Ny+j*Nx+i;
+				idx=Map(i,j,k);
+				if (!(idx<0)){
+
+					neighbor=Map(i-1,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 8*Np;
+					}
+
+					neighbor=Map(i+1,j+1,k);
+					if (neighbor==-1)	{
+						bb_interactions_tmp[local_count] = (i+1) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 7*Np;
+					}
+
+					neighbor=Map(i-1,j+1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i-1) + (j+1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 10*Np;
+					}
+
+					neighbor=Map(i+1,j-1,k);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j-1)*Nx + (k)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 9*Np;
+					}
+
+					neighbor=Map(i-1,j,k-1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 12*Np;
+					}
+
+					neighbor=Map(i+1,j,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 11*Np;
+					}
+
+					neighbor=Map(i-1,j,k+1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i-1) + (j)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 14*Np;
+					}
+
+					neighbor=Map(i+1,j,k-1);
+					if (neighbor==-1) {
+						bb_interactions_tmp[local_count] = (i+1) + (j)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 13*Np;
+					}
+
+					neighbor=Map(i,j-1,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 16*Np;
+					}
+
+					neighbor=Map(i,j+1,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 15*Np;
+					}
+
+					neighbor=Map(i,j-1,k+1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j-1)*Nx + (k+1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 18*Np;
+					}
+
+					neighbor=Map(i,j+1,k-1);
+					if (neighbor==-1){
+						bb_interactions_tmp[local_count] = (i) + (j+1)*Nx + (k-1)*Nx*Ny;
+						bb_dist_tmp[local_count++]=idx + 17*Np;
+					}
+				}
+			}
+		}
+	}
+	n_bb_d3q19 = local_count; // this gives the d3q19 distributions not part of d3q7 model
+	ScaLBL_CopyToDevice(bb_dist, bb_dist_tmp, local_count*sizeof(int));
+	ScaLBL_CopyToDevice(bb_interactions, bb_interactions_tmp, local_count*sizeof(int));
+	ScaLBL_DeviceBarrier();
+    
+	delete [] bb_dist_tmp;
+	delete [] bb_interactions_tmp;
+}
+
+void ScaLBL_Communicator::SolidDirichletD3Q7(double *fq, double *BoundaryValue){
+	// fq is a D3Q7 distribution
+	// BoundaryValues is a list of values to assign at bounce-back solid sites
+    ScaLBL_Solid_Dirichlet_D3Q7(fq,BoundaryValue,bb_dist,bb_interactions,n_bb_d3q7);
+}
+
+void ScaLBL_Communicator::SolidNeumannD3Q7(double *fq, double *BoundaryValue){
+	// fq is a D3Q7 distribution
+	// BoundaryValues is a list of values to assign at bounce-back solid sites
+    ScaLBL_Solid_Neumann_D3Q7(fq,BoundaryValue,bb_dist,bb_interactions,n_bb_d3q7);
+}
+
+
 void ScaLBL_Communicator::SendD3Q19AA(double *dist){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
@@ -1026,19 +1264,6 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
 	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Y,3*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N);
 	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Y,4*recvCount_Y,recvCount_Y,recvbuf_Y,dist,N);
 	//...................................................................................
-	//...Packing for z face(6,12,13,16,17)................................
-	ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
-	//...Packing for Z face(5,11,14,15,18)................................
-	ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
-	//..................................................................................
 	//...Pack the xy edge (8)................................
 	ScaLBL_D3Q19_Unpack(8,dvcRecvDist_xy,0,recvCount_xy,recvbuf_xy,dist,N);
 	//...Pack the Xy edge (9)................................
@@ -1047,22 +1272,75 @@ void ScaLBL_Communicator::RecvD3Q19AA(double *dist){
 	ScaLBL_D3Q19_Unpack(10,dvcRecvDist_xY,0,recvCount_xY,recvbuf_xY,dist,N);
 	//...Pack the XY edge (7)................................
 	ScaLBL_D3Q19_Unpack(7,dvcRecvDist_XY,0,recvCount_XY,recvbuf_XY,dist,N);
-	//...Pack the xz edge (12)................................
-	ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
-	//...Pack the xZ edge (14)................................
-	ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
-	//...Pack the Xz edge (13)................................
-	ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
-	//...Pack the XZ edge (11)................................
-	ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
-	//...Pack the yz edge (16)................................
-	ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
-	//...Pack the yZ edge (18)................................
-	ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
-	//...Pack the Yz edge (17)................................
-	ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
-	//...Pack the YZ edge (15)................................
-	ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+			//...Pack the xz edge (12)................................
+			ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
+			//...Pack the Xz edge (13)................................
+			ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
+			//...Pack the yz edge (16)................................
+			ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
+			//...Pack the Yz edge (17)................................
+			ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
+			//..................................................................................
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+			//...Pack the xZ edge (14)................................
+			ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
+			//...Pack the XZ edge (11)................................
+			ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
+			//...Pack the yZ edge (18)................................
+			ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
+			//...Pack the YZ edge (15)................................
+			ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+			//..................................................................................
+		}
+	}
+	else {
+		//...Packing for z face(6,12,13,16,17)................................
+		ScaLBL_D3Q19_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(12,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(13,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(16,dvcRecvDist_z,3*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		ScaLBL_D3Q19_Unpack(17,dvcRecvDist_z,4*recvCount_z,recvCount_z,recvbuf_z,dist,N);
+		//...Packing for Z face(5,11,14,15,18)................................
+		ScaLBL_D3Q19_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(11,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(14,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(15,dvcRecvDist_Z,3*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		ScaLBL_D3Q19_Unpack(18,dvcRecvDist_Z,4*recvCount_Z,recvCount_Z,recvbuf_Z,dist,N);
+		//..................................................................................
+		//...Pack the xz edge (12)................................
+		ScaLBL_D3Q19_Unpack(12,dvcRecvDist_xz,0,recvCount_xz,recvbuf_xz,dist,N);
+		//...Pack the xZ edge (14)................................
+		ScaLBL_D3Q19_Unpack(14,dvcRecvDist_xZ,0,recvCount_xZ,recvbuf_xZ,dist,N);
+		//...Pack the Xz edge (13)................................
+		ScaLBL_D3Q19_Unpack(13,dvcRecvDist_Xz,0,recvCount_Xz,recvbuf_Xz,dist,N);
+		//...Pack the XZ edge (11)................................
+		ScaLBL_D3Q19_Unpack(11,dvcRecvDist_XZ,0,recvCount_XZ,recvbuf_XZ,dist,N);
+		//...Pack the yz edge (16)................................
+		ScaLBL_D3Q19_Unpack(16,dvcRecvDist_yz,0,recvCount_yz,recvbuf_yz,dist,N);
+		//...Pack the yZ edge (18)................................
+		ScaLBL_D3Q19_Unpack(18,dvcRecvDist_yZ,0,recvCount_yZ,recvbuf_yZ,dist,N);
+		//...Pack the Yz edge (17)................................
+		ScaLBL_D3Q19_Unpack(17,dvcRecvDist_Yz,0,recvCount_Yz,recvbuf_Yz,dist,N);
+		//...Pack the YZ edge (15)................................
+		ScaLBL_D3Q19_Unpack(15,dvcRecvDist_YZ,0,recvCount_YZ,recvbuf_YZ,dist,N);
+	}
+
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
@@ -1085,6 +1363,7 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
 	ScaLBL_Gradient_Unpack(1.0,-1,0,0,dvcRecvDist_x,0,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,-1,0,dvcRecvDist_x,recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,1,0,dvcRecvDist_x,2*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
+	ScaLBL_Gradient_Unpack(0.5,-1,0,-1,dvcRecvDist_x,3*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	ScaLBL_Gradient_Unpack(0.5,-1,0,1,dvcRecvDist_x,4*recvCount_x,recvCount_x,recvbuf_x,phi,grad,N);
 	//...................................................................................
 	//...Packing for X face(1,7,9,11,13)................................
@@ -1150,12 +1429,120 @@ void ScaLBL_Communicator::RecvGrad(double *phi, double *grad){
 	//...................................................................................
 
 }
+/**
+ * 
+ */
+void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){
+
+	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
+	if (Lock==true){
+		ERROR("ScaLBL Error (SendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+	}
+	else{
+		Lock=true;
+	}
+	// assign tag of 19 to D3Q19 communication
+	sendtag = recvtag = 7;
+	ScaLBL_DeviceBarrier();
+	// Pack the distributions
+	//...Packing for x face(2,8,10,12,14)................................
+	ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_x, sendCount_x,MPI_DOUBLE,rank_x,sendtag,MPI_COMM_SCALBL,&req1[0]);
+	MPI_Irecv(recvbuf_X, recvCount_X,MPI_DOUBLE,rank_X,recvtag,MPI_COMM_SCALBL,&req2[0]);
+	
+	//...Packing for X face(1,7,9,11,13)................................
+	ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
+	
+	MPI_Isend(sendbuf_X, sendCount_X,MPI_DOUBLE,rank_X,sendtag,MPI_COMM_SCALBL,&req1[1]);
+	MPI_Irecv(recvbuf_x, recvCount_x,MPI_DOUBLE,rank_x,recvtag,MPI_COMM_SCALBL,&req2[1]);
+
+	//...Packing for y face(4,8,9,16,18).................................
+	ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_y, sendCount_y,MPI_DOUBLE,rank_y,sendtag,MPI_COMM_SCALBL,&req1[2]);
+	MPI_Irecv(recvbuf_Y, recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,MPI_COMM_SCALBL,&req2[2]);
+	
+	//...Packing for Y face(3,7,10,15,17).................................
+	ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
+
+	MPI_Isend(sendbuf_Y, sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,MPI_COMM_SCALBL,&req1[3]);
+	MPI_Irecv(recvbuf_y, recvCount_y,MPI_DOUBLE,rank_y,recvtag,MPI_COMM_SCALBL,&req2[3]);
+	
+	//...Packing for z face(6,12,13,16,17)................................
+	ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
+	
+	MPI_Isend(sendbuf_z, sendCount_z,MPI_DOUBLE,rank_z,sendtag,MPI_COMM_SCALBL,&req1[4]);
+	MPI_Irecv(recvbuf_Z, recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,MPI_COMM_SCALBL,&req2[4]);
+	
+	//...Packing for Z face(5,11,14,15,18)................................
+	ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
+
+	//...................................................................................
+	// Send all the distributions
+	MPI_Isend(sendbuf_Z, sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,MPI_COMM_SCALBL,&req1[5]);
+	MPI_Irecv(recvbuf_z, recvCount_z,MPI_DOUBLE,rank_z,recvtag,MPI_COMM_SCALBL,&req2[5]);
+
+}
+
+
+void ScaLBL_Communicator::RecvD3Q7AA(double *Aq, int Component){
+
+	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
+	//...................................................................................
+	// Wait for completion of D3Q19 communication
+	MPI_Waitall(6,req1,stat1);
+	MPI_Waitall(6,req2,stat2);
+	ScaLBL_DeviceBarrier();
+
+	//...................................................................................
+	// NOTE: AA Routine writes to opposite
+	// Unpack the distributions on the device
+	//...................................................................................
+	//...Unpacking for x face(2,8,10,12,14)................................
+	ScaLBL_D3Q7_Unpack(2,dvcRecvDist_x,0,recvCount_x,recvbuf_x,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for X face(1,7,9,11,13)................................
+	ScaLBL_D3Q7_Unpack(1,dvcRecvDist_X,0,recvCount_X,recvbuf_X,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for y face(4,8,9,16,18).................................
+	ScaLBL_D3Q7_Unpack(4,dvcRecvDist_y,0,recvCount_y,recvbuf_y,&Aq[Component*7*N],N);
+	//...................................................................................
+	//...Packing for Y face(3,7,10,15,17).................................
+	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,&Aq[Component*7*N],N);
+	//...................................................................................
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N);
+		}
+	}
+	else {
+		//...Packing for z face(6,12,13,16,17)................................
+		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,&Aq[Component*7*N],N);
+		//...Packing for Z face(5,11,14,15,18)................................
+		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,&Aq[Component*7*N],N);
+	}
+
+	//...................................................................................
+	Lock=false; // unlock the communicator after communications complete
+	//...................................................................................
+
+}
+/* 
+ */
+
 
 void ScaLBL_Communicator::BiSendD3Q7AA(double *Aq, double *Bq){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
 	if (Lock==true){
-		ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+		ERROR("ScaLBL Error (BiSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
 	}
 	else{
 		Lock=true;
@@ -1240,18 +1627,18 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,0,recvCount_Y,recvbuf_Y,Aq,N);
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,recvCount_Y,recvCount_Y,recvbuf_Y,Bq,N);
 	//...................................................................................
-	
-	if (BoundaryCondition > 0 && kproc == 0){
-		// don't unpack little z
-		//...Packing for Z face(5,11,14,15,18)................................
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
-	}
-	else if (BoundaryCondition > 0 && kproc == nprocz-1){
-		// don't unpack big z
-		//...Packing for z face(6,12,13,16,17)................................
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
+		}
 	}
 	else {
 		//...Packing for z face(6,12,13,16,17)................................
@@ -1261,7 +1648,17 @@ void ScaLBL_Communicator::BiRecvD3Q7AA(double *Aq, double *Bq){
 		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
 		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
 	}
-	
+/*	if (BoundaryCondition == 5){
+		if (kproc == 0){
+			ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Aq, sendCount_z, N);
+			ScaLBL_D3Q7_Reflection_BC_z(dvcSendList_z, Bq, sendCount_z, N);
+		}
+		if (kproc == nprocz-1){
+			ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Aq, sendCount_Z, N);
+			ScaLBL_D3Q7_Reflection_BC_Z(dvcSendList_Z, Bq, sendCount_Z, N);
+		}
+	}
+	*/
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
@@ -1272,7 +1669,7 @@ void ScaLBL_Communicator::TriSendD3Q7AA(double *Aq, double *Bq, double *Cq){
 
 	// NOTE: the center distribution f0 must NOT be at the start of feven, provide offset to start of f2
 	if (Lock==true){
-		ERROR("ScaLBL Error (SendD3Q19): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
+		ERROR("ScaLBL Error (TriSendD3Q7): ScaLBL_Communicator is locked -- did you forget to match Send/Recv calls?");
 	}
 	else{
 		Lock=true;
@@ -1358,19 +1755,19 @@ void ScaLBL_Communicator::TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq){
 	ScaLBL_D3Q7_Unpack(3,dvcRecvDist_Y,2*recvCount_Y,recvCount_Y,recvbuf_Y,Cq,N);
 	//...................................................................................
 	
-	if (BoundaryCondition > 0 && kproc == 0){
-		// don't unpack little z
-		//...Packing for Z face(5,11,14,15,18)................................
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
-		ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N);
-	}
-	else if (BoundaryCondition > 0 && kproc == nprocz-1){
-		// don't unpack big z
-		//...Packing for z face(6,12,13,16,17)................................
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
-		ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N);
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,0,recvCount_z,recvbuf_z,Aq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,recvCount_z,recvCount_z,recvbuf_z,Bq,N);
+			ScaLBL_D3Q7_Unpack(6,dvcRecvDist_z,2*recvCount_z,recvCount_z,recvbuf_z,Cq,N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,0,recvCount_Z,recvbuf_Z,Aq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,recvCount_Z,recvCount_Z,recvbuf_Z,Bq,N);
+			ScaLBL_D3Q7_Unpack(5,dvcRecvDist_Z,2*recvCount_Z,recvCount_Z,recvbuf_Z,Cq,N);
+		}
 	}
 	else {
 		//...Packing for z face(6,12,13,16,17)................................
@@ -1472,30 +1869,57 @@ void ScaLBL_Communicator::RecvHalo(double *data){
 	//...................................................................................
 	ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, N);
 	ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
-	ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+	
+	if (BoundaryCondition > 0){
+		if (kproc != 0){
+			//...Packing for z face(6,12,13,16,17)................................
+			ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
+		}
+		if (kproc != nprocz-1){
+			//...Packing for Z face(5,11,14,15,18)................................
+			ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
+			ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+		}
+	}
+	else {
+		ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, N);
+		ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, N);
+	}
 	//...................................................................................
 	Lock=false; // unlock the communicator after communications complete
 	//...................................................................................
+	if (BoundaryCondition == 5 && kproc == 0){
+		ScaLBL_CopySlice_z(data,Nx,Ny,Nz,1,0);
+	}
+	if (BoundaryCondition == 5 && kproc == nprocz-1){
+		ScaLBL_CopySlice_z(data,Nx,Ny,Nz,Nz-2,Nz-1);
+	}	
 }
 
 void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, DoubleArray &regdata){
 	// Gets data from the device and stores in regular layout
-	int i,j,k,n,idx;
+	int i,j,k,idx;
 	int Nx = map.size(0);
 	int Ny = map.size(1);
 	int Nz = map.size(2);
@@ -1524,23 +1948,30 @@ void ScaLBL_Communicator::RegularLayout(IntArray map, const double *data, Double
 	delete [] TmpDat;
 }
 
-
 void ScaLBL_Communicator::Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB){
-	double Value=(vA-vB)/(vA+vB);
 	if (kproc == 0) {
-		// Set the phase indicator field and density on the z inlet
-		ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N);
+		if (BoundaryCondition == 5){
+			//ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,1,0);
+		}
+		else {
+			// Set the phase indicator field and density on the z inlet
+			ScaLBL_Color_BC_z(dvcSendList_z, Map, Phi, Den, vA, vB, sendCount_z, N);
+		}
 		//ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,0);
 	}
 }
 
 void ScaLBL_Communicator::Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB){
-	double Value=(vA-vB)/(vA+vB);
 	if (kproc == nprocz-1){
+		if (BoundaryCondition == 5){
+			//ScaLBL_CopySlice_z(Phi,Nx,Ny,Nz,Nz-2,Nz-1);
+		}
+		else {
 		// Set the phase indicator field and density on the Z outlet
-		ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N);
-		//ScaLBL_SetSlice_z(Phi,Value,Nx,Ny,Nz,Nz-1);
+			ScaLBL_Color_BC_Z(dvcSendList_Z, Map, Phi, Den, vA, vB, sendCount_Z, N);
+		}
 	}
+
 }
 
 void ScaLBL_Communicator::D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time){
@@ -1608,6 +2039,17 @@ double ScaLBL_Communicator::D3Q19_Flux_BC_z(int *neighborList, double *fq, doubl
 	return din;
 }
 
+void ScaLBL_Communicator::D3Q19_Reflection_BC_z(double *fq){
+	if (kproc == 0)
+		ScaLBL_D3Q19_Reflection_BC_z(dvcSendList_z, fq, sendCount_z, N);
+	
+}
+
+void ScaLBL_Communicator::D3Q19_Reflection_BC_Z(double *fq){
+	if (kproc == nprocz-1)
+		ScaLBL_D3Q19_Reflection_BC_Z(dvcSendList_Z, fq, sendCount_Z, N);
+}
+
 void ScaLBL_Communicator::PrintD3Q19(){
 	printf("Printing D3Q19 communication buffer contents \n");
 
@@ -1628,3 +2070,80 @@ void ScaLBL_Communicator::PrintD3Q19(){
 	delete [] TempBuffer;
 }
 
+void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(dvcSendList_z, fq, Vin, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(neighborList, dvcSendList_z, fq, Vin, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(dvcSendList_Z, fq, Vout, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(neighborList, dvcSendList_Z, fq, Vout, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin){
+	if (kproc == 0) {
+		ScaLBL_Poisson_D3Q7_BC_z(dvcSendList_z, Map, Psi, Vin, sendCount_z);
+	}
+}
+
+void ScaLBL_Communicator::Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout){
+	if (kproc == nprocz-1){
+		ScaLBL_Poisson_D3Q7_BC_Z(dvcSendList_Z, Map, Psi, Vout, sendCount_Z);
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(dvcSendList_z, fq, Cin, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(neighborList, dvcSendList_z, fq, Cin, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(dvcSendList_Z, fq, Cout, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(neighborList, dvcSendList_Z, fq, Cout, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+	}
+}

common/ScaLBL.h

@@ -70,6 +70,234 @@ extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int
 
 extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
 
+// GREYSCALE MODEL (Single-component)
+
+extern "C" void ScaLBL_D3Q19_GreyIMRT_Init(double *Dist, int Np, double Den);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+// ION TRANSPORT MODEL
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
+
+// LBM Poisson solver
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+//maybe deprecated
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np);
+
+// LBM Stokes Model (adapted from MRT model)
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
+                                              double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz, 
+                                             double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
+
+extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+        double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
+
+// LBM Poisson solver
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
+        int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
+
+//maybe deprecated
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np);
+
+// LBM Stokes Model (adapted from MRT model)
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
+                double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
+
+// GREYSCALE FREE-ENERGY MODEL (Two-component)
+
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den,
+//                double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,
+//                double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
+//                double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
+//                double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
+//                double Gx, double Gy, double Gz,
+//                double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
+//
+//extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity,
+//                double *Pressure, double rhoA,double rhoB, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap,
+//      		     double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np);
+
+// GREYSCALE SHAN-CHEN MODEL (Two-component)
+
+//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB, 
+//                double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure, 
+//                double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,                                                 
+//                int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
+//
+//extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
+//
+//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
+
+// GREYSCALE COLOR MODEL (Two-component)
+//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np);
+
+//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+//        double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+//        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+//
+//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+//		double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel, 
+//        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+        double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure,
+        double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure, 
+        double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np);
+
 // MRT MODEL
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
 		double Fy, double Fz);
@@ -78,7 +306,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int st
 		double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz);
 
 // COLOR MODEL
-
 extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
@@ -116,11 +343,6 @@ extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double
 
 // BOUNDARY CONDITION ROUTINES
 
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din,
-//		int Nx, int Ny, int Nz);
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout,
-//		int Nx, int Ny, int Nz, int outlet);
-
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np);
@@ -139,8 +361,50 @@ extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den,
 
 extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
 
+extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np);
+
 extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice);
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination);
+
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np);
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count);
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+
 class ScaLBL_Communicator{
 public:
 	//......................................................................................
@@ -149,8 +413,10 @@ public:
 	//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
 	~ScaLBL_Communicator();
 	//......................................................................................
+	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator
 	unsigned long int CommunicationCount,SendCount,RecvCount;
 	int Nx,Ny,Nz,N;
+	int n_bb_d3q7, n_bb_d3q19; 
 	int BoundaryCondition;
 	
 	int next;
@@ -173,19 +439,11 @@ public:
 	int FirstInterior();
 	int LastInterior();
 	
-	int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np);
-//	void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
-//	void SendD3Q19(double *f_even, double *f_odd);
-//	void RecvD3Q19(double *f_even, double *f_odd);
-//	void SendD3Q19AA(double *f_even, double *f_odd);
-//	void RecvD3Q19AA(double *f_even, double *f_odd);
+	int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np);
 	void SendD3Q19AA(double *dist);
 	void RecvD3Q19AA(double *dist);
-//	void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
-//	void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
+	void SendD3Q7AA(double *fq, int Component);
+	void RecvD3Q7AA(double *fq, int Component);
 	void BiSendD3Q7AA(double *Aq, double *Bq);
 	void BiRecvD3Q7AA(double *Aq, double *Bq);
 	void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
@@ -194,17 +452,34 @@ public:
 	void RecvHalo(double *data);
 	void RecvGrad(double *Phi, double *Gradient);
 	void RegularLayout(IntArray map, const double *data, DoubleArray &regdata);
+	void SetupBounceBackList(IntArray &Map, signed char *id, int Np);
+    void SolidDirichletD3Q7(double *fq, double *BoundaryValue);
+    void SolidNeumannD3Q7(double *fq, double *BoundaryValue);
 
 	// Routines to set boundary conditions
 	void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
 	void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB);
 	void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time);
 	void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
+	void D3Q19_Reflection_BC_z(double *fq);
+	void D3Q19_Reflection_BC_Z(double *fq);
 	double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
-
-//	void TestSendD3Q19(double *f_even, double *f_odd);
-//	void TestRecvD3Q19(double *f_even, double *f_odd);
-
+	void D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time);
+	void D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time);
+	void Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin);
+	void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout);
+	void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time);
+	void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time);
+	void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
+	void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
+	void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
+	void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
+        void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
+        void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
+	void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
+	void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
+        void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
+        void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
 	// Debugging and unit testing functions
 	void PrintD3Q19();
 
@@ -222,7 +497,6 @@ private:
 	// Give the object it's own MPI communicator
 	RankInfoStruct rank_info;
 	MPI_Group Group;	// Group of processors associated with this domain
-	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator for this domain
 	MPI_Request req1[18],req2[18];
 	MPI_Status stat1[18],stat2[18];
 	//......................................................................................
@@ -259,6 +533,9 @@ private:
 	int *dvcRecvDist_xy, *dvcRecvDist_yz, *dvcRecvDist_xz, *dvcRecvDist_Xy, *dvcRecvDist_Yz, *dvcRecvDist_xZ;
 	int *dvcRecvDist_xY, *dvcRecvDist_yZ, *dvcRecvDist_Xz, *dvcRecvDist_XY, *dvcRecvDist_YZ, *dvcRecvDist_XZ;
 	//......................................................................................
+	int *bb_dist;
+	int *bb_interactions;
+	//......................................................................................
 
 };
 

common/Utilities.cpp

@@ -14,307 +14,132 @@
   along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 */
 #include "common/Utilities.h"
-#include "common/StackTrace.h"
+#include "StackTrace/StackTrace.h"
+#include "StackTrace/ErrorHandlers.h"
 
-#include <iostream>
-#include <sstream>
-#include <stdexcept>
-#include <fstream>
-#include <string.h>
-#include <signal.h>
-#include <math.h>
-#include <algorithm>
+#ifdef USE_TIMER
+#include "MemoryApp.h"
+#include "ProfilerApp.h"
+#endif
 
 #ifdef USE_MPI
-    #include "mpi.h"
+#include "mpi.h"
 #endif
 
-// Detect the OS and include system dependent headers
-#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) || defined(_MSC_VER)
-    // Note: windows has not been testeds
+#include <algorithm>
+#include <math.h>
+#include <mutex>
+
+
+// OS specific includes / definitions
+// clang-format off
+#if defined( WIN32 ) || defined( _WIN32 ) || defined( WIN64 ) || defined( _WIN64 )
     #define USE_WINDOWS
-    #include <windows.h>
-    #include <process.h>
-    #include <stdio.h>   
-    #include <tchar.h>
-    #include <psapi.h>
-    #include <DbgHelp.h>
-    #define mkdir(path, mode) _mkdir(path)
-    //#pragma comment(lib, psapi.lib) //added
-    //#pragma comment(linker, /DEFAULTLIB:psapi.lib)
-#elif defined(__APPLE__)
+#elif defined( __APPLE__ )
     #define USE_MAC
-    #include <sys/time.h>
-    #include <signal.h>
-    #include <execinfo.h>
-    #include <dlfcn.h>
-    #include <mach/mach.h>
-    #include <unistd.h>
-#elif defined(__linux) || defined(__unix) || defined(__posix)
+#elif defined( __linux ) || defined( __linux__ ) || defined( __unix ) || defined( __posix )
     #define USE_LINUX
-    #include <sys/time.h>
-    #include <execinfo.h>
-    #include <dlfcn.h>
-    #include <malloc.h>
-    #include <unistd.h>
 #else
     #error Unknown OS
 #endif
+// clang-format on
 
 
-#ifdef __GNUC__
-    #define USE_ABI
-    #include <cxxabi.h>
+// Mutex for Utility functions
+static std::mutex Utilities_mutex;
+
+
+/****************************************************************************
+ *  Function to perform the default startup/shutdown sequences               *
+ ****************************************************************************/
+void Utilities::startup( int argc, char **argv )
+{
+    NULL_USE( argc );
+    NULL_USE( argv );
+    // Disable OpenMP
+    Utilities::setenv( "OMP_NUM_THREADS", "1" );
+    Utilities::setenv( "MKL_NUM_THREADS", "1" );
+    // Start MPI
+#ifdef USE_MPI
+    int provided;
+    MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+    if ( provided < MPI_THREAD_MULTIPLE ) {
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+        if ( rank == 0 )
+            std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
+    }
+    StackTrace::globalCallStackInitialize( MPI_COMM_WORLD );
 #endif
-
-
-/****************************************************************************
-*  Function to terminate the program                                        *
-****************************************************************************/
-static bool abort_printMemory = true;
-static bool abort_printStack = true;
-static bool abort_throwException = false;
-static int force_exit = 0;
-void Utilities::setAbortBehavior( bool printMemory, bool printStack, bool throwException )
-{
-    abort_printMemory = printMemory;
-    abort_printStack = printStack;
-    abort_throwException = throwException;
+    // Set the error handlers
+    Utilities::setAbortBehavior( true, 3 );
+    Utilities::setErrorHandlers();
 }
-void Utilities::abort(const std::string &message, const std::string &filename, const int line) 
+void Utilities::shutdown()
 {
-    std::stringstream msg;
-    msg << "Program abort called in file `" << filename << "' at line " << line << std::endl;
-    // Add the memory usage and call stack to the error message
-    if ( abort_printMemory ) {
-        size_t N_bytes = Utilities::getMemoryUsage();
-        msg << "Bytes used = " << N_bytes << std::endl;
-    }
-    if ( abort_printStack ) {
-        std::vector<StackTrace::stack_info> stack = StackTrace::getCallStack();
-        msg << std::endl;
-        msg << "Stack Trace:\n";
-        for (size_t i=0; i<stack.size(); i++)
-            msg << "   " << stack[i].print() << std::endl;
-    }
-    msg << std::endl << message << std::endl;
-    // Print the message and abort
-    if ( force_exit>1 ) {
-        exit(-1);
-    } else if ( !abort_throwException ) {
-        // Use MPI_abort (will terminate all processes)
-        force_exit = 2;
-        std::cerr << msg.str();
-        #if defined(USE_MPI) || defined(HAVE_MPI)
-            int initialized=0, finalized=0;
-            MPI_Initialized(&initialized);
-            MPI_Finalized(&finalized);
-            if ( initialized!=0 && finalized==0 )
-                MPI_Abort(MPI_COMM_WORLD,-1);
-        #endif
-        exit(-1);
-    } else if ( force_exit>0 ) {
-        exit(-1);
-    } else {
-        // Throw and standard exception (allows the use of try, catch)
-        throw std::logic_error(msg.str());
-    }
-}
-
-
-/****************************************************************************
-*  Function to handle MPI errors                                            *
-****************************************************************************/
-/*#if defined(USE_MPI) || defined(HAVE_MPI)
-MPI_Errhandler mpierr;
-void MPI_error_handler_fun( MPI_Comm *comm, int *err, ... )
-{
-    if ( *err==MPI_ERR_COMM && *comm==MPI_COMM_WORLD ) {
-        // Special error handling for an invalid MPI_COMM_WORLD
-        std::cerr << "Error invalid MPI_COMM_WORLD";
-        exit(-1);
-    }
-    int msg_len=0;
-    char message[1000];
-    MPI_Error_string( *err, message, &msg_len );
-    if ( msg_len <= 0 )
-         abort("Unkown error in MPI");
-    abort( "Error calling MPI routine:\n" + std::string(message) );
-}
-#endif*/
-
-
-/****************************************************************************
-*  Function to handle unhandled exceptions                                  *
-****************************************************************************/
-bool tried_MPI_Abort=false;
-void term_func_abort(int err) 
-{
-    printf("Exiting due to abort (%i)\n",err);
-    std::vector<StackTrace::stack_info> stack = StackTrace::getCallStack();
-    std::string message = "Stack Trace:\n";
-    for (size_t i=0; i<stack.size(); i++)
-        message += "   " + stack[i].print() += "\n";
-    message += "\nExiting\n";
-    // Print the message and abort
-    std::cerr << message;
-    #ifdef USE_MPI
-        if ( !abort_throwException && !tried_MPI_Abort ) {
-            tried_MPI_Abort = true;
-            MPI_Abort(MPI_COMM_WORLD,-1);
-        }
-    #endif
-    exit(-1);
-}
-#if defined(USE_LINUX) || defined(USE_MAC)
-    static int tried_throw = 0;
+    // Clear the error handlers
+    Utilities::clearErrorHandlers();
+    StackTrace::clearSignals();
+    StackTrace::clearSymbols();
+    int rank = 0;
+#ifdef USE_MPI
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+    StackTrace::globalCallStackFinalize();
+    MPI_Barrier( MPI_COMM_WORLD );
+    MPI_Finalize();
 #endif
-void term_func() 
-{
-    // Try to re-throw the last error to get the last message
-    std::string last_message;
-    #if defined(USE_LINUX) || defined(USE_MAC)
-        try {
-            if ( tried_throw==0 ) { 
-                tried_throw = 1;
-                throw;
-            }
-            // No active exception
-        } catch (const std::exception &err) {
-            // Caught a std::runtime_error
-            last_message = err.what();
-        } catch (...) {
-            // Caught an unknown exception
-            last_message = "unknown exception occurred.";
-        }
-    #endif
-    std::stringstream msg;
-    msg << "Unhandled exception:" << std::endl;
-    msg << "   " << last_message << std::endl;
-    Utilities::abort( msg.str(), __FILE__, __LINE__ );
-}
-
-
-/****************************************************************************
-*  Functions to set the error handler                                       *
-****************************************************************************/
-static void setTerminateErrorHandler()
-{
-    std::set_terminate( term_func );
-    signal(SIGABRT,&term_func_abort);
-    signal(SIGFPE,&term_func_abort);
-    signal(SIGILL,&term_func_abort);
-    signal(SIGINT,&term_func_abort);
-    signal(SIGSEGV,&term_func_abort);
-    signal(SIGTERM,&term_func_abort);
-}
-void Utilities::setErrorHandlers()
-{
-    //d_use_MPI_Abort = use_MPI_Abort;
-    //setMPIErrorHandler( SAMRAI::tbox::SAMRAI_MPI::getSAMRAIWorld() );
-    setTerminateErrorHandler();
-}
-/*void Utilities::setMPIErrorHandler( const SAMRAI::tbox::SAMRAI_MPI& mpi )
-{
-    #if defined(USE_MPI) || defined(HAVE_MPI)
-        if ( mpierr.get()==NULL ) {
-            mpierr = boost::shared_ptr<MPI_Errhandler>( new MPI_Errhandler );
-            MPI_Comm_create_errhandler( MPI_error_handler_fun, mpierr.get() );
-        }
-        MPI_Comm_set_errhandler( mpi.getCommunicator(), *mpierr );
-        MPI_Comm_set_errhandler( MPI_COMM_WORLD, *mpierr );
-    #endif
-}
-void Utilities::clearMPIErrorHandler(  )
-{
-    #if defined(USE_MPI) || defined(HAVE_MPI)
-        if ( mpierr.get()!=NULL )
-            MPI_Errhandler_free( mpierr.get() );    // Delete the error handler
-        mpierr.reset();
-        MPI_Comm_set_errhandler( MPI_COMM_SELF, MPI_ERRORS_ARE_FATAL );
-        MPI_Comm_set_errhandler( MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
-    #endif
-}*/
-
-
-/****************************************************************************
-*  Function to get the memory usage                                         *
-*  Note: this function should be thread-safe                                *
-****************************************************************************/
-#if defined(USE_MAC)
-    // Get the page size on mac
-    static size_t page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
+#ifdef USE_TIMER
+    PROFILE_DISABLE();
+    auto memory = MemoryApp::getMemoryStats();
+    if ( rank == 0 && memory.N_new > memory.N_delete )
+        MemoryApp::print( std::cout );
 #endif
-static size_t N_bytes_initialization = Utilities::getMemoryUsage();
-size_t Utilities::getMemoryUsage()
-{
-    size_t N_bytes = 0;
-    #if defined(USE_LINUX)
-        struct mallinfo meminfo = mallinfo();
-        size_t size_hblkhd = static_cast<unsigned int>( meminfo.hblkhd );
-        size_t size_uordblks = static_cast<unsigned int>( meminfo.uordblks );
-        N_bytes = size_hblkhd + size_uordblks;
-    #elif defined(USE_MAC)
-        struct task_basic_info t_info;
-        mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
-        if (KERN_SUCCESS != task_info(mach_task_self(),
-                              TASK_BASIC_INFO, (task_info_t)&t_info, 
-                              &t_info_count)) {
-            return 0;
-        }
-        N_bytes = t_info.virtual_size;
-    #elif defined(USE_WINDOWS)
-        PROCESS_MEMORY_COUNTERS memCounter;
-        GetProcessMemoryInfo( GetCurrentProcess(), &memCounter, sizeof(memCounter) );
-        N_bytes = memCounter.WorkingSetSize;
-    #endif
-    return N_bytes;
 }
 
 
 /****************************************************************************
-*  Functions to get the time and timer resolution                           *
-****************************************************************************/
-#if defined(USE_WINDOWS)
-    double Utilities::time() 
-    { 
-        LARGE_INTEGER end, f;
-        QueryPerformanceFrequency(&f);
-        QueryPerformanceCounter(&end);       
-        double time = ((double)end.QuadPart)/((double)f.QuadPart);
-        return time;
-    }
-    double Utilities::tick() 
-    { 
-        LARGE_INTEGER f;
-        QueryPerformanceFrequency(&f);
-        double resolution = ((double)1.0)/((double)f.QuadPart);
-        return resolution;
-    }
-#elif defined(USE_LINUX) || defined(USE_MAC)
-    double Utilities::time() 
-    { 
-        timeval current_time;
-        gettimeofday(&current_time,NULL);
-        double time = ((double)current_time.tv_sec)+1e-6*((double)current_time.tv_usec);
-        return time;
-    }
-    double Utilities::tick() 
-    { 
-        timeval start, end;
-        gettimeofday(&start,NULL);
-        gettimeofday(&end,NULL);
-        while ( end.tv_sec==start.tv_sec &&  end.tv_usec==start.tv_usec )
-            gettimeofday(&end,NULL);
-        double resolution = ((double)(end.tv_sec-start.tv_sec))+1e-6*((double)(end.tv_usec-start.tv_usec));
-        return resolution;
-    }
+ *  Function to set an environemental variable                               *
+ ****************************************************************************/
+void Utilities::setenv( const std::string &name, const std::string &value )
+{
+    Utilities_mutex.lock();
+#if defined( USE_LINUX ) || defined( USE_MAC )
+    bool pass = false;
+    if ( !value.empty() )
+        pass = ::setenv( name.data(), value.data(), 1 ) == 0;
+    else
+        pass = ::unsetenv( name.data() ) == 0;
+#elif defined( USE_WINDOWS )
+    bool pass = SetEnvironmentVariable( name.data(), value.data() ) != 0;
 #else
-    #error Unknown OS
+#error Unknown OS
 #endif
+    Utilities_mutex.unlock();
+    if ( !pass ) {
+        char msg[1024];
+        if ( !value.empty() )
+            sprintf(
+                msg, "Error setting enviornmental variable: %s=%s\n", name.data(), value.data() );
+        else
+            sprintf( msg, "Error clearing enviornmental variable: %s\n", name.data() );
+        ERROR( msg );
+    }
+}
+std::string Utilities::getenv( const std::string &name )
+{
+    std::string var;
+    Utilities_mutex.lock();
+    auto tmp = std::getenv( name.data() );
+    if ( tmp )
+        var = std::string( tmp );
+    Utilities_mutex.unlock();
+    return var;
+}
 
 
-// Factor a number into it's prime factors
+/****************************************************************************
+ *  Factor a number into it's prime factors                                  *
+ ****************************************************************************/
 std::vector<int> Utilities::factor(size_t number)
 {
     if ( number<=3 ) 
@@ -364,9 +189,13 @@ std::vector<int> Utilities::factor(size_t number)
 }
 
 
-// Dummy function to prevent compiler from optimizing away variable
+/****************************************************************************
+ *  Dummy function to prevent compiler from optimizing away variable         *
+ ****************************************************************************/
 void Utilities::nullUse( void* data )
 {
     NULL_USE(data);
 }
 
+
+

common/Utilities.h

@@ -16,91 +16,73 @@
 #ifndef included_Utilities
 #define included_Utilities
 
-#include <chrono>
 #include <cstdarg>
-#include <iostream>
-#include <mutex>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/stat.h>
-#include <thread>
 #include <vector>
 
+#include "StackTrace/Utilities.h"
+
 
 namespace Utilities {
 
 
-/*!
- * Aborts the run after printing an error message with file and
- * linenumber information.
- */
-void abort( const std::string &message, const std::string &filename, const int line );
+// Functions inherited from StackTrace::Utilities
+using StackTrace::Utilities::abort;
+using StackTrace::Utilities::cause_segfault;
+using StackTrace::Utilities::clearErrorHandlers;
+using StackTrace::Utilities::exec;
+using StackTrace::Utilities::getMemoryUsage;
+using StackTrace::Utilities::getSystemMemory;
+using StackTrace::Utilities::setAbortBehavior;
+using StackTrace::Utilities::setErrorHandlers;
+using StackTrace::Utilities::tick;
+using StackTrace::Utilities::time;
+using StackTrace::Utilities::sleep_ms;
+using StackTrace::Utilities::sleep_s;
 
 
 /*!
- * Set the behavior of abort
- * @param printMemory       Print the current memory usage (default is true)
- * @param printStack        Print the current call stack (default is true)
- * @param throwException    Throw an exception instead of MPI_Abort (default is false)
+ * \brief Start MPI, error handlers
+ * \details This routine will peform the default startup sequence
+ * \param argc              argc from main
+ * \param argv              argv from main
  */
-void setAbortBehavior( bool printMemory, bool printStack, bool throwException );
+void startup( int argc, char **argv );
 
-//! Function to set the error handlers
-void setErrorHandlers();
+/*!
+ * \brief Stop MPI, error handlers
+ * \details This routine will peform the default shutdown sequence to match startup
+ */
+void shutdown();
 
 
 /*!
- * Function to get the memory availible.
- * This function will return the total memory availible
- * Note: depending on the implimentation, this number may be rounded to
- * to a multiple of the page size.
- * If this function fails, it will return 0.
+ * Get an environmental variable
+ * @param name              The name of the environmental variable
+ * @return                  The value of the enviornmental variable
  */
-size_t getSystemMemory();
+std::string getenv( const std::string &name );
 
 
 /*!
- * Function to get the memory usage.
- * This function will return the total memory used by the application.
- * Note: depending on the implimentation, this number may be rounded to
- * to a multiple of the page size.
- * If this function fails, it will return 0.
+ * Set an environmental variable
+ * @param name              The name of the environmental variable
+ * @param value             The value to set
  */
-size_t getMemoryUsage();
-
-
-//! Function to get an arbitrary point in time
-double time();
-
-
-//! Function to get the resolution of time
-double tick();
+void setenv( const std::string &name, const std::string &value );
 
 
 //! std::string version of sprintf
 inline std::string stringf( const char *format, ... );
 
 
-/*!
- * Sleep for X ms
- * @param N         Time to sleep (ms)
- */
-inline void sleep_ms( int N ) { std::this_thread::sleep_for( std::chrono::milliseconds( N ) ); }
-
-
-/*!
- * Sleep for X s
- * @param N         Time to sleep (s)
- */
-inline void sleep_s( int N ) { std::this_thread::sleep_for( std::chrono::seconds( N ) ); }
-
-
 //! Factor a number into it's prime factors
 std::vector<int> factor(size_t number);
 
-//! Print AMP Banner
+
+//! Null use function
 void nullUse( void* );
 
+
 } // namespace Utilities
 
 

cpu/Color.cpp

@@ -1379,11 +1379,12 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do
 
 		//...........Normalize the Color Gradient.................................
 		C = sqrt(nx*nx+ny*ny+nz*nz);
-		if (C==0.0) C=1.0;
-		nx = nx/C;
-		ny = ny/C;
-		nz = nz/C;		
-
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
+		
 		// q=0
 		fq = dist[n];
 		rho = fq;
@@ -1658,7 +1659,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, do
 		//..............carry out relaxation process..............................
 		//..........Toelke, Fruediger et. al. 2006................................
 		if (C == 0.0)	nx = ny = nz = 0.0;
-		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -alpha*C - m1);
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -19*alpha*C - m1);
 		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2);
 		m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4);
 		m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6);
@@ -1883,7 +1884,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 	const double mrt_V12=0.04166666666666666;
 
 	for (int n=start; n<finish; n++){
-		
+	
 		// read the component number densities
 		nA = Den[n];
 		nB = Den[Np + n];
@@ -1964,10 +1965,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 
 		//...........Normalize the Color Gradient.................................
 		C = sqrt(nx*nx+ny*ny+nz*nz);
-		if (C==0.0) C=1.0;
-		nx = nx/C;
-		ny = ny/C;
-		nz = nz/C;		
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
 
 		// q=0
 		fq = dist[n];
@@ -2294,7 +2296,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 		//..............carry out relaxation process..............................
 		//..........Toelke, Fruediger et. al. 2006................................
 		if (C == 0.0)	nx = ny = nz = 0.0;
-		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -alpha*C - m1);
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -19*alpha*C - m1);
 		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2);
 		m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4);
 		m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6);
@@ -2823,3 +2825,11 @@ extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, doubl
 	}
 }
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	int n; double value;
+	for (n=0; n<Nx*Ny; n++){
+		value = Phi[Source*Nx*Ny+n];
+		Phi[Dest*Nx*Ny+n] = value;
+	}
+}
+

cpu/D3Q7.cpp

@@ -87,6 +87,23 @@ extern  "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int
 	}
 }
 
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+  int n;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		double f5 = 0.222222222222222222222222 - dist[6*Np+n];
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+  int n;
+  for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		double f6 = 0.222222222222222222222222 - dist[5*Np+n];
+		dist[5*Np+n] = f6;
+	}
+}
 extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
 {
 	int n,N;

cpu/D3Q7BC.cpp

@@ -0,0 +1,347 @@
+// CPU Functions for D3Q7 Lattice Boltzmann Methods
+// Boundary Conditions 
+
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	for (idx=0; idx<N; idx++){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
+	}
+}
+
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	for (idx=0; idx<N; idx++){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = value_q + value_b;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f6 = dist[5*Np+n];
+		//...................................................
+		double f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f5 = dist[6*Np+n];
+		//...................................................
+		double f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
+    int nread,nr5;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		double f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		double f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
+    int nread,nr6;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		double f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		double f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
+{
+	int idx,n,nm;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vin;
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
+{
+	int idx,n,nm;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vout;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f6 = dist[5*Np+n];
+		//...................................................
+		double f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+		double f1 = dist[2*Np+n];
+		double f2 = dist[1*Np+n];
+		double f3 = dist[4*Np+n];
+		double f4 = dist[3*Np+n];
+		double f5 = dist[6*Np+n];
+		//...................................................
+		double f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
+    int nread,nr5;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		double f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		double f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
+    int nread,nr6;
+	for (int idx=0; idx<count; idx++){
+		int n = list[idx];
+		double f0 = dist[n];
+
+		nread = d_neighborList[n];
+		double f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		double f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		double f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		double f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		double f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		double f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    int nread,nr5;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+    //NOTE: FluxIn is the inward flux
+    double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    int n;
+    int nread,nr6;
+    double uz;
+
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}

cpu/Ion.cpp

@@ -0,0 +1,254 @@
+#include <stdio.h>
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+    int n,nread;
+    double fq,Ci;
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+        Ci = fq;
+
+		// q=1
+		nread = neighborList[n]; 
+		fq = dist[nread]; 
+		Ci += fq;
+        
+        // q=2
+		nread = neighborList[n+Np]; 
+		fq = dist[nread];  
+		Ci += fq;
+		
+        // q=3
+		nread = neighborList[n+2*Np]; 
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=4
+		nread = neighborList[n+3*Np]; 
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		Ci += fq;
+		
+        // q=6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		Ci += fq;
+
+        Den[n]=Ci;
+    }
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+    int n;
+    double fq,Ci;
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+		Ci = fq;
+		
+		// q=1
+		fq = dist[2*Np+n];
+		Ci += fq;
+
+		// q=2
+		fq = dist[1*Np+n];
+		Ci += fq;
+
+		// q=3
+		fq = dist[4*Np+n];
+		Ci += fq;
+
+		// q=4
+		fq = dist[3*Np+n];
+		Ci += fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		Ci += fq;
+
+		// q=6
+		fq = dist[5*Np+n];
+		Ci += fq;
+
+        Den[n]=Ci;
+    }
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+
+	for (n=start; n<finish; n++){
+		
+        //Load data
+        Ci=Den[n];
+        Ex=ElectricField[n+0*Np];
+        Ey=ElectricField[n+1*Np];
+        Ez=ElectricField[n+2*Np];
+        ux=Velocity[n+0*Np];
+        uy=Velocity[n+1*Np];
+        uz=Velocity[n+2*Np];
+        uEPx=zi*Di/Vt*Ex;
+        uEPy=zi*Di/Vt*Ey;
+        uEPz=zi*Di/Vt*Ez;
+
+		// q=0
+		f0 = dist[n];
+		// q=1
+		nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		f1 = dist[nr1]; // reading the f1 data into register fq
+		// q=2
+		nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		f2 = dist[nr2];  // reading the f2 data into register fq
+		// q=3
+		nr3 = neighborList[n+2*Np]; // neighbor 4
+		f3 = dist[nr3];
+		// q=4
+		nr4 = neighborList[n+3*Np]; // neighbor 3
+		f4 = dist[nr4];
+		// q=5
+		nr5 = neighborList[n+4*Np];
+		f5 = dist[nr5];
+		// q=6
+		nr6 = neighborList[n+5*Np];
+		f6 = dist[nr6];
+		
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+
+		// q = 1
+		dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+
+		// q=2
+		dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+
+		// q = 3
+		dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+
+		// q = 4
+		dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+
+		// q = 5
+		dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+
+		// q = 6
+		dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+
+	for (n=start; n<finish; n++){
+		
+        //Load data
+        Ci=Den[n];
+        Ex=ElectricField[n+0*Np];
+        Ey=ElectricField[n+1*Np];
+        Ez=ElectricField[n+2*Np];
+        ux=Velocity[n+0*Np];
+        uy=Velocity[n+1*Np];
+        uz=Velocity[n+2*Np];
+        uEPx=zi*Di/Vt*Ex;
+        uEPy=zi*Di/Vt*Ey;
+        uEPz=zi*Di/Vt*Ez;
+
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		f6 = dist[5*Np+n];
+		
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+
+		// q = 1
+		dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+
+		// q=2
+		dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+
+		// q = 3
+		dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+
+		// q = 4
+		dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+
+		// q = 5
+		dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+
+		// q = 6
+		dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+
+	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np)
+{
+	int n;
+	for (n=0; n<Np; n++){
+		dist[0*Np+n] = 0.25*DenInit;
+		dist[1*Np+n] = 0.125*DenInit;		
+		dist[2*Np+n] = 0.125*DenInit;	
+		dist[3*Np+n] = 0.125*DenInit;	
+		dist[4*Np+n] = 0.125*DenInit;	
+		dist[5*Np+n] = 0.125*DenInit;	
+		dist[6*Np+n] = 0.125*DenInit;	
+        Den[n] = DenInit;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np)
+{
+	int n;
+    double DenInit;
+	for (n=0; n<Np; n++){
+        DenInit = Den[n];
+		dist[0*Np+n] = 0.25*DenInit;
+		dist[1*Np+n] = 0.125*DenInit;		
+		dist[2*Np+n] = 0.125*DenInit;	
+		dist[3*Np+n] = 0.125*DenInit;	
+		dist[4*Np+n] = 0.125*DenInit;	
+		dist[5*Np+n] = 0.125*DenInit;	
+		dist[6*Np+n] = 0.125*DenInit;	
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+    
+    int n;
+    double Ci;//ion concentration of species i
+    double CD;//charge density
+    double CD_tmp;
+    double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
+
+	for (n=start; n<finish; n++){
+        Ci = Den[n+ion_component*Np];
+        CD = ChargeDensity[n];
+        CD_tmp = F*IonValence*Ci;
+        ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
+    }
+}
+

cpu/Poisson.cpp

@@ -0,0 +1,372 @@
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+	int nread;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+        psi = fq;
+
+		// q=1
+		nread = neighborList[n]; 
+		fq = dist[nread];
+		psi += fq;
+		
+		// q=2
+		nread = neighborList[n+Np]; 
+		fq = dist[nread];  
+		psi += fq;
+
+		// q=3
+		nread = neighborList[n+2*Np]; 
+		fq = dist[nread];
+		psi += fq;
+
+		// q = 4
+		nread = neighborList[n+3*Np]; 
+		fq = dist[nread];
+		psi += fq;
+
+		// q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		psi += fq;
+
+		// q = 6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		psi += fq;
+        
+        idx=Map[n];
+        Psi[idx] = psi;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+	double fq;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+		// q=0
+		fq = dist[n];
+		psi = fq;
+		
+		// q=1
+		fq = dist[2*Np+n];
+		psi += fq;
+
+		// q=2 
+		fq = dist[1*Np+n];
+		psi += fq;
+
+		// q=3
+		fq = dist[4*Np+n];
+		psi += fq;
+
+		// q=4
+		fq = dist[3*Np+n];
+		psi += fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		psi += fq;
+
+		// q=6
+		fq = dist[5*Np+n];
+		psi += fq;
+
+        idx=Map[n];
+        Psi[idx] = psi;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+    double rlx=1.0/tau;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+        //Load data
+        rho_e = Den_charge[n];
+        rho_e = rho_e/epsilon_LB;
+        idx=Map[n];
+        psi = Psi[idx];
+		
+		// q=0
+		f0 = dist[n];
+		// q=1
+		nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		f1 = dist[nr1]; // reading the f1 data into register fq
+
+		nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		f2 = dist[nr2];  // reading the f2 data into register fq
+
+		// q=3
+		nr3 = neighborList[n+2*Np]; // neighbor 4
+		f3 = dist[nr3];
+
+		// q = 4
+		nr4 = neighborList[n+3*Np]; // neighbor 3
+		f4 = dist[nr4];
+
+		// q=5
+		nr5 = neighborList[n+4*Np];
+		f5 = dist[nr5];
+
+		// q = 6
+		nr6 = neighborList[n+5*Np];
+		f6 = dist[nr6];
+		
+		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ez = (f5-f6)*rlx*4.0;
+        ElectricField[n+0*Np] = Ex;
+        ElectricField[n+1*Np] = Ey;
+        ElectricField[n+2*Np] = Ez;
+
+		// q = 0
+		dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+		// q = 1
+		dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 2
+		dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 3
+		dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 4
+		dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 5
+		dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 6
+		dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
+	int n;
+	double psi;//electric potential
+    double Ex,Ey,Ez;//electric field
+    double rho_e;//local charge density
+	double f0,f1,f2,f3,f4,f5,f6;
+    double rlx=1.0/tau;
+    int idx;
+
+	for (n=start; n<finish; n++){
+
+        //Load data
+        rho_e = Den_charge[n];
+        rho_e = rho_e/epsilon_LB;
+        idx=Map[n];
+        psi = Psi[idx];
+
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		f6 = dist[5*Np+n];
+
+
+		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ez = (f5-f6)*rlx*4.0;
+        ElectricField[n+0*Np] = Ex;
+        ElectricField[n+1*Np] = Ey;
+        ElectricField[n+2*Np] = Ez;
+
+		// q = 0
+		dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
+
+		// q = 1
+		dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 2
+		dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 3
+		dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
+
+		// q = 4
+		dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 5
+		dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+
+		// q = 6
+		dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e); 
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np)
+{
+	int n;
+    int ijk;
+	for (n=start; n<finish; n++){
+        ijk = Map[n];
+		dist[0*Np+n] = 0.25*Psi[ijk];
+		dist[1*Np+n] = 0.125*Psi[ijk];		
+		dist[2*Np+n] = 0.125*Psi[ijk];	
+		dist[3*Np+n] = 0.125*Psi[ijk];	
+		dist[4*Np+n] = 0.125*Psi[ijk];	
+		dist[5*Np+n] = 0.125*Psi[ijk];	
+		dist[6*Np+n] = 0.125*Psi[ijk];	
+	}
+}
+
+//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
+//        int strideY, int strideZ,int start, int finish, int Np){
+//
+//	int n,nn;
+//    int ijk;
+//    int id;
+//	// distributions
+//	double m1,m2,m3,m4,m5,m6,m7,m8,m9;
+//	double m10,m11,m12,m13,m14,m15,m16,m17,m18;
+//	double nx,ny,nz;
+//
+//	for (n=start; n<finish; n++){
+//
+//		// Get the 1D index based on regular data layout
+//		ijk = Map[n];
+//		//					COMPUTE THE COLOR GRADIENT
+//		//........................................................................
+//		//.................Read Phase Indicator Values............................
+//		//........................................................................
+//		nn = ijk-1;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m1 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1
+//		//........................................................................
+//		nn = ijk+1;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2
+//		//........................................................................
+//		nn = ijk-strideY;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3
+//		//........................................................................
+//		nn = ijk+strideY;							// neighbor index (get convention)
+//        id = ID[nn];
+//		m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4
+//		//........................................................................
+//		nn = ijk-strideZ;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5
+//		//........................................................................
+//		nn = ijk+strideZ;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6
+//		//........................................................................
+//		nn = ijk-strideY-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7
+//		//........................................................................
+//		nn = ijk+strideY+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8
+//		//........................................................................
+//		nn = ijk+strideY-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9
+//		//........................................................................
+//		nn = ijk-strideY+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10
+//		//........................................................................
+//		nn = ijk-strideZ-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11
+//		//........................................................................
+//		nn = ijk+strideZ+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12
+//		//........................................................................
+//		nn = ijk+strideZ-1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13
+//		//........................................................................
+//		nn = ijk-strideZ+1;						// neighbor index (get convention)
+//        id = ID[nn];
+//		m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14
+//		//........................................................................
+//		nn = ijk-strideZ-strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15
+//		//........................................................................
+//		nn = ijk+strideZ+strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16
+//		//........................................................................
+//		nn = ijk+strideZ-strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17
+//		//........................................................................
+//		nn = ijk-strideZ+strideY;					// neighbor index (get convention)
+//        id = ID[nn];
+//		m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18
+//		//............Compute the Color Gradient...................................
+//		//nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14));
+//		//ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18));
+//		//nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18));
+//		nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3
+//		ny = 1.f/6.f*(m3-m4);
+//		nz = 1.f/6.f*(m5-m6);
+//		
+//		ElectricField[n] = nx;
+//		ElectricField[Np+n] = ny;
+//		ElectricField[2*Np+n] = nz;
+//	}
+//}
+
+//extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np){
+//	int n;
+//	// distributions
+//	double f1,f2,f3,f4,f5,f6;
+//	double Ex,Ey,Ez;
+//    double rlx=1.0/tau;
+//
+//	for (n=0; n<Np; n++){
+//		//........................................................................
+//		// Registers to store the distributions
+//		//........................................................................
+//		f1 = dist[Np+n];
+//		f2 = dist[2*Np+n];
+//		f3 = dist[3*Np+n];
+//		f4 = dist[4*Np+n];
+//		f5 = dist[5*Np+n];
+//		f6 = dist[6*Np+n];
+//		//.................Compute the Electric Field...................................
+//		//Ex = (f1-f2)*rlx*4.5;//NOTE the unit of electric field here is V/lu
+//		//Ey = (f3-f4)*rlx*4.5;
+//		//Ez = (f5-f6)*rlx*4.5;
+//		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
+//		Ey = (f3-f4)*rlx*4.0;
+//		Ez = (f5-f6)*rlx*4.0;
+//		//..................Write the Electric Field.....................................
+//		ElectricField[0*Np+n] = Ex;
+//		ElectricField[1*Np+n] = Ey;
+//		ElectricField[2*Np+n] = Ez;
+//		//........................................................................
+//	}
+//}

cpu/Stokes.cpp

@@ -0,0 +1,999 @@
+#include <stdio.h>
+
+extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np)
+{
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	constexpr double mrt_V1=0.05263157894736842;
+	constexpr double mrt_V2=0.012531328320802;
+	constexpr double mrt_V3=0.04761904761904762;
+	constexpr double mrt_V4=0.004594820384294068;
+	constexpr double mrt_V5=0.01587301587301587;
+	constexpr double mrt_V6=0.0555555555555555555555555;
+	constexpr double mrt_V7=0.02777777777777778;
+	constexpr double mrt_V8=0.08333333333333333;
+	constexpr double mrt_V9=0.003341687552213868;
+	constexpr double mrt_V10=0.003968253968253968;
+	constexpr double mrt_V11=0.01388888888888889;
+	constexpr double mrt_V12=0.04166666666666666;
+
+	for (int n=start; n<finish; n++){
+        
+        //Load data
+        rhoE = ChargeDensity[n];
+        Ex = ElectricField[n+0*Np];
+        Ey = ElectricField[n+1*Np];
+        Ez = ElectricField[n+2*Np];
+        //compute total body force, including input body force (Gx,Gy,Gz)
+        Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
+        Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+		// q=0
+		fq = dist[n];
+		rho = fq;
+		m1  = -30.0*fq;
+		m2  = 12.0*fq;
+
+		// q=1
+		fq = dist[2*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jx = fq;
+		m4 = -4.0*fq;
+		m9 = 2.0*fq;
+		m10 = -4.0*fq;
+
+		// f2 = dist[10*Np+n];
+		fq = dist[1*Np+n];
+		rho += fq;
+		m1 -= 11.0*(fq);
+		m2 -= 4.0*(fq);
+		jx -= fq;
+		m4 += 4.0*(fq);
+		m9 += 2.0*(fq);
+		m10 -= 4.0*(fq);
+
+		// q=3
+		fq = dist[4*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy = fq;
+		m6 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 = fq;
+		m12 = -2.0*fq;
+
+		// q = 4
+		fq = dist[3*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy -= fq;
+		m6 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 += fq;
+		m12 -= 2.0*fq;
+
+		// q=5
+		fq = dist[6*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz = fq;
+		m8 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q = 6
+		fq = dist[5*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz -= fq;
+		m8 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q=7
+		fq = dist[8*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy += fq;
+		m6 += fq;
+		m9  += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 = fq;
+		m16 = fq;
+		m17 = -fq;
+
+		// q = 8
+		fq = dist[7*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 += fq;
+		m16 -= fq;
+		m17 += fq;
+
+		// q=9
+		fq = dist[10*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 += fq;
+		m17 += fq;
+
+		// q = 10
+		fq = dist[9*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy += fq;
+		m6 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 -= fq;
+		m17 -= fq;
+
+		// q=11
+		fq = dist[12*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 = fq;
+		m16 -= fq;
+		m18 = fq;
+
+		// q=12
+		fq = dist[11*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 += fq;
+		m16 += fq;
+		m18 -= fq;
+
+		// q=13
+		fq = dist[14*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 -= fq;
+		m18 -= fq;
+
+		// q=14
+		fq = dist[13*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 += fq;
+		m18 += fq;
+
+		// q=15
+		fq = dist[16*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 = fq;
+		m17 += fq;
+		m18 -= fq;
+
+		// q=16
+		fq = dist[15*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 += fq;
+		m17 -= fq;
+		m18 += fq;
+
+		// q=17
+		fq = dist[18*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 += fq;
+		m18 += fq;
+
+		// q=18
+		fq = dist[17*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 -= fq;
+		m18 -= fq;
+
+		// write the velocity 
+		ux = jx / rho0;
+		uy = jy / rho0;
+		uz = jz / rho0;
+		Velocity[n] = ux;
+		Velocity[Np+n] = uy;
+		Velocity[2*Np+n] = uz;
+
+
+		//........................................................................
+		//					READ THE DISTRIBUTIONS
+		//		(read from opposite array due to previous swap operation)
+		//........................................................................
+
+		//..............incorporate external force................................................
+		//..............carry out relaxation process...............................................
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+		m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+		m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+		m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+		m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+		m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+		m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+		m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+		m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+		m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+		m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+		m16 = m16 + rlx_setB*( - m16);
+		m17 = m17 + rlx_setB*( - m17);
+		m18 = m18 + rlx_setB*( - m18);
+		//.......................................................................................................
+		//.................inverse transformation......................................................
+
+		// q=0
+		fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+		dist[n] = fq;
+
+		// q = 1
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
+		dist[1*Np+n] = fq;
+
+		// q=2
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+		dist[2*Np+n] = fq;
+
+		// q = 3
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+		dist[3*Np+n] = fq;
+
+		// q = 4
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+		dist[4*Np+n] = fq;
+
+		// q = 5
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+		dist[5*Np+n] = fq;
+
+		// q = 6
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+		dist[6*Np+n] = fq;
+
+		// q = 7
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+		dist[7*Np+n] = fq;
+
+
+		// q = 8
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+				+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+		dist[8*Np+n] = fq;
+
+		// q = 9
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+		dist[9*Np+n] = fq;
+
+		// q = 10
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+		dist[10*Np+n] = fq;
+
+
+		// q = 11
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+		dist[11*Np+n] = fq;
+
+		// q = 12
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                        								+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                        								-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+		dist[12*Np+n] = fq;
+
+		// q = 13
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+		dist[13*Np+n] = fq;
+
+		// q= 14
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+
+		dist[14*Np+n] = fq;
+
+		// q = 15
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+		dist[15*Np+n] = fq;
+
+		// q = 16
+		fq =  mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+		dist[16*Np+n] = fq;
+
+
+		// q = 17
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+		dist[17*Np+n] = fq;
+
+		// q = 18
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+		dist[18*Np+n] = fq;
+
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np)
+{
+    double fq;
+	// conserved momemnts
+	double rho,jx,jy,jz;
+    double ux,uy,uz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+	int nread;
+    // body force due to electric field
+    double rhoE;//charge density
+    double Ex,Ey,Ez;
+    // total body force
+    double Fx,Fy,Fz;
+
+	constexpr double mrt_V1=0.05263157894736842;
+	constexpr double mrt_V2=0.012531328320802;
+	constexpr double mrt_V3=0.04761904761904762;
+	constexpr double mrt_V4=0.004594820384294068;
+	constexpr double mrt_V5=0.01587301587301587;
+	constexpr double mrt_V6=0.0555555555555555555555555;
+	constexpr double mrt_V7=0.02777777777777778;
+	constexpr double mrt_V8=0.08333333333333333;
+	constexpr double mrt_V9=0.003341687552213868;
+	constexpr double mrt_V10=0.003968253968253968;
+	constexpr double mrt_V11=0.01388888888888889;
+	constexpr double mrt_V12=0.04166666666666666;
+
+	for (int n=start; n<finish; n++){
+
+        //Load data
+        rhoE = ChargeDensity[n];
+        Ex = ElectricField[n+0*Np];
+        Ey = ElectricField[n+1*Np];
+        Ez = ElectricField[n+2*Np];
+        //compute total body force, including input body force (Gx,Gy,Gz)
+        Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+        Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
+
+		// q=0
+		fq = dist[n];
+		rho = fq;
+		m1  = -30.0*fq;
+		m2  = 12.0*fq;
+
+		// q=1
+		nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		fq = dist[nread]; // reading the f1 data into register fq
+		//fp = dist[10*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jx = fq;
+		m4 = -4.0*fq;
+		m9 = 2.0*fq;
+		m10 = -4.0*fq;
+
+		// f2 = dist[10*Np+n];
+		nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		fq = dist[nread];  // reading the f2 data into register fq
+		//fq = dist[Np+n];
+		rho += fq;
+		m1 -= 11.0*(fq);
+		m2 -= 4.0*(fq);
+		jx -= fq;
+		m4 += 4.0*(fq);
+		m9 += 2.0*(fq);
+		m10 -= 4.0*(fq);
+
+		// q=3
+		nread = neighborList[n+2*Np]; // neighbor 4
+		fq = dist[nread];
+		//fq = dist[11*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy = fq;
+		m6 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 = fq;
+		m12 = -2.0*fq;
+
+		// q = 4
+		nread = neighborList[n+3*Np]; // neighbor 3
+		fq = dist[nread];
+		//fq = dist[2*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jy -= fq;
+		m6 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 += fq;
+		m12 -= 2.0*fq;
+
+		// q=5
+		nread = neighborList[n+4*Np];
+		fq = dist[nread];
+		//fq = dist[12*Np+n];
+		rho += fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz = fq;
+		m8 = -4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+
+		// q = 6
+		nread = neighborList[n+5*Np];
+		fq = dist[nread];
+		//fq = dist[3*Np+n];
+		rho+= fq;
+		m1 -= 11.0*fq;
+		m2 -= 4.0*fq;
+		jz -= fq;
+		m8 += 4.0*fq;
+		m9 -= fq;
+		m10 += 2.0*fq;
+		m11 -= fq;
+		m12 += 2.0*fq;
+
+		// q=7
+		nread = neighborList[n+6*Np];
+		fq = dist[nread];
+		//fq = dist[13*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy += fq;
+		m6 += fq;
+		m9  += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 = fq;
+		m16 = fq;
+		m17 = -fq;
+
+		// q = 8
+		nread = neighborList[n+7*Np];
+		fq = dist[nread];
+		//fq = dist[4*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 += fq;
+		m16 -= fq;
+		m17 += fq;
+
+		// q=9
+		nread = neighborList[n+8*Np];
+		fq = dist[nread];
+		//fq = dist[14*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jy -= fq;
+		m6 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 += fq;
+		m17 += fq;
+
+		// q = 10
+		nread = neighborList[n+9*Np];
+		fq = dist[nread];
+		//fq = dist[5*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jy += fq;
+		m6 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 += fq;
+		m12 += fq;
+		m13 -= fq;
+		m16 -= fq;
+		m17 -= fq;
+
+		// q=11
+		nread = neighborList[n+10*Np];
+		fq = dist[nread];
+		//fq = dist[15*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 = fq;
+		m16 -= fq;
+		m18 = fq;
+
+		// q=12
+		nread = neighborList[n+11*Np];
+		fq = dist[nread];
+		//fq = dist[6*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 += fq;
+		m16 += fq;
+		m18 -= fq;
+
+		// q=13
+		nread = neighborList[n+12*Np];
+		fq = dist[nread];
+		//fq = dist[16*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx += fq;
+		m4 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 -= fq;
+		m18 -= fq;
+
+		// q=14
+		nread = neighborList[n+13*Np];
+		fq = dist[nread];
+		//fq = dist[7*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jx -= fq;
+		m4 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 += fq;
+		m10 += fq;
+		m11 -= fq;
+		m12 -= fq;
+		m15 -= fq;
+		m16 += fq;
+		m18 += fq;
+
+		// q=15
+		nread = neighborList[n+14*Np];
+		fq = dist[nread];
+		//fq = dist[17*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 = fq;
+		m17 += fq;
+		m18 -= fq;
+
+		// q=16
+		nread = neighborList[n+15*Np];
+		fq = dist[nread];
+		//fq = dist[8*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 += fq;
+		m17 -= fq;
+		m18 += fq;
+
+		// q=17
+		//fq = dist[18*Np+n];
+		nread = neighborList[n+16*Np];
+		fq = dist[nread];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy += fq;
+		m6 += fq;
+		jz -= fq;
+		m8 -= fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 += fq;
+		m18 += fq;
+
+		// q=18
+		nread = neighborList[n+17*Np];
+		fq = dist[nread];
+		//fq = dist[9*Np+n];
+		rho += fq;
+		m1 += 8.0*fq;
+		m2 += fq;
+		jy -= fq;
+		m6 -= fq;
+		jz += fq;
+		m8 += fq;
+		m9 -= 2.0*fq;
+		m10 -= 2.0*fq;
+		m14 -= fq;
+		m17 -= fq;
+		m18 -= fq;
+
+		// write the velocity 
+		ux = jx / rho0;
+		uy = jy / rho0;
+		uz = jz / rho0;
+		Velocity[n] = ux;
+		Velocity[Np+n] = uy;
+		Velocity[2*Np+n] = uz;
+
+		//..............incorporate external force................................................
+		//..............carry out relaxation process...............................................
+		m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
+		m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
+		m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+		m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+		m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+		m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
+		m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+		m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
+		m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
+		m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
+		m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
+		m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
+		m16 = m16 + rlx_setB*( - m16);
+		m17 = m17 + rlx_setB*( - m17);
+		m18 = m18 + rlx_setB*( - m18);
+		//.......................................................................................................
+		//.................inverse transformation......................................................
+
+		// q=0
+		fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
+		dist[n] = fq;
+
+		// q = 1
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
+		nread = neighborList[n+Np];
+		dist[nread] = fq;
+
+		// q=2
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) -  0.16666666*Fx;
+		nread = neighborList[n];
+		dist[nread] = fq;
+
+		// q = 3
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
+		nread = neighborList[n+3*Np];
+		dist[nread] = fq;
+
+		// q = 4
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
+		nread = neighborList[n+2*Np];
+		dist[nread] = fq;
+
+		// q = 5
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
+		nread = neighborList[n+5*Np];
+		dist[nread] = fq;
+
+		// q = 6
+		fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
+		nread = neighborList[n+4*Np];
+		dist[nread] = fq;
+
+		// q = 7
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
+		nread = neighborList[n+7*Np];
+		dist[nread] = fq;
+
+		// q = 8
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+				+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
+		nread = neighborList[n+6*Np];
+		dist[nread] = fq;
+
+		// q = 9
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
+		nread = neighborList[n+9*Np];
+		dist[nread] = fq;
+
+		// q = 10
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+                                                								+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+                                                								+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
+		nread = neighborList[n+8*Np];
+		dist[nread] = fq;
+
+		// q = 11
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
+		nread = neighborList[n+11*Np];
+		dist[nread] = fq;
+
+		// q = 12
+		fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+                                        								+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+                                        								-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
+		nread = neighborList[n+10*Np];
+		dist[nread]= fq;
+
+		// q = 13
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
+		nread = neighborList[n+13*Np];
+		dist[nread] = fq;
+
+		// q= 14
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+				+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
+				-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
+		nread = neighborList[n+12*Np];
+		dist[nread] = fq;
+
+
+		// q = 15
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
+		nread = neighborList[n+15*Np];
+		dist[nread] = fq;
+
+		// q = 16
+		fq =  mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
+				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
+		nread = neighborList[n+14*Np];
+		dist[nread] = fq;
+
+
+		// q = 17
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
+		nread = neighborList[n+17*Np];
+		dist[nread] = fq;
+
+		// q = 18
+		fq = mrt_V1*rho+mrt_V9*m1
+				+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
+				-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
+		nread = neighborList[n+16*Np];
+		dist[nread] = fq;
+
+	}
+}
+
+//extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np)
+//{
+//    //h: resolution [um/lu]
+//    //time_conv: time conversion factor [sec/lt]
+//	int n;
+//	// distributions
+//	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
+//	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
+//	double vx,vy,vz;
+//
+//	for (n=0; n<Np; n++){
+//		//........................................................................
+//		// Registers to store the distributions
+//		//........................................................................
+//		f2 = dist[2*Np+n];
+//		f4 = dist[4*Np+n];
+//		f6 = dist[6*Np+n];
+//		f8 = dist[8*Np+n];
+//		f10 = dist[10*Np+n];
+//		f12 = dist[12*Np+n];
+//		f14 = dist[14*Np+n];
+//		f16 = dist[16*Np+n];
+//		f18 = dist[18*Np+n];
+//		//........................................................................
+//		f1 = dist[Np+n];
+//		f3 = dist[3*Np+n];
+//		f5 = dist[5*Np+n];
+//		f7 = dist[7*Np+n];
+//		f9 = dist[9*Np+n];
+//		f11 = dist[11*Np+n];
+//		f13 = dist[13*Np+n];
+//		f15 = dist[15*Np+n];
+//		f17 = dist[17*Np+n];
+//		//.................Compute the velocity...................................
+//		vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+//		vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+//		vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+//		//..................Write the velocity.....................................
+//		vel[0*Np+n] = vx*(h*1.0e-6)/time_conv;
+//		vel[1*Np+n] = vy*(h*1.0e-6)/time_conv;
+//		vel[2*Np+n] = vz*(h*1.0e-6)/time_conv;
+//		//........................................................................
+//	}
+//}

cpu/dfh.cpp

@@ -53,11 +53,10 @@ extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, do
 	}
 }
 
-extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np){
-	int idx,n;
-	double phi,nA,nB;
-
-	for (idx=start; idx<finish; idx++){
+extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np)
+{
+	for (int idx=start; idx<finish; idx++){
+	    double phi,nA,nB;
 		phi = Phi[idx];
 		if (phi > 0.f){
 			nA = 1.0; nB = 0.f;
@@ -90,15 +89,13 @@ extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq
 // LBM based on density functional hydrodynamics
 extern "C" void ScaLBL_D3Q19_AAeven_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
-		double Fx, double Fy, double Fz, int start, int finish, int Np){
-
-	int ijk,nn,n;
+		double Fx, double Fy, double Fz, int start, int finish, int Np)
+{
 	double fq;
 	// conserved momemnts
 	double rho,jx,jy,jz;
 	// non-conserved moments
 	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
-	double m3,m5,m7;
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
@@ -616,7 +613,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 		double *Phi, double *Gradient, double *SolidForce, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int start, int finish, int Np){
 	
-	int n,nn,ijk,nread;
+	int nread;
 	int nr1,nr2,nr3,nr4,nr5,nr6;
 	int nr7,nr8,nr9,nr10;
 	int nr11,nr12,nr13,nr14;
@@ -626,7 +623,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 	double rho,jx,jy,jz;
 	// non-conserved moments
 	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
-	double m3,m5,m7;
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
@@ -1212,12 +1208,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *
 }
 
 extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq, 
-			double *Den, double *Phi, int start, int finish, int Np){
-
-	int idx,n,nread;
-	double fq,nA,nB;
+			double *Den, double *Phi, int start, int finish, int Np)
+{
 
 	for (int n=start; n<finish; n++){
+		int nread;
+		double fq,nA,nB;
 		
 		//..........Compute the number density for component A............
 		// q=0
@@ -1300,11 +1296,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *neighborList, double *Aq, double *Bq,
 }
 
 extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, double *Phi, 
-			int start, int finish, int Np){
-	int idx,n,nread;
-	double fq,nA,nB;
+			int start, int finish, int Np)
+{
 	for (int n=start; n<finish; n++){
-		
+		double fq,nA,nB;
 		// compute number density for component A
 		// q=0
 		fq = Aq[n];

example/Bubble/input.db

@@ -7,15 +7,7 @@ Color {
     beta  = 0.95;
     F = 0, 0, 0
     Restart = false
-    pBC = 0
-    din = 1.0
-    dout = 1.0
-    flux = 1.0e-3
     timestepMax = 200
-    interval = 1000
-    tol = 1e-5;
-    das = 0.1
-    dbs = 0.9
     ComponentLabels = 0, 1, 2
     ComponentAffinity = -1.0, 1.0, -1.0
 }
@@ -38,4 +30,5 @@ Analysis {
     load_balance = "independent" // Load balance method to use: "none", "default", "independent"
 }
 
-
+Visualization {
+}

example/CMakeLists.txt

@@ -1,5 +1,5 @@
 # Copy the examples to the install folder
-INSTALL_EXAMPLE (NonNewtonianChannelFlow )
+#INSTALL_EXAMPLE (NonNewtonianChannelFlow )
 INSTALL_EXAMPLE( Bubble )
 INSTALL_EXAMPLE( ConstrainedBubble )
 INSTALL_EXAMPLE( Piston )
@@ -10,9 +10,10 @@ INSTALL_EXAMPLE( Sph1896 )
 INSTALL_EXAMPLE( drainage )
 INSTALL_EXAMPLE( imbibition )
 INSTALL_EXAMPLE( relperm )
-INSTALL_EXAMPLE( Poiseuille )
+#INSTALL_EXAMPLE( Poiseuille )
 INSTALL_EXAMPLE( Juanes )
 INSTALL_EXAMPLE( MicroModel )
+INSTALL_EXAMPLE( CornerFlow )
 INSTALL_EXAMPLE( Tiff )
 INSTALL_EXAMPLE( uCT )
 

example/CornerFlow/CornerFlow.py

@@ -0,0 +1,61 @@
+import numpy
+import math
+
+nx=128
+ny=64
+nz=64
+N=nx*ny*nz
+
+mesh=(nx,ny,nz)
+data=numpy.ones(mesh,dtype=numpy.int8)
+
+#ylinder centers
+c1x=-1.0
+c1y=0.0
+c2x=1.0
+c2y=0.0
+c3x=0.0
+c3y=math.sqrt(3.0)
+
+# domain size
+x0=-0.5
+y0=0.0
+L=1.0
+#2.0*(1.0-1.0/math.sqrt(2.0))
+dx=L/nz
+radius=1.0
+
+print("Mesh size: "+repr(mesh))
+print("Length: "+repr(L))
+print("dx: "+repr(dx))
+print("box corner: "+repr(x0)+","+repr(y0))
+print("cylinder 1: "+repr(c1x)+","+repr(c1y))
+print("cylinder 2: "+repr(c2x)+","+repr(c2y))
+print("cylinder 3: "+repr(c3x)+","+repr(c3y))
+
+#print(data)
+count=0
+# set up the tube with corner flow
+for i in range(0,nx):
+  for j in range(0,ny):
+    for k in range(0,nz):
+       z = i*dx
+       x = j*dx+x0
+       y = k*dx+y0
+       label=2
+       d1 = math.sqrt((x-c1x)*(x-c1x)+(y-c1y)*(y-c1y)) - radius
+       d2 = math.sqrt((x-c2x)*(x-c2x)+(y-c2y)*(y-c2y)) - radius
+       d3 = math.sqrt((x-c3x)*(x-c3x)+(y-c3y)*(y-c3y)) - radius
+       if d1 < 0.0:
+         label=0
+       if d2 < 0.0:
+         label=0
+       if d3 < 0.0:
+         label=0
+       data[i,j,k]=label
+       if label==2:
+         count+=1
+
+print(count)
+
+data.tofile("CornerFlow.raw")

example/CornerFlow/input.db

@@ -0,0 +1,51 @@
+Color {
+    tauA   = 0.7;  
+    tauB   = 0.7;  
+    rhoA   = 1.0;  
+    rhoB   = 1.0;  
+    alpha = 1e-3;
+    beta  = 0.95;
+    F = 0, 0, 0
+    Restart = false
+    pBC = 0
+    din = 1.0
+    dout = 1.0
+    timestepMax = 3000
+    interval = 1000
+    tol = 1e-5;
+    das = 0.1
+    dbs = 0.9
+    flux = 0.0
+    ComponentLabels = 0, -1        // labels for immobile components
+    ComponentAffinity = -1.0, 0.5  // wetting condition for immobile components (-1 = water wet / +1 oil wet)
+}
+
+Domain {
+    Filename = "CornerFlow.raw"
+    nproc = 1, 1, 2     // Number of processors (Npx,Npy,Npz)
+    n = 64, 64, 64      // Size of local domain (Nx,Ny,Nz)
+    N = 64, 64, 128      // size of the input image
+    n_spheres = 1       // Number of spheres
+    L = 1, 1, 1         // Length of domain (x,y,z)
+    BC = 0              // Boundary condition type
+    ReadType = "8bit"
+    ReadValues = 0, 1, 2  // labels within the original input image
+    WriteValues = 0, 1, 2 // labels to write (if they aren't the same)
+    ComponentLabels = 0   // labels that are immobile
+    HistoryLabels = -1    // new labels to assign to each immobile component based on fluid history
+    Sw = 0.3              // target saturation for morphological routines
+} 
+
+Analysis {
+    blobid_interval = 1000      // Frequency to perform blob identification
+    analysis_interval = 1000    // Frequency to perform analysis
+    restart_interval = 1000    // Frequency to write restart data
+    visualization_interval = 2000        // Frequency to write visualization data
+    restart_file = "Restart"    // Filename to use for restart file (will append rank)
+    N_threads    = 4            // Number of threads to use
+    load_balance = "independent" // Load balance method to use: "none", "default", "independent"
+}
+
+
+Visualization {
+}

example/InkBottle/Color.in

@@ -1,6 +0,0 @@
-1.0
-1.0e-2 0.95 -1.0
-0.7
-0.0 0.0 0.0
-0 1 1.01 0.99
-50000 20000 1e-5

example/InkBottle/Domain.in

@@ -1,3 +0,0 @@
-1 1 1
-80 80 80
-1.0 1.0 1.0

example/Piston/Color.in

@@ -1,6 +0,0 @@
-1.0 1.0
-1.0 1.0
-1.0e-3 0.95
-0.0 0.0 0.0
-0 4 10.0 1.0
-10000 2000 1e-5

example/Piston/ComponentLabels.csv

@@ -1,3 +0,0 @@
-0 -0.5
-1 1.0
-2 -1.0

example/Piston/Domain.in

@@ -1,4 +0,0 @@
-1 1 10
-40 40 40
-229
-1.0 1.0 1.0

example/Piston/input.db

@@ -7,14 +7,7 @@ Color {
     beta  = 0.95;
     F = 0, 0, 0
     Restart = false
-    pBC = 0
-    din = 1.0
-    dout = 1.0
     timestepMax = 3000
-    interval = 1000
-    tol = 1e-5;
-    das = 0.1
-    dbs = 0.9
     flux = 2.0
     ComponentLabels = 0
     ComponentAffinity = -1.0;
@@ -25,7 +18,6 @@ Domain {
     nproc = 1, 1, 4     // Number of processors (Npx,Npy,Npz)
     n = 24, 24, 24      // Size of local domain (Nx,Ny,Nz)
     N = 24, 24, 96      // size of the input image
-    n_spheres = 1       // Number of spheres
     L = 1, 1, 1         // Length of domain (x,y,z)
     BC = 4              // Boundary condition type
     ReadType = "8bit"   

example/Piston/poiseuille.db

@@ -18,3 +18,5 @@ Domain {
     BC = 0              // Boundary condition type
 }
 
+Visualization {
+}

example/Plates/input.db

@@ -7,14 +7,7 @@ Color {
     beta  = 0.95;
     F = 0, 0, 0
     Restart = false
-    pBC = 0
-    din = 1.0
-    dout = 1.0
     timestepMax = 3000
-    interval = 1000
-    tol = 1e-5;
-    das = 0.1
-    dbs = 0.9
     flux = 0.0
     ComponentLabels = -2, -1
     ComponentAffinity = -1.0, -0.5;
@@ -25,7 +18,6 @@ Domain {
     nproc = 1, 1, 4     // Number of processors (Npx,Npy,Npz)
     n = 24, 24, 24      // Size of local domain (Nx,Ny,Nz)
     N = 24, 24, 96      // size of the input image
-    n_spheres = 1       // Number of spheres
     L = 1, 1, 1         // Length of domain (x,y,z)
     BC = 0              // Boundary condition type
     ReadType = "8bit"

example/Poiseuille/Domain.in

@@ -1,3 +0,0 @@
-1 1 1
-100 100 100
-1.0 1.0 1.0

example/Poiseuille/Permeability.in

@@ -1,4 +0,0 @@
-1.0
-0.0 0.0 1.0e-5
-0 0 1.0 1.0
-1000 1000 1.0e-5

example/Poiseuille/README.md

@@ -1,11 +0,0 @@
-# 1. Edit the Domain.in file based on the desired system size
-
-
-# 2. Run the pre-processor 
-export TUBEWIDTH=20
-export LAYERWIDTH=0
-mpirun -np 1 ../../tests/lbpm_plates_pp $TUBEWIDTH $LAYERWIDTH
-
-
-# 3. Run single-phase simulation within the domain
-mpirun -np 1 ../../tests/lbpm_permeability_simulator

example/Sph125/Domain.in

@@ -1,3 +0,0 @@
-2 2 2
-100 100 100
-1.0 1.0 1.0

example/Sph125/input.db

@@ -37,3 +37,5 @@ Analysis {
 }
 
 
+Visualization {
+}

example/Sph1896/BlobSize.in

@@ -1,2 +0,0 @@
-1 
-32 32 32

example/Sph1896/Color.in

@@ -1,6 +0,0 @@
-1.0 1.0
-1.0 1.0
-1.0e-3 0.95
-0.0 0.0 1.0e-5
-0 0 10.0 1.0
-10000 2000 1e-5

example/Sph1896/Domain.in

@@ -1,3 +0,0 @@
-2 2 2
-320 320 320
-1.0 1.0 1.0

example/Sph1896/input.db

@@ -26,28 +26,30 @@ Color {
 }
 
 Domain {
-    nproc = 2, 2, 2        // Number of processors (Npx,Npy,Npz)
-    n = 120, 120, 120      // Size of local domain (Nx,Ny,Nz)
+    nproc = 1, 1, 1        // Number of processors (Npx,Npy,Npz)
+    n = 320, 320, 320      // Size of local domain (Nx,Ny,Nz)
+    N = 320, 320, 320      
     nspheres = 1896        // Number of spheres (only needed if using a sphere packing)
     L = 1, 1, 1            // Length of domain (x,y,z)
     BC = 0                 // Boundary condition type 
                            //     BC = 0  for periodic BC
                            //     BC = 1  for pressure BC (applied in z direction)
                            //     BC = 4  for flux BC (applied in z direction
-    Filename = "mineralmodel.raw"                    // name of raw binary file to read digital image from 
     ReadType = "8bit"
-    ReadValues = 0, 1, 2, 3, 4, 5, 6, 7, 8           // list of labels within the binary file (read)
-    WriteValues = 1, -1, -2, -2, -3, -3, -4, -4, -4  // list of labels within the output files (write)
+    ReadValues = 0, 1, 2           // list of labels within the binary file (read)
+    WriteValues = 0, 1, 2          // list of labels within the output files (write)
 
 }
 
 Analysis {
-    blobid_interval = 1000      // Frequency to perform blob identification
     analysis_interval = 1000    // Frequency to perform analysis
-    restart_interval = 2000    // Frequency to write restart data
-    visualization_interval = 2000        // Frequency to write visualization data
+    restart_interval = 50000    // Frequency to write restart data
+    visualization_interval = 50000        // Frequency to write visualization data
     restart_file = "Restart"    // Filename to use for restart file (will append rank)
     N_threads    = 4            // Number of threads to use
     load_balance = "independent" // Load balance method to use: "none", "default", "independent"
 }
 
+Visualization {
+
+}

example/SquareTube/input.db

@@ -43,3 +43,5 @@ Analysis {
 }
 
 
+Visualization {
+}

example/Workflow/ComputeSaturation.py

@@ -0,0 +1,37 @@
+import sys
+import numpy as np
+import matplotlib.pylab as plt
+
+FILENAME=sys.argv[1]
+Nx=int(sys.argv[2])
+Ny=int(sys.argv[3])
+Nz=int(sys.argv[4])
+
+# read the input image
+Output = np.fromfile(FILENAME,dtype = np.uint8)
+Output.shape = (Nz,Ny,Nx)
+
+Oil=np.count_nonzero(Output==1)
+Water=np.count_nonzero(Output==2)
+Sw=Water/(Oil+Water)
+
+Porosity=1.0-(Oil+Water)/(Nx*Ny*Nz)
+
+print(FILENAME,"Porosity=", Porosity)
+
+SaturationProfile=np.zeros(Nz)
+PorosityProfile=np.zeros(Nz)
+# Compute saturation slice by slice 
+for idx in range(0, Nz):
+   Slice = Output[idx,:,:]
+   Oil=np.count_nonzero(Slice==1)
+   Water=np.count_nonzero(Slice==2)
+   SaturationProfile[idx]=Water/(Oil+Water)
+   PorosityProfile[idx]=(Oil+Water)/(Nx*Ny)
+   
+
+plt.figure()
+plt.plot(SaturationProfile)
+plt.xlabel('Position (z)')
+plt.ylabel('Water Saturation')
+plt.show()

example/Workflow/HelperFunctions.R

@@ -0,0 +1,112 @@
+require("ggplot2")
+
+GG_THEME=theme_bw()+theme(panel.grid.major = element_blank(),panel.grid.minor = element_blank())
+
+ReadDatabase<-function(FILE){
+
+    INPUT<-gsub(';','',readLines(FILE))
+
+    S<-gsub('tauA = ','',gsub("\\s+"," ",(grep("tauA",INPUT,value=TRUE))))
+    TAU_A = as.numeric(gsub("/.*","",S))
+    S<-gsub('tauB = ','',gsub("\\s+"," ",(grep("tauB",INPUT,value=TRUE))))
+    TAU_B = as.numeric(gsub("/.*","",S))
+    S<-gsub('rhoA = ','',gsub("\\s+"," ",(grep("rhoA",INPUT,value=TRUE))))
+    RHO_A = as.numeric(gsub("/.*","",S))
+    S<-gsub('rhoB = ','',gsub("\\s+"," ",(grep("rhoB",INPUT,value=TRUE))))
+    RHO_B = as.numeric(gsub("/.*","",S))
+
+    S<-gsub('alpha = ','',gsub("\\s+"," ",(grep("alpha",INPUT,value=TRUE))))
+    ALPHA = as.numeric(gsub("/.*","",S))
+
+    # Read the affinity
+    S<-gsub('ComponentAffinity = ','',gsub("\\s+"," ",(grep("ComponentAffinity",INPUT,value=TRUE))))
+    S<-gsub("/.*","",S)
+    AFFINITY<-as.numeric(unlist(strsplit(S,", ")))
+
+    PARAMETERS<-c(TAU_A,TAU_B,RHO_A,RHO_B,ALPHA,AFFINITY)
+    
+    return(PARAMETERS)
+}
+
+ReadSubphase<-function(PATH){
+  FILE=paste0(PATH,"/subphase.csv")
+  S<-read.csv(FILE,head=TRUE,sep=" ")
+  S$Vw<-S$Vwc+S$Vwd
+  S$Vn<-S$Vnc+S$Vnd
+  S$Aw<-S$Awc+S$Awd
+  S$An<-S$Anc+S$And
+  S$Hw<-S$Hwc+S$Hwd
+  S$Hn<-S$Hnc+S$Hnd
+  S$Xw<-S$Xwc+S$Xwd
+  S$Xn<-S$Xnc+S$Xnd
+
+  S$Sw<-S$Vw/(S$Vn+S$Vw)
+  S$pw<-(S$pwc*S$Vwc+S$pwd*S$Vwd) / (S$Vwc+S$Vwd)
+  S$pn<-(S$pnc*S$Vnc+S$pnd*S$Vnd) / (S$Vnc+S$Vnd)
+
+  S$Qwx<-S$Vw*(S$Pwc_x+S$Pwd_x)/(S$Mwc+S$Mwd)
+  S$Qnx<-S$Vn*(S$Pnc_x+S$Pnd_x)/(S$Mnc+S$Mnd)
+  S$Qwy<-S$Vw*(S$Pwc_y+S$Pwd_y)/(S$Mwc+S$Mwd)
+  S$Qny<-S$Vn*(S$Pnc_y+S$Pnd_y)/(S$Mnc+S$Mnd)
+  S$Qwz<-S$Vw*(S$Pwc_z+S$Pwd_z)/(S$Mwc+S$Mwd)
+  S$Qnz<-S$Vn*(S$Pnc_z+S$Pnd_z)/(S$Mnc+S$Mnd)
+
+  S$Krn<-S$nun*S$Qnz/S$Fz
+  S$Krw<-S$nuw*S$Qwz/S$Fz
+  S$Case<-PATH
+  return(S)
+}
+
+ReadTimelog<-function(PATH){
+	FILE=paste0(PATH,"/timelog.csv")
+	D<-read.csv(file=FILE,head=TRUE,sep=" ")
+	D$time<-seq(1,nrow(D))
+	return(D)
+}
+
+ReadRelperm<-function(PATH){
+	FILE=paste0(PATH,"/relperm.csv")
+	D<-read.csv(file=FILE,head=TRUE,sep=" ")
+	D$Case<-PATH
+
+	p<-ggplot(D)+
+	  geom_line(aes(sat.water,eff.perm.oil,color="oil"))+
+	  geom_line(aes(sat.water,eff.perm.water,color="water"))+
+	  xlab("Water saturation")+ylab("Effective permeability")+
+	  theme(panel.grid.major = element_blank(),panel.grid.minor = element_blank())+
+	  theme_bw()
+
+	FILE=paste0(PATH,"-relperm.png")
+ 	ggsave(FILE,p,height=4.0,width=6.0)
+
+	return(D)
+}
+
+ReadCase<-function(PATTERN){
+    list<-list.files(pattern=PATTERN)
+    Data=NULL
+    for (k in 1:length(list)){
+        print(list[k])
+        tmp=ReadRelperm(list[k])
+	tmp$Case<-list[k]
+        Data<-rbind(Data,tmp)
+    }
+    return(Data)
+}
+
+
+ReadMorphDrain<-function(PATH){
+	FILE=paste0(PATH,"/morphdrain.csv")
+	B<-read.csv(FILE,head=TRUE,sep=" ")
+	B$Media<-PATH
+
+	p<-ggplot()+
+	  geom_line(data=B,aes(Sw,2/R,colour=Media))+
+	  theme(panel.grid.major = element_blank(),panel.grid.minor = element_blank())+
+	  theme_bw()
+
+	FILE=paste0(PATH,"-morphdrain.png")
+ 	ggsave(FILE,p,height=4.0,width=6.0)
+	return(B)
+}
+

example/systems/huckleberry/lbpm-8p100.slurm

@@ -0,0 +1,42 @@
+#!/bin/bash
+#SBATCH -p normal_q
+#SBATCH -N 2
+#SBATCH -n 20
+#SBATCH -t 48:00:00
+#SBATCH --mem=500G
+#SBATCH --exclusive
+#SBATCH --gres=gpu:4
+#SBATCH --account=slurmtest
+
+echo "LBPM on huckleberry"
+
+module load gcc/7.3.0 openmpi/3.1.2 cuda 
+module load szip 
+module load zlib 
+module load hdf5/1.8.12 
+module load silo 
+#module load lbpm
+#module list
+
+export LBPM_BIN=$HOME/install/huckleberry/LBPM/bin
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+export OMPI_MCA_btl_smcuda_use_cuda_ipc=0
+export OMPI_MCA_btl_mpi_common_cuda_verbose=100
+export OMPI_MCA_pml=ob1
+export OMPI_MCA_mpi_warn_on_fork=0
+
+rm host.list
+for gpu in `seq 1 4`; do
+  scontrol show hostname $SLURM_NODELIST >> host.list
+done
+
+MPIARGS="-x OMPI_MCA_pml --mca mpi_common_cuda_verbose 100 --mca pml ob1 --mca btl_smcuda_use_cuda_ipc 0 --bind-to core --hostfile host.list"
+
+
+MPIARGS="--hostfile host.list --bind-to core --mca pml ob1 --mca btl vader,self,smcuda,openib  --mca btl_openib_warn_default_gid_prefix 0  --mca btl_smcuda_use_cuda_ipc_same_gpu 0  --mca btl_openib_want_cuda_gdr 0  --mca btl_openib_cuda_async_recv false --mca btl_smcuda_use_cuda_ipc 0 --mca btl_openib_allow_ib true --mca btl_openib_cuda_rdma_limit 1000 -x LD_LIBRARY_PATH"
+#mpirun -np 8 -mca pml ob1 -mca btl_smcuda_use_cuda_ipc 0 --bind-to core:overload-allowed $LBPM_BIN/lbpm_color_simulator input.db
+#mpirun -np 8 $MPIARGS  $LBPM_BIN/lbpm_morphdrain_pp input.db
+mpirun -np 8 $MPIARGS  $LBPM_BIN/lbpm_color_simulator input.db
+
+exit;

example/systems/huckleberry/preprocess.slurm

@@ -0,0 +1,34 @@
+#!/bin/bash
+#SBATCH -p normal_q
+#SBATCH -N 1
+#SBATCH -n 20
+#SBATCH -t 0:30:00
+#SBATCH --exclusive
+#SBATCH --gres=gpu:4
+#SBATCH --account=slurmtest
+
+echo "LBPM on huckleberry"
+
+module load gcc/7.3.0 openmpi/3.1.2 cuda 
+module load szip 
+module load zlib 
+module list
+module load hdf5/1.8.12 
+module load silo 
+#module load lbpm
+
+export LBPM_BIN=$HOME/install/huckleberry/LBPM/bin
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+
+export OMPI_MCA_btl_smcuda_use_cuda_ipc=0
+export OMPI_MCA_pml=ob1
+export OMPI_MCA_mpi_warn_on_fork=0
+
+#srun --exclusive -n 4 --cpu-bind=v,map_cpu=0,40,80,120 --accel-bind=g --gres=gpu:4 $LBPM_BIN/lbpm_color_simulator input.db
+#
+#
+
+mpirun -np 1 $LBPM_BIN/lbpm_serial_decomp input.db
+mpirun --oversubscribe --npernode 8 -mca pml ob1 -mca btl_smcuda_use_cuda_ipc 0 --bind-to core:overload-allowed  $LBPM_BIN/lbpm_morphdrain_pp input.db
+
+exit;

example/systems/summit/input.db

@@ -0,0 +1,53 @@
+B1;95;0cMRT {
+    tau = 1.0          // relaxation time
+    F = 0, 0, 1e-4     // external body force applied to system
+    timestepMax = 1000 // max number of timesteps
+    din = 1.0
+    dout = 1.0
+    Restart = false
+    flux = 0.0
+}
+
+Color {
+    tauA   = 0.7;     // relaxation time for fluid A
+    tauB   = 0.7;     // relaxation time for fluid B
+    rhoA   = 1.0;     // mass density for fluid A
+    rhoB   = 1.0;     // mass density for fluid B
+    alpha = 1e-3;     // controls interfacial tension between fluids
+    beta  = 0.95;     // controls interface width
+    F = 0, 0, 1.0e-5  // external body force applied to the system
+    Restart = false   // restart from checkpoint file?
+    din = 1.0         // density at inlet (if external BC is applied)
+    dout = 1.0                 // density at outlet (if external BC is applied )
+    timestepMax = 3000         // maximum number of timesteps to simulate
+    flux = 0.0                 // volumetric flux in voxels per timestep (if flux BC is applied)
+    ComponentLabels = 0        // comma separated list of solid mineral labels 
+    ComponentAffinity = -1.0   // comma separated list of phase indicato field value to assign for each mineral label
+}
+
+Domain {
+    nproc = 1, 1, 1        // Number of processors (Npx,Npy,Npz)
+    n = 400, 400, 400      // Size of local domain (Nx,Ny,Nz)
+    nspheres = 1896        // Number of spheres (only needed if using a sphere packing)
+    L = 1, 1, 1            // Length of domain (x,y,z)
+    BC = 0                 // Boundary condition type 
+                           //     BC = 0  for periodic BC
+                           //     BC = 1  for pressure BC (applied in z direction)
+                           //     BC = 4  for flux BC (applied in z direction
+    Filename = "mineralmodel.raw"                    // name of raw binary file to read digital image from 
+    ReadType = "8bit"
+    ReadValues = 0, 1, 2, 3, 4, 5, 6, 7, 8           // list of labels within the binary file (read)
+    WriteValues = 1, -1, -2, -2, -3, -3, -4, -4, -4  // list of labels within the output files (write)
+
+}
+
+Analysis {
+    blobid_interval = 1000      // Frequency to perform blob identification
+    analysis_interval = 1000    // Frequency to perform analysis
+    restart_interval = 2000    // Frequency to write restart data
+    visualization_interval = 2000        // Frequency to write visualization data
+    restart_file = "Restart"    // Filename to use for restart file (will append rank)
+    N_threads    = 4            // Number of threads to use
+    load_balance = "independent" // Load balance method to use: "none", "default", "independent"
+}
+

example/systems/summit/summit-spheres-1g.lsf

@@ -1,5 +1,5 @@
 #!/bin/bash
-#BSUB -P CSC275MCCLURE
+#BSUB -P CSC380
 #BSUB -J spheres
 #BSUB -o spheres.o%J
 #BSUB -W 10
@@ -10,10 +10,9 @@ date
 
 module load gcc cuda 
 
-export SCALBL_DIR=$HOME/summit/build/LBPM-WIA/tests
-
-jsrun -n1 -r1 -g1 -c1 -brs $SCALBL_DIR/GenerateSphereTest 1896
+export LBPM_DIR=/ccs/proj/csc380/mcclurej/install/LBPM/tests
 
+jsrun -n1 -r1 -g1 -c1 -brs $LBPM_DIR/GenerateSphereTest input.db
 
 
 # Create the 1200 GPU case
@@ -45,4 +44,3 @@ for i in `seq -w 0 $NRANKS`; do distfile="$BASEDIST$i"; echo $distfile; cp SignD
 
 
 exit;
-

gpu/Color.cu

@@ -143,7 +143,7 @@ __global__  void dvc_ScaLBL_Color_InitDistance(char *ID, double *Den, double *Ph
 
 __global__  void dvc_ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np)
 {
-	int idx,n,nm;
+	int idx,n;
 	// Fill the outlet with component b
 	idx = blockIdx.x*blockDim.x + threadIdx.x;
 	if (idx < count){
@@ -1255,6 +1255,15 @@ __global__  void dvc_ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny
 }
 
 
+__global__  void dvc_ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	double value;
+	int n =  blockIdx.x*blockDim.x + threadIdx.x;
+	if (n < Nx*Ny){
+		value = Phi[Source*Nx*Ny+n];
+		Phi[Dest*Nx*Ny+n] = value;
+	}
+}
+
 
 __global__  void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Velocity, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
@@ -1371,10 +1380,11 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *A
 
 			//...........Normalize the Color Gradient.................................
 			C = sqrt(nx*nx+ny*ny+nz*nz);
-			if (C==0.0) C=1.0;
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;		
+			double ColorMag = C;
+			if (C==0.0) ColorMag=1.0;
+			nx = nx/ColorMag;
+			ny = ny/ColorMag;
+			nz = nz/ColorMag;		
 
 			// q=0
 			fq = dist[n];
@@ -1650,7 +1660,7 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *A
 			//..............carry out relaxation process..............................
 			//..........Toelke, Fruediger et. al. 2006................................
 			if (C == 0.0)	nx = ny = nz = 0.0;
-			m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -alpha*C - m1);
+			m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -19*alpha*C - m1);
 			m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2);
 			m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4);
 			m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6);
@@ -1957,10 +1967,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double
 
 			//...........Normalize the Color Gradient.................................
 			C = sqrt(nx*nx+ny*ny+nz*nz);
-			if (C==0.0) C=1.0;
-			nx = nx/C;
-			ny = ny/C;
-			nz = nz/C;		
+			double ColorMag = C;
+			if (C==0.0) ColorMag=1.0;
+			nx = nx/ColorMag;
+			ny = ny/ColorMag;
+			nz = nz/ColorMag;		
 
 			// q=0
 			fq = dist[n];
@@ -2287,7 +2298,7 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double
 			//..............carry out relaxation process..............................
 			//..........Toelke, Fruediger et. al. 2006................................
 			if (C == 0.0)	nx = ny = nz = 0.0;
-			m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -alpha*C - m1);
+			m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) -19*alpha*C - m1);
 			m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0)- m2);
 			m4 = m4 + rlx_setB*((-0.6666666666666666*jx)- m4);
 			m6 = m6 + rlx_setB*((-0.6666666666666666*jy)- m6);
@@ -3484,13 +3495,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_ColorMass(double *Aq, double *Bq, double
 		double *Velocity, double *ColorGrad, double beta, int start, int finish, int Np){
 
 	int n;
-	double fq;
 	// non-conserved moments
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
 	double ux,uy,uz;
-	double phi,tau,rho0,rlx_setA,rlx_setB;
 
 	int S = Np/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -3578,13 +3587,11 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_ColorMass(int *neighborList, double *Aq,
 		double *Velocity, double *ColorGrad, double beta, int start, int finish, int Np){
 
 	int n,nread;
-	double fq;
 	// non-conserved moments
 	double nA,nB; // number density
 	double a1,b1,a2,b2,nAB,delta;
 	double C,nx,ny,nz; //color gradient magnitude and direction
 	double ux,uy,uz;
-	double phi,tau,rho0,rlx_setA,rlx_setB;
 
 	int S = Np/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -4151,5 +4158,9 @@ extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den,
 	}
 }
 
+extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Dest){
+	int GRID = Nx*Ny / 512 + 1;
+	dvc_ScaLBL_CopySlice_z<<<GRID,512>>>(Phi,Nx,Ny,Nz,Source,Dest);
+}
 
 

gpu/D3Q19.cu

@@ -104,7 +104,7 @@ __global__ void sum_kernel_block(double *sum, double *input, int n)
 __inline__ __device__
 double warpReduceSum(double val) {
 	for (int offset = warpSize/2; offset > 0; offset /= 2)
-		val += __shfl_down(val, offset);
+		val += __shfl_down_sync(0xFFFFFFFF, val, offset, 32);
 	return val;
 }
 
@@ -282,7 +282,6 @@ __global__ void dvc_ScaLBL_D3Q19_Init(double *dist, int Np)
 	}
 }
 
-
 //*************************************************************************
 __global__  void dvc_ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np, int q){
 	int n,nn;
@@ -1553,7 +1552,6 @@ __global__  void dvc_ScaLBL_D3Q19_Momentum(double *dist, double *vel, int N)
 	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
 	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
 	double vx,vy,vz;
-	char id;
 
 	int S = N/NBLOCKS/NTHREADS + 1;
 	for (int s=0; s<S; s++){
@@ -1744,6 +1742,43 @@ __global__  void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist,
 		//...................................................
 	}
 }
+__global__  void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f5 = 0.111111111111111111111111 - dist[6*Np+n];
+		double f11 = 0.05555555555555555555556 - dist[12*Np+n];
+		double f14 = 0.05555555555555555555556 - dist[13*Np+n];
+		double f15 = 0.05555555555555555555556 - dist[16*Np+n];
+		double f18 = 0.05555555555555555555556 - dist[17*Np+n];
+		
+		dist[6*Np+n] = f5;
+		dist[12*Np+n] = f11;
+		dist[13*Np+n] = f14;
+		dist[16*Np+n] = f15;
+		dist[17*Np+n] = f18;
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f6 = 0.111111111111111111111111 - dist[5*Np+n];
+		double f12 = 0.05555555555555555555556 - dist[11*Np+n];
+		double f13 = 0.05555555555555555555556 - dist[14*Np+n] ;
+		double f16 = 0.05555555555555555555556 - dist[15*Np+n];
+		double f17 = 0.05555555555555555555556 - dist[18*Np+n];
+		
+		dist[5*Np+n] = f6;
+		dist[11*Np+n] = f12;
+		dist[14*Np+n] = f13;
+		dist[15*Np+n] = f16;
+		dist[18*Np+n] = f17;
+	}
+}
 
 __global__  void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
 {
@@ -2340,10 +2375,11 @@ extern "C" void ScaLBL_D3Q19_Init(double *dist, int Np){
 	dvc_ScaLBL_D3Q19_Init<<<NBLOCKS,NTHREADS >>>(dist, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q19_AA_Init: %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q19_Init: %s \n",cudaGetErrorString(err));
 	}
 }
 
+
 extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz){
 	dvc_ScaLBL_D3Q19_Swap<<<NBLOCKS,NTHREADS >>>(ID, disteven, distodd, Nx, Ny, Nz);
 	cudaError_t err = cudaGetLastError();
@@ -2404,11 +2440,19 @@ extern "C" void ScaLBL_D3Q19_Pressure(double *fq, double *Pressure, int Np){
 extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,int Nx, int Ny, int Nz){
 	int GRID = Nx*Ny / 512 + 1;
 	dvc_D3Q19_Velocity_BC_z<<<GRID,512>>>(disteven,distodd, uz, Nx, Ny, Nz);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_z: %s \n",cudaGetErrorString(err));
+	}
 }
 
 extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz, int Nx, int Ny, int Nz, int outlet){
 	int GRID = Nx*Ny / 512 + 1;
 	dvc_D3Q19_Velocity_BC_Z<<<GRID,512>>>(disteven, distodd, uz, Nx, Ny, Nz, outlet);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Velocity_BC_Z: %s \n",cudaGetErrorString(err));
+	}
 }
 
 extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, double flux,int Nx, int Ny, int Nz){
@@ -2417,7 +2461,7 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
 
 	// IMPORTANT -- this routine may fail if Nx*Ny > 512*512
 	if (Nx*Ny > 512*512){
-		printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if Nx*Ny > 512*512");
+		printf("WARNING (ScaLBL_D3Q19_Flux_BC_z): CUDA reduction operation may fail if Nx*Ny > 512*512");
 	}
 
 	// Allocate memory to store the sums
@@ -2427,13 +2471,28 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
 	int sharedBytes = NTHREADS*sizeof(double);
 	cudaMalloc((void **)&dvcsum,sizeof(double)*Nx*Ny);
 	cudaMemset(dvcsum,0,sizeof(double)*Nx*Ny);
+	
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err));
+	}
 
 	// compute the local flux and store the result
 	dvc_D3Q19_Flux_BC_z<<<GRID,512,sharedBytes>>>(disteven, distodd, flux, dvcsum, Nx, Ny, Nz);
+	
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 1): %s \n",cudaGetErrorString(err));
+	}
 
 	// Now read the total flux
 	cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost);
 	din=sum[0];
+	
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Flux_BC_z (flux calculation, step 2): %s \n",cudaGetErrorString(err));
+	}
 
 	// free the memory needed for reduction
 	cudaFree(dvcsum);
@@ -2445,21 +2504,37 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, doub
 extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_z<<<GRID,512>>>(list, dist, din, count, N);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
 }
 
 extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z<<<GRID,512>>>(list, dist, dout, count, N);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
 }
 
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z<<<GRID,512>>>(neighborList, list, dist, din, count, N);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
 }
 
 extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_Z<<<GRID,512>>>(neighborList, list, dist, dout, count, N);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_Pressure_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
 }
 
 
@@ -2480,13 +2555,26 @@ extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double
 	cudaMalloc((void **)&dvcsum,sizeof(double)*count);
 	cudaMemset(dvcsum,0,sizeof(double)*count);
 	int sharedBytes = 512*sizeof(double);
+	
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err));
+	}
 
 	// compute the local flux and store the result
 	dvc_ScaLBL_D3Q19_AAeven_Flux_BC_z<<<GRID,512,sharedBytes>>>(list, dist, flux, area, dvcsum, count, N);
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
 
 	// Now read the total flux
 	cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost);
 	din=sum[0];
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAeven_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err));
+	}
 
 	// free the memory needed for reduction
 	cudaFree(dvcsum);
@@ -2501,7 +2589,7 @@ extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, dou
 
 	// IMPORTANT -- this routine may fail if Nx*Ny > 512*512
 	if (count > 512*512){
-		printf("WARNING (ScaLBL_D3Q19_Flux_BC_Z): CUDA reduction operation may fail if count > 512*512");
+		printf("WARNING (ScaLBL_D3Q19_AAodd_Flux_BC_z): CUDA reduction operation may fail if count > 512*512");
 	}
 
 	// Allocate memory to store the sums
@@ -2511,13 +2599,24 @@ extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, dou
 	cudaMalloc((void **)&dvcsum,sizeof(double)*count);
 	cudaMemset(dvcsum,0,sizeof(double)*count);
 	int sharedBytes = 512*sizeof(double);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (memory allocation): %s \n",cudaGetErrorString(err));
+	}
 
 	// compute the local flux and store the result
 	dvc_ScaLBL_D3Q19_AAodd_Flux_BC_z<<<GRID,512,sharedBytes>>>(neighborList, list, dist, flux, area, dvcsum, count, N);
-
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
 	// Now read the total flux
 	cudaMemcpy(&sum[0],dvcsum,sizeof(double),cudaMemcpyDeviceToHost);
 	din=sum[0];
+	err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_AAodd_Flux_BC_z (reduction): %s \n",cudaGetErrorString(err));
+	}
 
 	// free the memory needed for reduction
 	cudaFree(dvcsum);
@@ -2567,11 +2666,23 @@ extern "C" double deviceReduce(double *in, double* out, int N) {
 	return sum;
 }
 
-//
-//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np){
-//	int GRID = count / 512 + 1;
-//	dvc_ScaLBL_D3Q19_Pressure_BC_Z<<<GRID,512>>>(disteven, distodd, dout, Nx, Ny, Nz, outlet);
-//}
+extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q19_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q19_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
 
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
        double Fy, double Fz){

gpu/D3Q7.cu

@@ -14,6 +14,7 @@
   along with OPM.  If not, see <http://www.gnu.org/licenses/>.
 */
 // GPU Functions for D3Q7 Lattice Boltzmann Methods
+#include <stdio.h>
 
 #define NBLOCKS 560
 #define NTHREADS 128
@@ -94,6 +95,25 @@ __global__ void dvc_ScaLBL_D3Q7_Unpack(int q,  int *list,  int start, int count,
 	}
 }
 
+__global__  void dvc_ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f5 = 0.222222222222222222222222 - dist[6*Np+n];
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int idx, n;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		double f6 = 0.222222222222222222222222 - dist[5*Np+n];
+		dist[5*Np+n] = f6;
+	}
+}
 __global__ void dvc_ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
 {
 	int n,N;
@@ -222,6 +242,24 @@ __global__  void dvc_ScaLBL_D3Q7_Density(char *ID, double *disteven, double *dis
 	}
 }
 
+extern "C" void ScaLBL_D3Q7_Reflection_BC_z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Reflection_BC_Z(int *list, double *dist, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Reflection_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
 extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list,  int start, int count, double *recvbuf, double *dist, int N){
 	int GRID = count / 512 + 1;
 	dvc_ScaLBL_D3Q7_Unpack <<<GRID,512 >>>(q, list, start, count, recvbuf, dist, N);

gpu/D3Q7BC.cu

@@ -0,0 +1,537 @@
+#include <math.h>
+#include <stdio.h>
+#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+
+__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
+{
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
+	}
+}
+
+__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
+{
+
+    int idx;
+    int iq,ib;
+    double value_b,value_q;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		iq = BounceBackDist_list[idx];
+        ib = BounceBackSolid_list[idx];
+		value_b = BoundaryValue[ib];//get boundary value from a solid site
+        value_q = dist[iq];
+		dist[iq] = value_q + value_b;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+		//...................................................
+		f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		//...................................................
+		f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np)
+{
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		f5 = Vin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np)
+{
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		f6 = Vout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
+{
+	int idx,n,nm;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vin;
+	}
+}
+
+
+__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
+{
+	int idx,n,nm;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		nm = Map[n];
+		Psi[nm] = Vout;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+		//...................................................
+		f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np)
+{
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		//...................................................
+		f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np)
+{
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		f5 = Cin - (f0+f1+f2+f3+f4+f6);
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np)
+{
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		f6 = Cout - (f0+f1+f2+f3+f4+f5);
+		dist[nr6] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
+//*************************************************************************
+
+extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_Solid_Dirichlet_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_Solid_Neumann_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<<GRID,512>>>(list, dist, Vin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<<GRID,512>>>(list, dist, Vout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Vin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Vout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){
+	int GRID = count / 512 + 1;
+    dvc_ScaLBL_Poisson_D3Q7_BC_z<<<GRID,512>>>(list, Map, Psi, Vin, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){
+	int GRID = count / 512 + 1;
+    dvc_ScaLBL_Poisson_D3Q7_BC_Z<<<GRID,512>>>(list, Map, Psi, Vout, count);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<<GRID,512>>>(list, dist, Cin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<<GRID,512>>>(list, dist, Cout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Cin, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Cout, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}

gpu/Extras.cu

@@ -24,6 +24,7 @@ extern "C" int ScaLBL_SetDevice(int rank){
 	//int device = local_rank % n_devices; 
 	int device = rank % n_devices; 
 	cudaSetDevice(device); 
+	if (rank < n_devices) printf("MPI rank=%i will use GPU ID %i / %i \n",rank,device,n_devices);
 	return device;
 }
 

gpu/Ion.cu

@@ -0,0 +1,391 @@
+#include <stdio.h>
+#include <math.h>
+//#include <cuda_profiler_api.h>
+
+#define NBLOCKS 1024
+#define NTHREADS 256
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+    int n,nread;
+    double fq,Ci;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            Ci = fq;
+
+            // q=1
+            nread = neighborList[n]; 
+            fq = dist[nread]; 
+            Ci += fq;
+            
+            // q=2
+            nread = neighborList[n+Np]; 
+            fq = dist[nread];  
+            Ci += fq;
+            
+            // q=3
+            nread = neighborList[n+2*Np]; 
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=4
+            nread = neighborList[n+3*Np]; 
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=5
+            nread = neighborList[n+4*Np];
+            fq = dist[nread];
+            Ci += fq;
+            
+            // q=6
+            nread = neighborList[n+5*Np];
+            fq = dist[nread];
+            Ci += fq;
+
+            Den[n]=Ci;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+    int n;
+    double fq,Ci;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            // q=0
+            fq = dist[n];
+            Ci = fq;
+            
+            // q=1
+            fq = dist[2*Np+n];
+            Ci += fq;
+
+            // q=2
+            fq = dist[1*Np+n];
+            Ci += fq;
+
+            // q=3
+            fq = dist[4*Np+n];
+            Ci += fq;
+
+            // q=4
+            fq = dist[3*Np+n];
+            Ci += fq;
+
+            // q=5
+            fq = dist[6*Np+n];
+            Ci += fq;
+
+            // q=6
+            fq = dist[5*Np+n];
+            Ci += fq;
+
+            Den[n]=Ci;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                           double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            Ci=Den[n];
+            Ex=ElectricField[n+0*Np];
+            Ey=ElectricField[n+1*Np];
+            Ez=ElectricField[n+2*Np];
+            ux=Velocity[n+0*Np];
+            uy=Velocity[n+1*Np];
+            uz=Velocity[n+2*Np];
+            uEPx=zi*Di/Vt*Ex;
+            uEPy=zi*Di/Vt*Ey;
+            uEPz=zi*Di/Vt*Ez;
+
+            // q=0
+            f0 = dist[n];
+            // q=1
+            nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+            f1 = dist[nr1]; // reading the f1 data into register fq
+            // q=2
+            nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+            f2 = dist[nr2];  // reading the f2 data into register fq
+            // q=3
+            nr3 = neighborList[n+2*Np]; // neighbor 4
+            f3 = dist[nr3];
+            // q=4
+            nr4 = neighborList[n+3*Np]; // neighbor 3
+            f4 = dist[nr4];
+            // q=5
+            nr5 = neighborList[n+4*Np];
+            f5 = dist[nr5];
+            // q=6
+            nr6 = neighborList[n+5*Np];
+            f6 = dist[nr6];
+            
+            // q=0
+            dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+            //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 1
+            dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+            //dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q=2
+            dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+            //dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 3
+            dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+            //dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 4
+            dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+            //dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 5
+            dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+            //dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 6
+            dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+            //dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                            double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	int n;
+	double Ci;
+    double ux,uy,uz;
+    double uEPx,uEPy,uEPz;//electrochemical induced velocity
+    double Ex,Ey,Ez;//electrical field
+	double f0,f1,f2,f3,f4,f5,f6;
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+
+            //Load data
+            Ci=Den[n];
+            Ex=ElectricField[n+0*Np];
+            Ey=ElectricField[n+1*Np];
+            Ez=ElectricField[n+2*Np];
+            ux=Velocity[n+0*Np];
+            uy=Velocity[n+1*Np];
+            uz=Velocity[n+2*Np];
+            uEPx=zi*Di/Vt*Ex;
+            uEPy=zi*Di/Vt*Ey;
+            uEPz=zi*Di/Vt*Ez;
+
+            f0 = dist[n];
+            f1 = dist[2*Np+n];
+            f2 = dist[1*Np+n];
+            f3 = dist[4*Np+n];
+            f4 = dist[3*Np+n];
+            f5 = dist[6*Np+n];
+            f6 = dist[5*Np+n];
+            
+            // q=0
+            dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
+            //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 1
+            dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
+            //dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q=2
+            dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
+            //dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 3
+            dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
+            //dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 4
+            dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
+            //dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 5
+            dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
+            //dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+
+            // q = 6
+            dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
+            //dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
+
+	int n;
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
+		if (n<Np) {
+            dist[0*Np+n] = 0.25*DenInit;
+            dist[1*Np+n] = 0.125*DenInit;		
+            dist[2*Np+n] = 0.125*DenInit;	
+            dist[3*Np+n] = 0.125*DenInit;	
+            dist[4*Np+n] = 0.125*DenInit;	
+            dist[5*Np+n] = 0.125*DenInit;	
+            dist[6*Np+n] = 0.125*DenInit;	
+            Den[n] = DenInit;
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
+
+	int n;
+    double DenInit;
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
+		if (n<Np) {
+            DenInit = Den[n];
+            dist[0*Np+n] = 0.25*DenInit;
+            dist[1*Np+n] = 0.125*DenInit;		
+            dist[2*Np+n] = 0.125*DenInit;	
+            dist[3*Np+n] = 0.125*DenInit;	
+            dist[4*Np+n] = 0.125*DenInit;	
+            dist[5*Np+n] = 0.125*DenInit;	
+            dist[6*Np+n] = 0.125*DenInit;	
+		}
+	}
+}
+
+__global__  void dvc_ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+
+    int n;
+    double Ci;//ion concentration of species i
+    double CD;//charge density
+    double CD_tmp;
+    double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
+
+	int S = Np/NBLOCKS/NTHREADS + 1;
+	for (int s=0; s<S; s++){
+		//........Get 1-D index for this thread....................
+		n =  S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
+		if (n<finish) {
+            Ci = Den[n+ion_component*Np];
+            CD = ChargeDensity[n];
+            CD_tmp = F*IonValence*Ci;
+            ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
+		}
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<<NBLOCKS,NTHREADS >>>(dist,Den,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,  
+                                      double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_Init<<<NBLOCKS,NTHREADS >>>(dist,Den,DenInit,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_Init: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_Init_FromFile<<<NBLOCKS,NTHREADS >>>(dist,Den,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}
+
+extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
+
+	//cudaProfilerStart();
+	dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<<NBLOCKS,NTHREADS >>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np);
+
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",cudaGetErrorString(err));
+	}
+	//cudaProfilerStop();
+}