Merging master

CMakeLists.txt

@@ -90,7 +90,7 @@ CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
 CHECK_ENABLE_FLAG( USE_LATEX 1 )
 FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
 IF ( USE_DOXYGEN )
-    SET( DOXYFILE_LATEX YES )
+    SET( DOXYFILE_LATEX NO )
     SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
     SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
     SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )

IO/HDF5Writer.cpp

@@ -0,0 +1,283 @@
+#include "IO/HDF5_IO.h"
+#include "IO/IOHelpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Writer.h"
+#include "IO/Xdmf.h"
+#include "common/MPI.h"
+#include "common/Utilities.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <sys/stat.h>
+#include <vector>
+
+
+#ifdef USE_HDF5
+
+
+std::string to_string( const ArraySize &s )
+{
+    std::string out = "[" + std::to_string( s[0] );
+    for ( size_t i = 1; i < s.ndim(); i++ )
+        out += "," + to_string( s[i] );
+    out += "]";
+    return out;
+}
+
+
+Xdmf::Center getXdmfType( IO::VariableType type )
+{
+    if ( type == IO::VariableType::NodeVariable ) {
+        return Xdmf::Center::Node;
+    } else if ( type == IO::VariableType::VolumeVariable ) {
+        return Xdmf::Center::Cell;
+    } else {
+        ERROR( "Variable type not supported" );
+    }
+    return Xdmf::Center::Null;
+}
+
+
+// Write a PointList mesh (and variables) to a file
+template<class TYPE>
+static void writeCoordinates( hid_t fid, const std::vector<Point> &points )
+{
+    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
+    for ( size_t i = 0; i < x.size(); i++ ) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    IO::HDF5::writeHDF5( fid, "x", x );
+    IO::HDF5::writeHDF5( fid, "y", y );
+    IO::HDF5::writeHDF5( fid, "z", z );
+}
+static void writeHDF5PointList( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &meshData, IO::MeshDatabase database, Xdmf &xmf )
+{
+    auto meshname    = database.domains[0].name;
+    const auto &mesh = dynamic_cast<IO::PointList &>( *meshData.mesh );
+    auto gid         = IO::HDF5::createGroup( fid, meshname );
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeCoordinates<double>( gid, mesh.getPoints() );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeCoordinates<float>( gid, mesh.getPoints() );
+    } else {
+        ERROR( "Unsupported format" );
+    }
+    auto path   = filename + ":/" + meshname + "/";
+    auto domain = Xdmf::createPointMesh(
+        meshname, 3, mesh.getPoints().size(), path + "x", path + "y", path + "z" );
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        auto &var     = *meshData.vars[i];
+        auto data     = var.data;
+        auto rankType = Xdmf::RankType::Null;
+        if ( data.ndim() == 1 ) {
+            rankType = Xdmf::RankType::Scalar;
+        } else if ( data.ndim() == 2 && data.size( 1 ) == 3 ) {
+            // Vector data, need to permute for visit
+            rankType = Xdmf::RankType::Vector;
+            data     = data.permute( { 1, 0 } );
+        } else {
+            ERROR( "Unable to determine variable rank: " + to_string( var.data.size() ) );
+        }
+        if ( var.precision == IO::DataType::Double ) {
+            IO::HDF5::writeHDF5( gid, var.name, data );
+        } else if ( var.precision == IO::DataType::Float ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<float>() );
+        } else if ( var.precision == IO::DataType::Int ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<int>() );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+        domain.addVariable(
+            meshname, var.name, data.size(), rankType, Xdmf::Center::Node, path + var.name );
+    }
+    xmf.addMesh( meshData.meshName, domain );
+}
+// Write a TriMesh mesh (and variables) to a file
+static void writeHDF5TriMesh2( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &meshData, const IO::TriMesh &mesh, IO::MeshDatabase database,
+    Xdmf &xmf )
+{
+    auto meshname = database.domains[0].name;
+    auto gid      = IO::HDF5::createGroup( fid, meshname );
+    auto path     = filename + ":/" + meshname + "/";
+    // Write the verticies
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeCoordinates<double>( gid, mesh.vertices->getPoints() );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeCoordinates<float>( gid, mesh.vertices->getPoints() );
+    } else {
+        ERROR( "Unsupported format" );
+    }
+    // Write the connectivity
+    Array<int> tri( 3, mesh.A.size() );
+    for ( size_t i = 0; i < mesh.A.size(); i++ ) {
+        tri( 0, i ) = mesh.A[i];
+        tri( 1, i ) = mesh.B[i];
+        tri( 2, i ) = mesh.C[i];
+    }
+    IO::HDF5::writeHDF5( gid, "tri", tri );
+    auto domain =
+        Xdmf::createUnstructuredMesh( meshname, 3, Xdmf::TopologyType::Triangle, tri.size( 1 ),
+            path + "tri", mesh.vertices->getPoints().size(), path + "x", path + "y", path + "z" );
+    // Write the variables
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        auto &var     = *meshData.vars[i];
+        auto data     = var.data;
+        auto rankType = Xdmf::RankType::Null;
+        if ( data.ndim() == 1 ) {
+            rankType = Xdmf::RankType::Scalar;
+        } else if ( data.ndim() == 2 && data.size( 1 ) == 3 ) {
+            // Vector data, need to permute for visit
+            rankType = Xdmf::RankType::Vector;
+            data     = data.permute( { 1, 0 } );
+        } else {
+            ERROR( "Unable to determine variable rank: " + to_string( var.data.size() ) );
+        }
+        if ( var.precision == IO::DataType::Double ) {
+            IO::HDF5::writeHDF5( gid, var.name, data );
+        } else if ( var.precision == IO::DataType::Float ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<float>() );
+        } else if ( var.precision == IO::DataType::Int ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<int>() );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+        domain.addVariable(
+            meshname, var.name, data.size(), rankType, getXdmfType( var.type ), path + var.name );
+    }
+    xmf.addMesh( meshData.meshName, domain );
+}
+static void writeHDF5TriMesh( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &meshData, IO::MeshDatabase database, Xdmf &xmf )
+{
+    const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
+    writeHDF5TriMesh2( fid, filename, meshData, mesh, database, xmf );
+}
+static void writeHDF5TriList( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &meshData, IO::MeshDatabase database, Xdmf &xmf )
+{
+    auto mesh = getTriMesh( meshData.mesh );
+    writeHDF5TriMesh2( fid, filename, meshData, *mesh, database, xmf );
+}
+// Write a DomainMesh mesh (and variables) to a file
+static void writeHDF5DomainMesh( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &meshData, IO::MeshDatabase database, Xdmf &xmf )
+{
+    auto &mesh    = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
+    auto meshname = database.domains[0].name;
+    auto gid      = IO::HDF5::createGroup( fid, meshname );
+    auto path     = filename + ":/" + meshname + "/";
+    // Write the mesh
+    RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
+    std::vector<double> range = { info.ix * mesh.Lx / info.nx, ( info.ix + 1 ) * mesh.Lx / info.nx,
+        info.jy * mesh.Ly / info.ny, ( info.jy + 1 ) * mesh.Ly / info.ny,
+        info.kz * mesh.Lz / info.nz, ( info.kz + 1 ) * mesh.Lz / info.nz };
+    std::vector<int> N        = { mesh.nx, mesh.ny, mesh.nz };
+    std::vector<int> rankinfo = { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz };
+    IO::HDF5::writeHDF5( gid, "range", range );
+    IO::HDF5::writeHDF5( gid, "N", N );
+    IO::HDF5::writeHDF5( gid, "rankinfo", rankinfo );
+    // xmf.addUniformMesh( meshname, range, ArraySize( N[0], N[1], N[2] ) );
+    // Write a curvilinear mesh due to bug with vector data on nodes loading into visit
+    Array<float> x( N[0] + 1, N[1] + 1, N[2] + 1 );
+    Array<float> y( N[0] + 1, N[1] + 1, N[2] + 1 );
+    Array<float> z( N[0] + 1, N[1] + 1, N[2] + 1 );
+    double dx = ( range[1] - range[0] ) / N[0];
+    double dy = ( range[3] - range[2] ) / N[1];
+    double dz = ( range[5] - range[4] ) / N[2];
+    for ( int k = 0; k <= N[2]; k++ ) {
+        for ( int j = 0; j <= N[1]; j++ ) {
+            for ( int i = 0; i <= N[0]; i++ ) {
+                x( i, j, k ) = range[0] + dx * i;
+                y( i, j, k ) = range[2] + dy * j;
+                z( i, j, k ) = range[4] + dz * k;
+            }
+        }
+    }
+    IO::HDF5::writeHDF5( gid, "x", x );
+    IO::HDF5::writeHDF5( gid, "y", y );
+    IO::HDF5::writeHDF5( gid, "z", z );
+    auto domain = Xdmf::createCurvilinearMesh(
+        meshname, ArraySize( N[0], N[1], N[2] ), path + "x", path + "y", path + "z" );
+    // Write the variables
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        auto &var     = *meshData.vars[i];
+        auto data     = var.data;
+        auto rankType = Xdmf::RankType::Null;
+        if ( data.ndim() == 3 ) {
+            rankType = Xdmf::RankType::Scalar;
+        } else if ( data.ndim() == 4 && data.size( 3 ) == 3 ) {
+            // Vector data, need to permute for visit
+            rankType = Xdmf::RankType::Vector;
+            data     = data.permute( { 3, 0, 1, 2 } );
+        } else {
+            ERROR( "Unable to determine variable rank: " + to_string( var.data.size() ) );
+        }
+        if ( var.precision == IO::DataType::Double ) {
+            IO::HDF5::writeHDF5( gid, var.name, data );
+        } else if ( var.precision == IO::DataType::Float ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<float>() );
+        } else if ( var.precision == IO::DataType::Int ) {
+            IO::HDF5::writeHDF5( gid, var.name, data.cloneTo<int>() );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+        domain.addVariable(
+            meshname, var.name, data.size(), rankType, getXdmfType( var.type ), path + var.name );
+    }
+    IO::HDF5::closeGroup( gid );
+    xmf.addMesh( meshData.meshName, domain );
+}
+// Write a mesh (and variables) to a file
+static IO::MeshDatabase write_domain_hdf5( hid_t fid, const std::string &filename,
+    const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank, Xdmf &xmf )
+{
+    // Create the MeshDatabase
+    auto database = getDatabase( filename, mesh, format, rank );
+    if ( database.meshClass == "PointList" ) {
+        writeHDF5PointList( fid, filename, mesh, database, xmf );
+    } else if ( database.meshClass == "TriMesh" ) {
+        writeHDF5TriMesh( fid, filename, mesh, database, xmf );
+    } else if ( database.meshClass == "TriList" ) {
+        writeHDF5TriList( fid, filename, mesh, database, xmf );
+    } else if ( database.meshClass == "DomainMesh" ) {
+        writeHDF5DomainMesh( fid, filename, mesh, database, xmf );
+    } else {
+        ERROR( "Unknown mesh class" );
+    }
+    return database;
+}
+// Write the mesh data to hdf5
+std::vector<IO::MeshDatabase> writeMeshesHDF5( const std::vector<IO::MeshDataStruct> &meshData,
+    const std::string &path, IO::FileFormat format, int rank, Xdmf &xmf )
+{
+
+    std::vector<IO::MeshDatabase> meshes_written;
+    char filename[100], fullpath[200];
+    sprintf( filename, "%05i.h5", rank );
+    sprintf( fullpath, "%s/%s", path.c_str(), filename );
+    auto fid = IO::HDF5::openHDF5( fullpath, "w", IO::HDF5::Compression::GZIP );
+    for ( size_t i = 0; i < meshData.size(); i++ ) {
+        meshes_written.push_back(
+            write_domain_hdf5( fid, filename, meshData[i], format, rank, xmf ) );
+    }
+    IO::HDF5::closeHDF5( fid );
+    return meshes_written;
+}
+
+
+#else
+
+
+std::vector<IO::MeshDatabase> writeMeshesHDF5(
+    const std::vector<IO::MeshDataStruct> &, const std::string &, IO::FileFormat, int );
+{
+    return std::vector<IO::MeshDatabase>();
+}
+
+
+#endif

IO/HDF5_IO.cpp

@@ -0,0 +1,585 @@
+#include "IO/HDF5_IO.h"
+#include "IO/HDF5_IO.hpp"
+#include "common/Array.h"
+#include "common/Utilities.h"
+
+#include <complex>
+#include <sstream>
+#include <string>
+#include <vector>
+
+
+namespace IO {
+namespace HDF5 {
+
+
+#ifdef USE_HDF5 // USE HDF5
+
+
+/************************************************************************
+ * HDF5 helper routines                                                  *
+ ************************************************************************/
+inline const void *H5Ptr( const void *x ) { return x == nullptr ? ( (void *) 1 ) : x; }
+bool H5Gexists( hid_t fid, const std::string &name )
+{
+    H5E_auto2_t func;
+    void *client;
+    H5Eget_auto2( H5E_DEFAULT, &func, &client );
+    H5Eset_auto2( H5E_DEFAULT, nullptr, nullptr );
+    int status = H5Gget_objinfo( fid, name.data(), 0, nullptr );
+    H5Eset_auto2( H5E_DEFAULT, func, client );
+    return status == 0;
+}
+bool H5Dexists( hid_t fid, const std::string &name )
+{
+    H5E_auto2_t func;
+    void *client;
+    H5Eget_auto2( H5E_DEFAULT, &func, &client );
+    H5Eset_auto2( H5E_DEFAULT, nullptr, nullptr );
+    hid_t dataset = H5Dopen2( fid, name.data(), H5P_DEFAULT );
+    H5Eset_auto2( H5E_DEFAULT, func, client );
+    bool exists = dataset > 0;
+    // if ( exists )
+    //    H5Dclose( dataset );
+    return exists;
+}
+hid_t createGroup( hid_t fid, const std::string &name )
+{
+    return H5Gcreate2( fid, name.data(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
+}
+hid_t openGroup( hid_t fid, const std::string &name )
+{
+    INSIST( H5Gexists( fid, name ), "Group " + name + " does not exist" );
+    return H5Gopen2( fid, name.data(), H5P_DEFAULT );
+}
+void closeGroup( hid_t gid ) { H5Gclose( gid ); }
+
+
+/************************************************************************
+ * Complex struct that is compatible with HDF5                           *
+ ************************************************************************/
+typedef struct {
+    double re;
+    double im;
+} complex_t;
+inline void convert( size_t N, const std::complex<double> *x, complex_t *y )
+{
+    for ( size_t i = 0; i < N; i++ ) {
+        y[i].re = x[i].real();
+        y[i].im = x[i].imag();
+    }
+}
+inline void convert( size_t N, const complex_t *x, std::complex<double> *y )
+{
+    for ( size_t i = 0; i < N; i++ ) {
+        y[i] = std::complex<double>( x[i].re, x[i].im );
+    }
+}
+
+
+/************************************************************************
+ * Get the HDF5 data type                                                *
+ ************************************************************************/
+template<>
+hid_t getHDF5datatype<bool>()
+{
+    return H5Tcopy( H5T_NATIVE_UCHAR );
+}
+template<>
+hid_t getHDF5datatype<char>()
+{
+    return H5Tcopy( H5T_NATIVE_CHAR );
+}
+template<>
+hid_t getHDF5datatype<uint8_t>()
+{
+    return H5Tcopy( H5T_NATIVE_UINT8 );
+}
+template<>
+hid_t getHDF5datatype<int8_t>()
+{
+    return H5Tcopy( H5T_NATIVE_INT8 );
+}
+template<>
+hid_t getHDF5datatype<uint16_t>()
+{
+    return H5Tcopy( H5T_NATIVE_UINT16 );
+}
+template<>
+hid_t getHDF5datatype<int16_t>()
+{
+    return H5Tcopy( H5T_NATIVE_INT16 );
+}
+template<>
+hid_t getHDF5datatype<int>()
+{
+    return H5Tcopy( H5T_NATIVE_INT );
+}
+template<>
+hid_t getHDF5datatype<unsigned int>()
+{
+    return H5Tcopy( H5T_NATIVE_UINT );
+}
+template<>
+hid_t getHDF5datatype<long int>()
+{
+    return H5Tcopy( H5T_NATIVE_LONG );
+}
+template<>
+hid_t getHDF5datatype<unsigned long int>()
+{
+    return H5Tcopy( H5T_NATIVE_ULONG );
+}
+template<>
+hid_t getHDF5datatype<float>()
+{
+    return H5Tcopy( H5T_NATIVE_FLOAT );
+}
+template<>
+hid_t getHDF5datatype<double>()
+{
+    return H5Tcopy( H5T_NATIVE_DOUBLE );
+}
+template<>
+hid_t getHDF5datatype<std::complex<double>>()
+{
+    hid_t datatype = H5Tcreate( H5T_COMPOUND, sizeof( complex_t ) );
+    H5Tinsert( datatype, "real", HOFFSET( complex_t, re ), H5T_NATIVE_DOUBLE );
+    H5Tinsert( datatype, "imag", HOFFSET( complex_t, im ), H5T_NATIVE_DOUBLE );
+    return datatype;
+}
+template<>
+hid_t getHDF5datatype<char *>()
+{
+    hid_t datatype = H5Tcopy( H5T_C_S1 );
+    H5Tset_size( datatype, H5T_VARIABLE );
+    return datatype;
+}
+
+
+/************************************************************************
+ * Read/write Array types                                                *
+ ************************************************************************/
+template<>
+void readHDF5<Array<std::string>>( hid_t fid, const std::string &name, Array<std::string> &data )
+{
+    if ( !H5Dexists( fid, name ) ) {
+        // Dataset does not exist
+        data.resize( 0 );
+        return;
+    }
+    hid_t dataset   = H5Dopen2( fid, name.data(), H5P_DEFAULT );
+    hid_t datatype  = H5Dget_type( dataset );
+    hid_t dataspace = H5Dget_space( dataset );
+    hsize_t dims0[10];
+    int ndim  = H5Sget_simple_extent_dims( dataspace, dims0, nullptr );
+    auto dims = convertSize( ndim, dims0 );
+    data.resize( dims );
+    hid_t datatype2 = getHDF5datatype<char *>();
+    if ( data.empty() ) {
+        // The data is empty
+    } else if ( H5Tequal( datatype, datatype2 ) ) {
+        // The type of Array and the data in HDF5 match
+        auto **tmp = new char *[data.length() * sizeof( char * )];
+        memset( tmp, 0, data.length() * sizeof( char * ) );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, tmp );
+        for ( size_t i = 0; i < data.length(); i++ )
+            data( i ) = std::string( tmp[i] );
+        H5Dvlen_reclaim( datatype, dataspace, H5P_DEFAULT, tmp );
+        delete[] tmp;
+    } else {
+        ERROR( "Unknown format for std::string" );
+    }
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Tclose( datatype2 );
+    H5Sclose( dataspace );
+}
+template<>
+void readHDF5<Array<std::complex<double>>>(
+    hid_t fid, const std::string &name, Array<std::complex<double>> &data )
+{
+    if ( !H5Dexists( fid, name ) ) {
+        // Dataset does not exist
+        data.resize( 0 );
+        return;
+    }
+    hid_t dataset   = H5Dopen2( fid, name.data(), H5P_DEFAULT );
+    hid_t datatype  = H5Dget_type( dataset );
+    hid_t dataspace = H5Dget_space( dataset );
+    hsize_t dims0[10];
+    int ndim  = H5Sget_simple_extent_dims( dataspace, dims0, nullptr );
+    auto dims = convertSize( ndim, dims0 );
+    data.resize( dims );
+    hid_t datatype2 = getHDF5datatype<std::complex<double>>();
+    if ( data.empty() ) {
+        // The data is empty
+    } else if ( H5Tequal( datatype, datatype2 ) ) {
+        // The type of Array and the data in HDF5 match
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data() );
+    } else {
+        ERROR( "We need to convert data formats" );
+    }
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Tclose( datatype2 );
+    H5Sclose( dataspace );
+}
+// clang-format off
+#define readWriteHDF5Array( TYPE )                                                          \
+    template<>                                                                              \
+    void writeHDF5<Array<TYPE>>( hid_t fid, const std::string &name, const Array<TYPE> &data ) \
+    {                                                                                       \
+        writeHDF5ArrayDefault<TYPE>( fid, name, data );                                     \
+    }                                                                                       \
+    template<>                                                                              \
+    void readHDF5<Array<TYPE>>( hid_t fid, const std::string &name, Array<TYPE> &data ) \
+    {                                                                                       \
+        readHDF5ArrayDefault<TYPE>( fid, name, data );                                      \
+    }
+readWriteHDF5Array( bool )
+readWriteHDF5Array( char )
+readWriteHDF5Array( int8_t )
+readWriteHDF5Array( int16_t )
+readWriteHDF5Array( int32_t )
+readWriteHDF5Array( int64_t )
+readWriteHDF5Array( uint8_t )
+readWriteHDF5Array( uint16_t )
+readWriteHDF5Array( uint32_t )
+readWriteHDF5Array( uint64_t )
+readWriteHDF5Array( float )
+readWriteHDF5Array( double )
+    // clang-format on
+
+
+    /************************************************************************
+     * Read/write scalar types                                               *
+     ************************************************************************/
+    template<>
+    void readHDF5<std::string>( hid_t fid, const std::string &name, std::string &data )
+{
+    hid_t dataset   = H5Dopen2( fid, name.data(), H5P_DEFAULT );
+    hid_t datatype  = H5Dget_type( dataset );
+    hid_t datatype0 = getHDF5datatype<char *>();
+    if ( H5Tequal( datatype, datatype0 ) ) {
+        hid_t dataspace = H5Dget_space( dataset );
+        char *tmp[1]    = { nullptr };
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, tmp );
+        data = std::string( tmp[0] );
+        H5Dvlen_reclaim( datatype, dataspace, H5P_DEFAULT, tmp );
+        H5Sclose( dataspace );
+    } else {
+        Array<char> tmp;
+        readHDF5( fid, name, tmp );
+        data = std::string( tmp.data(), tmp.length() );
+    }
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Tclose( datatype0 );
+}
+template<>
+void writeHDF5<std::string>( hid_t fid, const std::string &name, const std::string &data )
+{
+    Array<char> tmp;
+    tmp.viewRaw( { data.length() }, (char *) data.data() );
+    writeHDF5( fid, name, tmp );
+}
+// clang-format off
+#define readWriteHDF5Scalar( TYPE )                                                         \
+    template<>                                                                              \
+    void writeHDF5<TYPE>( hid_t fid, const std::string &name, const TYPE &data )       \
+    {                                                                                       \
+        hid_t dataspace = H5Screate( H5S_SCALAR );                                          \
+        hid_t datatype  = getHDF5datatype<TYPE>();                                          \
+        hid_t dataset   = H5Dcreate2(                                                       \
+            fid, name.data(), datatype, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT ); \
+        H5Dwrite( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, H5Ptr( &data ) );       \
+        H5Dclose( dataset );                                                                \
+        H5Tclose( datatype );                                                               \
+        H5Sclose( dataspace );                                                              \
+    }                                                                                       \
+    template<>                                                                              \
+    void readHDF5<TYPE>( hid_t fid, const std::string &name, TYPE &data )              \
+    {                                                                                       \
+        Array<TYPE> tmp;                                                                    \
+        readHDF5( fid, name, tmp );                                                         \
+        INSIST( tmp.ndim() == 1 && tmp.length() == 1, "Error loading " + std::string( name ) ); \
+        data = tmp( 0 );                                                                    \
+    }
+readWriteHDF5Scalar( bool )
+readWriteHDF5Scalar( char )
+readWriteHDF5Scalar( int8_t )
+readWriteHDF5Scalar( int16_t )
+readWriteHDF5Scalar( int32_t )
+readWriteHDF5Scalar( int64_t )
+readWriteHDF5Scalar( uint8_t )
+readWriteHDF5Scalar( uint16_t )
+readWriteHDF5Scalar( uint32_t )
+readWriteHDF5Scalar( uint64_t )
+readWriteHDF5Scalar( float )
+readWriteHDF5Scalar( double )
+readWriteHDF5Scalar( std::complex<double> )
+    // clang-format on
+
+
+    /******************************************************************
+     * Create custom error handler                                     *
+     ******************************************************************/
+    herr_t hdf5_error_handler( hid_t err_stack, void * )
+{
+    FILE *fid = tmpfile();
+    H5Eprint2( err_stack, fid );
+    H5Eclear2( err_stack );
+    rewind( fid );
+    char msg[1024];
+    size_t N = fread( msg, 1, sizeof( msg ) - 1, fid );
+    fclose( fid );
+    msg[N]           = 0;
+    std::string msg2 = "Error calling HDF5 routine:\n";
+    ERROR( msg2 + msg );
+    return 0;
+}
+bool set_hdf5_error_handler()
+{
+    hid_t error_stack = 0;
+    H5E_auto2_t fun   = hdf5_error_handler;
+    H5Eset_auto2( error_stack, fun, nullptr );
+    return true;
+}
+bool global_is_hdf5_error_handler_set = set_hdf5_error_handler();
+
+
+/******************************************************************
+ * Open/close HDF5 files                                           *
+ ******************************************************************/
+hid_t openHDF5( const std::string &filename, const char *mode, Compression compress )
+{
+    // Set cache size to 3MBs and instruct the cache to discard the fully read chunk
+    auto pid = H5P_DEFAULT;
+    /*auto pid = H5Pcreate( H5P_FILE_ACCESS );
+    int nelemts;
+    size_t nslots, nbytes;
+    double w0;
+    H5Pget_cache(pid,& nelemts,& nslots,& nbytes,& w0);
+    H5Pset_cache(pid, nelemts, 1999, 3*1024*1024, 1.0); */
+    // Open the file
+    hid_t fid = 0;
+    if ( strcmp( mode, "r" ) == 0 ) {
+        fid = H5Fopen( filename.data(), H5F_ACC_RDONLY, pid );
+    } else if ( strcmp( mode, "w" ) == 0 ) {
+        fid = H5Fcreate( filename.data(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT );
+    } else if ( strcmp( mode, "rw" ) == 0 ) {
+        fid = H5Fopen( filename.data(), H5F_ACC_RDWR, H5P_DEFAULT );
+    } else {
+        ERROR( "Invalid mode for opening HDF5 file" );
+    }
+    if ( strcmp( mode, "w" ) == 0 ) {
+        if ( compress == Compression::None ) {
+            writeHDF5<int>( fid, "DefaultCompression", 0 );
+        } else if ( compress == Compression::GZIP ) {
+            writeHDF5<int>( fid, "DefaultCompression", 1 );
+        } else if ( compress == Compression::SZIP ) {
+            writeHDF5<int>( fid, "DefaultCompression", 2 );
+        } else {
+            ERROR( "Internal error" );
+        }
+    }
+    // H5Pclose( pid );
+    return fid;
+}
+void closeHDF5( hid_t fid )
+{
+    // Try to close any remaining objects (needed to ensure we can reopen the data if desired)
+    hid_t file[1000], set[1000], group[1000], type[1000], attr[1000];
+    size_t N_file  = H5Fget_obj_ids( fid, H5F_OBJ_FILE, 1000, file );
+    size_t N_set   = H5Fget_obj_ids( fid, H5F_OBJ_DATASET, 1000, set );
+    size_t N_group = H5Fget_obj_ids( fid, H5F_OBJ_GROUP, 1000, group );
+    size_t N_type  = H5Fget_obj_ids( fid, H5F_OBJ_DATATYPE, 1000, type );
+    size_t N_attr  = H5Fget_obj_ids( fid, H5F_OBJ_ATTR, 1000, attr );
+    for ( size_t i = 0; i < N_file; i++ ) {
+        if ( file[i] != fid )
+            H5Fclose( file[i] );
+    }
+    for ( size_t i = 0; i < N_set; i++ )
+        H5Dclose( set[i] );
+    for ( size_t i = 0; i < N_group; i++ )
+        H5Gclose( group[i] );
+    for ( size_t i = 0; i < N_type; i++ )
+        H5Tclose( type[i] );
+    for ( size_t i = 0; i < N_attr; i++ )
+        H5Aclose( attr[i] );
+    // Flush the data (needed to ensure we can reopen the data if desired)
+    unsigned intent;
+    H5Fget_intent( fid, &intent );
+    if ( intent == H5F_ACC_RDWR || intent == H5F_ACC_TRUNC )
+        H5Fflush( fid, H5F_SCOPE_GLOBAL );
+    // Close the file
+    H5Fclose( fid );
+}
+
+
+/************************************************************************
+ * Check if we support compression                                       *
+ ************************************************************************/
+Compression defaultCompression( hid_t fid )
+{
+    hid_t root = H5Gopen2( fid, "/", H5P_DEFAULT );
+    if ( !H5Dexists( root, "DefaultCompression" ) )
+        return Compression::None;
+    int tmp;
+    readHDF5( root, "DefaultCompression", tmp );
+    Compression compress = Compression::None;
+    if ( tmp == 0 ) {
+        compress = Compression::None;
+    } else if ( tmp == 1 ) {
+        compress = Compression::GZIP;
+    } else if ( tmp == 2 ) {
+        compress = Compression::SZIP;
+    } else {
+        ERROR( "Internal error" );
+    }
+    return compress;
+}
+
+
+/************************************************************************
+ * Create a default chunk size                                           *
+ ************************************************************************/
+hid_t createChunk( const std::vector<hsize_t> &dims, Compression compress )
+{
+    if ( compress == Compression::None || dims.empty() )
+        return H5P_DEFAULT;
+    hsize_t length = 1;
+    for ( auto d : dims )
+        length *= d;
+    if ( length < 512 )
+        return H5P_DEFAULT;
+    hid_t plist = H5Pcreate( H5P_DATASET_CREATE );
+    auto status = H5Pset_chunk( plist, dims.size(), dims.data() );
+    ASSERT( status == 0 );
+    if ( compress == Compression::GZIP ) {
+        status = H5Pset_deflate( plist, 7 );
+        ASSERT( status == 0 );
+    } else if ( compress == Compression::SZIP ) {
+        status = H5Pset_szip( plist, H5_SZIP_NN_OPTION_MASK, 16 );
+        ASSERT( status == 0 );
+    }
+    return plist;
+}
+
+
+/************************************************************************
+ * Write Array                                                           *
+ ************************************************************************/
+template<>
+void writeHDF5<Array<std::complex<double>>>(
+    hid_t fid, const std::string &name, const Array<std::complex<double>> &data )
+{
+    hid_t datatype = getHDF5datatype<std::complex<double>>();
+    // Copy the data
+    size_t N = data.length();
+    auto *y  = new complex_t[N];
+    convert( N, data.data(), y );
+    // Save the array
+    auto dim        = arraySize( data );
+    hid_t dataspace = H5Screate_simple( dim.size(), dim.data(), nullptr );
+    hid_t dataset =
+        H5Dcreate2( fid, name.data(), datatype, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
+    H5Dwrite( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, H5Ptr( y ) );
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Sclose( dataspace );
+    delete[] y;
+}
+template<>
+void writeHDF5<Array<std::string>>(
+    hid_t fid, const std::string &name, const Array<std::string> &data )
+{
+    auto dim        = arraySize( data );
+    hid_t dataspace = H5Screate_simple( dim.size(), dim.data(), nullptr );
+    auto **tmp      = new char *[data.length() + 1];
+    memset( tmp, 0, ( data.length() + 1 ) * sizeof( char * ) );
+    for ( size_t i = 0; i < data.length(); i++ ) {
+        tmp[i] = const_cast<char *>( data( i ).data() );
+    }
+    hid_t datatype = getHDF5datatype<char *>();
+    hid_t props    = H5Pcreate( H5P_DATASET_CREATE );
+    hid_t dataset  = H5Dcreate1( fid, name.data(), datatype, dataspace, props );
+    H5Dwrite( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, tmp );
+    H5Pclose( props );
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Sclose( dataspace );
+    delete[] tmp;
+}
+
+
+/************************************************************************
+ * Specializations for std::vector                                       *
+ ************************************************************************/
+template<>
+void readHDF5<std::vector<bool>>( hid_t fid, const std::string &name, std::vector<bool> &data )
+{
+    Array<bool> tmp;
+    readHDF5( fid, name, tmp );
+    data.resize( tmp.length() );
+    for ( size_t i = 0; i < data.size(); i++ )
+        data[i] = tmp( i );
+}
+template<>
+void writeHDF5<std::vector<bool>>( hid_t fid, const std::string &name, const std::vector<bool> &x )
+{
+    Array<bool> y( x.size() );
+    for ( size_t i = 0; i < x.size(); i++ )
+        y( i ) = x[i];
+    writeHDF5( fid, name, y );
+}
+
+
+/************************************************************************
+ * Explicit instantiations for std::vector                              *
+ ***********************************************************************/
+// clang-format off
+#define INSTANTIATE_STD_VECTOR( TYPE )                              \
+    template<> void readHDF5<std::vector<TYPE>>( hid_t fid, const std::string &name, std::vector<TYPE> &x ) \
+    {                                                               \
+        Array<TYPE> y;                                              \
+        readHDF5( fid, name, y );                                   \
+        x.resize( y.length() );                                     \
+        for ( size_t i = 0; i < x.size(); i++ )                     \
+            x[i] = y( i );                                          \
+    }                                                               \
+    template<> void writeHDF5<std::vector<TYPE>>( hid_t fid, const std::string &name, const std::vector<TYPE> &x ) \
+    {                                                               \
+        Array<TYPE> y;                                              \
+        y.viewRaw( { x.size() }, const_cast<TYPE*>( x.data() ) );   \
+        writeHDF5( fid, name, y );                                  \
+    }
+INSTANTIATE_STD_VECTOR( char )
+INSTANTIATE_STD_VECTOR( unsigned char )
+INSTANTIATE_STD_VECTOR( int )
+INSTANTIATE_STD_VECTOR( unsigned int )
+INSTANTIATE_STD_VECTOR( int16_t )
+INSTANTIATE_STD_VECTOR( uint16_t )
+INSTANTIATE_STD_VECTOR( int64_t )
+INSTANTIATE_STD_VECTOR( uint64_t )
+INSTANTIATE_STD_VECTOR( float )
+INSTANTIATE_STD_VECTOR( double )
+INSTANTIATE_STD_VECTOR( std::string )
+// clang-format on
+
+
+#else // No HDF5
+// Dummy implimentations for no HDF5
+hid_t openHDF5( const std::string &, const char *, Compression ) { return 0; }
+void closeHDF5( hid_t ) {}
+bool H5Gexists( hid_t, const std::string & ) { return false; }
+bool H5Dexists( hid_t, const std::string & ) { return false; }
+hid_t createGroup( hid_t, const std::string & ) { return 0; }
+hid_t openGroup( hid_t, const std::string & ) { return 0; }
+void closeGroup( hid_t ) {}
+#endif
+
+
+} // namespace HDF5
+} // namespace IO

IO/HDF5_IO.h

@@ -0,0 +1,169 @@
+// This file contains helper functions and interfaces for reading/writing HDF5
+#ifndef included_HDF5_h
+#define included_HDF5_h
+
+#include "common/ArraySize.h"
+
+#include <cstring>
+#include <string>
+
+
+// Include the headers and define some basic types
+#ifdef USE_HDF5
+// Using HDF5
+#include "hdf5.h"
+#else
+// Not using HDF5
+typedef int hid_t;
+typedef size_t hsize_t;
+#endif
+
+
+namespace IO {
+namespace HDF5 {
+
+
+enum class Compression : uint8_t { None, GZIP, SZIP };
+
+
+/**
+ * \brief Open an HDF5 file
+ * \details This function opens and HDF5 file for reading/writing.
+ *     Once complete, we must close the file using closeHDF5
+ * @param[in] filename  File to open
+ * @param[in] mode      C string containing a file access mode. It can be:
+ *                      "r"    read: Open file for input operations. The file must exist.
+ *                      "w"    write: Create an empty file for output operations.
+ *                          If a file with the same name already exists, its contents
+ *                          are discarded and the file is treated as a new empty file.
+ *                      "rw" read+write: Open file for reading and writing.  The file must exist.
+ * @param[in] compress  Default compression
+ * @return              Return a handle to the file.
+ */
+hid_t openHDF5(
+    const std::string &filename, const char *mode, Compression compress = Compression::None );
+
+
+/**
+ * \brief Open an HDF5 file
+ * \details This function opens and HDF5 file for reading/writing
+ * @param[in] fid       File to open
+ */
+void closeHDF5( hid_t fid );
+
+
+/**
+ * \brief Retrun the the default compression
+ * \details This function returns the default compression used when the file was created
+ * @param[in] fid       File/Group id
+ */
+Compression defaultCompression( hid_t fid );
+
+
+/**
+ * \brief Open an HDF5 file
+ * \details This function create a chunk for HDF5
+ * @param[in] dims      Chunk size
+ * @param[in] compress  Compression to use
+ * @return              Return a handle to the file.
+ */
+hid_t createChunk( const std::vector<hsize_t> &dims, Compression compress );
+
+
+/**
+ * \brief Write a structure to HDF5
+ * \details This function writes a C++ class/struct to HDF5.
+ *    This is a templated function and users can impliment their own data
+ *    types by creating explicit instantiations for a given type.
+ *    There is no default instantiation except when compiled without HDF5 which is a no-op.
+ * @param[in] fid       File or group to write to
+ * @param[in] name      The name of the variable
+ * @param[in] data      The structure to write
+ */
+template<class T>
+void writeHDF5( hid_t fid, const std::string &name, const T &data );
+
+
+/**
+ * \brief Read a structure from HDF5
+ * \details This function reads a C++ class/struct from HDF5.
+ *    This is a templated function and users can impliment their own data
+ *    types by creating explicit instantiations for a given type.
+ *    There is no default instantiation except when compiled without HDF5 which is a no-op.
+ * @param[in] fid       File or group to read from
+ * @param[in] name      The name of the variable
+ * @param[out] data     The structure to read
+ */
+template<class T>
+void readHDF5( hid_t fid, const std::string &name, T &data );
+
+
+/**
+ * \brief Check if group exists
+ * \details This function checks if an HDF5 group exists in the file
+ * @param[in] fid       ID of group or database to read
+ * @param[in] name      The name of the group
+ */
+bool H5Gexists( hid_t fid, const std::string &name );
+
+
+/**
+ * \brief Check if dataset exists
+ * \details This function checks if an HDF5 dataset exists in the file
+ * @param[in] fid       File to open
+ * @param[in] name      The name of the dataset
+ */
+bool H5Dexists( hid_t fid, const std::string &name );
+
+
+/**
+ * \brief Create a group
+ * \details This function creates a new HDF5 group
+ * @param[in] fid       File or group to write to
+ * @param[in] name      The name of the group
+ */
+hid_t createGroup( hid_t fid, const std::string &name );
+
+
+/**
+ * \brief Open a group
+ * \details This function opens an HDF5 group
+ * @param[in] fid       File or group to write to
+ * @param[in] name      The name of the group
+ */
+hid_t openGroup( hid_t fid, const std::string &name );
+
+
+/**
+ * \brief Close a group
+ * \details This function closes an HDF5 group
+ * @param[in] fid       Group to close
+ */
+void closeGroup( hid_t fid );
+
+
+/**
+ * \brief Get HDF5 data type
+ * \details This function returns the id of the data type
+ */
+template<class T>
+hid_t getHDF5datatype();
+
+
+// Default no-op implimentations for use without HDF5
+// clang-format off
+#ifndef USE_HDF5
+template<class T> void readHDF5( hid_t, const std::string&, T& ) {}
+template<class T> void writeHDF5( hid_t, const std::string&, const T& ) {}
+template<class T> void readHDF5Array( hid_t, const std::string&, Array<T>& ) {}
+template<class T> void writeHDF5Array( hid_t, const std::string&, const Array<T>& ) {}
+template<class T> hid_t getHDF5datatype() { return 0; }
+#endif
+// clang-format on
+
+
+} // namespace HDF5
+} // namespace IO
+
+
+#endif

IO/HDF5_IO.hpp

@@ -0,0 +1,348 @@
+// This file contains helper functions and interfaces for reading/writing HDF5
+#ifndef included_HDF5_hpp
+#define included_HDF5_hpp
+#ifdef USE_HDF5
+
+#include "IO/HDF5_IO.h"
+#include "common/Array.h"
+#include "common/Array.hpp"
+#include "common/Utilities.h"
+
+#include <array>
+#include <complex>
+#include <memory>
+#include <type_traits>
+#include <vector>
+
+
+namespace IO {
+namespace HDF5 {
+
+
+/********************************************************
+ *  External instantiations (scalar)                     *
+ ********************************************************/
+// clang-format off
+template<> void writeHDF5<char>( hid_t, const std::string &, const char & );
+template<> void readHDF5<char>( hid_t, const std::string &, char & );
+template<> void writeHDF5<bool>( hid_t, const std::string &, const bool & );
+template<> void readHDF5<bool>( hid_t, const std::string &, bool & );
+template<> void writeHDF5<int>( hid_t, const std::string &, const int & );
+template<> void readHDF5<int>( hid_t, const std::string &, int & );
+template<> void writeHDF5<long>( hid_t, const std::string &, const long & );
+template<> void readHDF5<long>( hid_t, const std::string &, long & );
+template<> void writeHDF5<float>( hid_t, const std::string &, const float & );
+template<> void readHDF5<float>( hid_t, const std::string &, float & );
+template<> void writeHDF5<double>( hid_t, const std::string &, const double & );
+template<> void readHDF5<double>( hid_t, const std::string &, double & );
+template<> void writeHDF5<unsigned char>( hid_t, const std::string &, const unsigned char & );
+template<> void readHDF5<unsigned char>( hid_t, const std::string &, unsigned char & );
+template<> void writeHDF5<unsigned int>( hid_t, const std::string &, const unsigned int & );
+template<> void readHDF5<unsigned int>( hid_t, const std::string &, unsigned int & );
+template<> void writeHDF5<unsigned long>( hid_t, const std::string &, const unsigned long & );
+template<> void readHDF5<unsigned long>( hid_t, const std::string &, unsigned long & );
+template<> void writeHDF5<std::string>( hid_t, const std::string &, const std::string & );
+template<> void readHDF5<std::string>( hid_t, const std::string &, std::string & );
+template<> void writeHDF5<std::complex<double>>( hid_t, const std::string &, const std::complex<double> & );
+template<> void readHDF5<std::complex<double>>( hid_t, const std::string &, std::complex<double> & );
+template<> void writeHDF5<std::complex<float>>( hid_t, const std::string &, const std::complex<float> & );
+template<> void readHDF5<std::complex<float>>( hid_t, const std::string &, std::complex<float> & );
+// clang-format on
+
+
+/********************************************************
+ *  External instantiations (Array)                      *
+ ********************************************************/
+// clang-format off
+template<> void writeHDF5<Array<char>>( hid_t, const std::string &, const Array<char> & );
+template<> void readHDF5<Array<char>>( hid_t, const std::string &, Array<char> & );
+template<> void writeHDF5<Array<bool>>( hid_t, const std::string &, const Array<bool> & );
+template<> void readHDF5<Array<bool>>( hid_t, const std::string &, Array<bool> & );
+template<> void writeHDF5<Array<int>>( hid_t, const std::string &, const Array<int> & );
+template<> void readHDF5<Array<int>>( hid_t, const std::string &, Array<int> & );
+template<> void writeHDF5<Array<long>>( hid_t, const std::string &, const Array<long> & );
+template<> void readHDF5<Array<long>>( hid_t, const std::string &, Array<long> & );
+template<> void writeHDF5<Array<float>>( hid_t, const std::string &, const Array<float> & );
+template<> void readHDF5<Array<float>>( hid_t, const std::string &, Array<float> & );
+template<> void writeHDF5<Array<double>>( hid_t, const std::string &, const Array<double> & );
+template<> void readHDF5<Array<double>>( hid_t, const std::string &, Array<double> & );
+template<> void writeHDF5<Array<unsigned char>>( hid_t, const std::string &, const Array<unsigned char> & );
+template<> void readHDF5<Array<unsigned char>>( hid_t, const std::string &, Array<unsigned char> & );
+template<> void writeHDF5<Array<unsigned int>>( hid_t, const std::string &, const Array<unsigned int> & );
+template<> void readHDF5<Array<unsigned int>>( hid_t, const std::string &, Array<unsigned int> & );
+template<> void writeHDF5<Array<unsigned long>>( hid_t, const std::string &, const Array<unsigned long> & );
+template<> void readHDF5<Array<unsigned long>>( hid_t, const std::string &, Array<unsigned long> & );
+template<> void writeHDF5<Array<std::string>>( hid_t, const std::string &, const Array<std::string> & );
+template<> void readHDF5<Array<std::string>>( hid_t, const std::string &, Array<std::string> & );
+template<> void writeHDF5<Array<std::string>>( hid_t, const std::string &, const Array<std::string> & );
+template<> void readHDF5<Array<std::string>>( hid_t, const std::string &, Array<std::string> & );
+template<> void writeHDF5<Array<std::complex<double>>>( hid_t, const std::string &, const Array<std::complex<double>> & );
+template<> void readHDF5<Array<std::complex<double>>>( hid_t, const std::string &, Array<std::complex<double>> & );
+template<> void writeHDF5<Array<std::complex<float>>>( hid_t, const std::string &, const Array<std::complex<float>> & );
+template<> void readHDF5<Array<std::complex<float>>>( hid_t, const std::string &, Array<std::complex<float>> & );
+// clang-format on
+
+
+/******************************************************************
+ * Default implimentation                                          *
+ ******************************************************************/
+/*template<class TYPE>
+void writeHDF5( hid_t fid, const std::string &name, const TYPE &x )
+{
+    NULL_USE( fid );
+    if constexpr ( is_shared_ptr<TYPE>::value ) {
+        // We are dealing with a std::shared_ptr
+        writeHDF5( fid, name, *x );
+    } else if constexpr ( is_vector<TYPE>::value ) {
+        // We are dealing with a std::vector
+        typedef decltype( *x.begin() ) TYPE2;
+        typedef typename std::remove_reference<TYPE2>::type TYPE3;
+        typedef typename std::remove_cv<TYPE3>::type TYPE4;
+        Array<TYPE4> y;
+        y.viewRaw( { x.size() }, const_cast<TYPE4 *>( x.data() ) );
+        writeHDF5( fid, name, y );
+    } else if constexpr ( std::is_array<TYPE>::value ) {
+        // We are dealing with a std::array
+        typedef decltype( *x.begin() ) TYPE2;
+        typedef typename std::remove_reference<TYPE2>::type TYPE3;
+        typedef typename std::remove_cv<TYPE3>::type TYPE4;
+        Array<TYPE4> y;
+        y.viewRaw( { x.size() }, const_cast<TYPE4 *>( x.data() ) );
+        writeHDF5( fid, name, y );
+    } else if constexpr ( is_Array<TYPE>::value ) {
+        // We are dealing with an Array
+        std::string typeName = Utilities::demangle( typeid( TYPE ).name() );
+        throw std::logic_error( "Unsupported type writeHDF5<Array<" + typeName + ">>" );
+    } else if constexpr ( std::is_same<TYPE, std::string>::value ) {
+        // We are dealing with a std::string (should be handled through specialization)
+        throw std::logic_error( "Internal error" );
+    } else if constexpr ( std::is_same<TYPE, std::string>::value ||
+                          std::is_same<TYPE, char *>::value ||
+                          std::is_same<TYPE, const char *>::value ) {
+        // We are dealing with a string or char array
+        writeHDF5( fid, name, std::string( x ) );
+    } else if constexpr ( has_size<TYPE>::value ) {
+        // We are dealing with a container
+        typedef decltype( *x.begin() ) TYPE2;
+        typedef typename std::remove_reference<TYPE2>::type TYPE3;
+        typedef typename std::remove_cv<TYPE3>::type TYPE4;
+        std::vector<TYPE4> x2( x.begin(), x.end() );
+        writeHDF5<std::vector<TYPE4>>( fid, name, x2 );
+    } else {
+        throw std::logic_error( "Unsupported type" );
+    }
+}
+template<class TYPE>
+void readHDF5( hid_t fid, const std::string &name, TYPE &x )
+{
+    NULL_USE( fid );
+    if constexpr ( is_shared_ptr<TYPE>::value ) {
+        // We are dealing with a std::shared_ptr
+        readHDF5( fid, name, *x );
+    } else if constexpr ( is_vector<TYPE>::value ) {
+        // We are dealing with a std::vector
+        typedef typename std::remove_reference<decltype( *x.begin() )>::type TYPE2;
+        Array<TYPE2> y;
+        readHDF5( fid, name, y );
+        x.resize( y.length() );
+        // Swap the elements in the arrays to use the move operator
+        for ( size_t i = 0; i < x.size(); i++ )
+            std::swap( x[i], y( i ) );
+    } else if constexpr ( std::is_array<TYPE>::value ) {
+        // We are dealing with a std::array
+        typedef typename std::remove_reference<decltype( *x.begin() )>::type TYPE2;
+        Array<TYPE2> y;
+        readHDF5( fid, name, y );
+        ASSERT( y.length() == x.size() );
+        // Swap the elements in the arrays to use the move operator
+        for ( size_t i = 0; i < x.size(); i++ )
+            std::swap( x[i], y( i ) );
+    } else if constexpr ( is_Array<TYPE>::value ) {
+        // We are dealing with an Array
+        std::string typeName = Utilities::demangle( typeid( TYPE ).name() );
+        throw std::logic_error( "Unsupported type readHDF5<Array<" + typeName + ">>" );
+    } else if constexpr ( std::is_same<TYPE, std::string>::value ) {
+        // We are dealing with a std::string (should be handled through specialization)
+        throw std::logic_error( "Internal error" );
+    } else if constexpr ( std::is_same<TYPE, std::string>::value ||
+                          std::is_same<TYPE, char *>::value ||
+                          std::is_same<TYPE, const char *>::value ) {
+        // We are dealing with a string or char array
+        throw std::logic_error(
+            "Reading data into a string, char*, const char* is not supported" );
+    } else if constexpr ( has_size<TYPE>::value ) {
+        // We are dealing with a container
+        typedef typename std::remove_reference<decltype( *x.begin() )>::type TYPE2;
+        Array<TYPE2> y;
+        readHDF5( fid, name, y );
+        if ( x.size() == y.length() ) {
+            auto it = x.begin();
+            for ( size_t i = 0; i < y.length(); i++, ++it )
+                *it = y( i );
+        } else {
+            throw std::logic_error( "Reading data into an arbitrary container is not finished" );
+        }
+    } else {
+        throw std::logic_error( "Unsupported type" );
+    }
+}*/
+
+
+/************************************************************************
+ * Helper function to get the size of an Array                           *
+ * Note that HDF5 uses C ordered arrays so we need to flip the dimensions*
+ ************************************************************************/
+template<class T>
+inline std::vector<hsize_t> arraySize( const Array<T> &x )
+{
+    int N   = x.ndim();
+    auto s1 = x.size();
+    std::vector<hsize_t> s2( std::max( N, 1 ), 0 );
+    for ( int i = 0; i < N; i++ )
+        s2[N - i - 1] = static_cast<hsize_t>( s1[i] );
+    return s2;
+}
+inline std::vector<size_t> convertSize( int N, const hsize_t *dims )
+{
+    if ( N == 0 )
+        return std::vector<size_t>( 1, 1 );
+    std::vector<size_t> size( N, 0 );
+    for ( int i = 0; i < N; i++ )
+        size[N - i - 1] = static_cast<size_t>( dims[i] );
+    return size;
+}
+
+
+/************************************************************************
+ * readAndConvertHDF5Data                                                *
+ ************************************************************************/
+template<class T>
+typename std::enable_if<std::is_integral<T>::value || std::is_floating_point<T>::value, void>::type
+readAndConvertHDF5Data( hid_t dataset, hid_t datatype, Array<T> &data )
+{
+    if ( H5Tequal( datatype, H5T_NATIVE_CHAR ) ) {
+        Array<char> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_UCHAR ) ) {
+        Array<unsigned char> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_INT8 ) ) {
+        Array<int8_t> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_UINT8 ) ) {
+        Array<uint8_t> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_INT ) ) {
+        Array<int> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_UINT ) ) {
+        Array<unsigned int> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_LONG ) ) {
+        Array<long int> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_ULONG ) ) {
+        Array<unsigned long int> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_FLOAT ) ) {
+        Array<float> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else if ( H5Tequal( datatype, H5T_NATIVE_DOUBLE ) ) {
+        Array<double> data2( data.size() );
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data2.data() );
+        data.copy( data2 );
+    } else {
+        ERROR( "We need to convert unknown data format" );
+    }
+}
+template<class T>
+typename std::enable_if<!std::is_integral<T>::value && !std::is_floating_point<T>::value,
+    void>::type
+readAndConvertHDF5Data( hid_t, hid_t, Array<T> & )
+{
+    ERROR( "Unable to convert data" );
+}
+
+
+/************************************************************************
+ * Default writeHDF5Array                                                *
+ ************************************************************************/
+template<class T>
+void writeHDF5ArrayDefault( hid_t fid, const std::string &name, const Array<T> &data )
+{
+    size_t N_bytes = data.length() * sizeof( T );
+    auto dim       = arraySize( data );
+    hid_t plist    = H5P_DEFAULT;
+    if ( N_bytes < 0x7500 ) {
+        // Use compact storage (limited to < 30K)
+        plist       = H5Pcreate( H5P_DATASET_CREATE );
+        auto status = H5Pset_layout( plist, H5D_COMPACT );
+        ASSERT( status == 0 );
+    } else if ( std::is_same<T, double>::value || std::is_same<T, float>::value ) {
+        // Use compression if availible
+        plist = createChunk( dim, defaultCompression( fid ) );
+    }
+    hid_t dataspace = H5Screate_simple( dim.size(), dim.data(), NULL );
+    hid_t datatype  = getHDF5datatype<T>();
+    hid_t dataset =
+        H5Dcreate2( fid, name.data(), datatype, dataspace, H5P_DEFAULT, plist, H5P_DEFAULT );
+    const void *ptr = data.data() == NULL ? ( (void *) 1 ) : data.data();
+    H5Dwrite( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, ptr );
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Sclose( dataspace );
+    if ( plist != H5P_DEFAULT )
+        H5Pclose( plist );
+}
+
+
+/************************************************************************
+ * Default readHDF5Array                                                 *
+ ************************************************************************/
+template<class T>
+void readHDF5ArrayDefault( hid_t fid, const std::string &name, Array<T> &data )
+{
+    if ( !H5Dexists( fid, name ) ) {
+        // Dataset does not exist
+        data.resize( 0 );
+        return;
+    }
+    hid_t dataset   = H5Dopen2( fid, name.data(), H5P_DEFAULT );
+    hid_t datatype  = H5Dget_type( dataset );
+    hid_t dataspace = H5Dget_space( dataset );
+    hsize_t dims0[10];
+    int ndim  = H5Sget_simple_extent_dims( dataspace, dims0, NULL );
+    auto dims = convertSize( ndim, dims0 );
+    data.resize( dims );
+    hid_t datatype2 = getHDF5datatype<T>();
+    if ( data.empty() ) {
+        // The data is empty
+    } else if ( H5Tequal( datatype, datatype2 ) ) {
+        // The type of Array and the data in HDF5 match
+        H5Dread( dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data.data() );
+    } else {
+        // Try to convert the data
+        readAndConvertHDF5Data( dataset, datatype, data );
+    }
+    H5Dclose( dataset );
+    H5Tclose( datatype );
+    H5Tclose( datatype2 );
+    H5Sclose( dataspace );
+}
+
+
+} // namespace HDF5
+} // namespace IO
+
+
+#endif
+#endif

IO/Mesh.cpp

@@ -133,7 +133,7 @@ size_t PointList::numberPointsVar( VariableType type ) const
 }
 std::pair<size_t, void *> PointList::pack( int level ) const
 {
-    std::pair<size_t, void *> data_out( 0, NULL );
+    std::pair<size_t, void *> data_out( 0, nullptr );
     if ( level == 0 ) {
         data_out.first     = ( 2 + 3 * points.size() ) * sizeof( double );
         double *data_ptr   = new double[2 + 3 * points.size()];
@@ -596,6 +596,8 @@ std::string getString( FileFormat type )
         return "new(single)";
     else if ( type == FileFormat::SILO )
         return "silo";
+    else if ( type == FileFormat::HDF5 )
+        return "hdf5";
     else
         ERROR( "Invalid type" );
     return "";
@@ -611,6 +613,8 @@ FileFormat getFileFormat( const std::string &type_in )
         return FileFormat::NEW_SINGLE;
     else if ( type == "silo" || type == "4" )
         return FileFormat::SILO;
+    else if ( type == "hdf5" || type == "5" )
+        return FileFormat::HDF5;
     else
         ERROR( "Invalid type: " + type );
     return FileFormat::SILO;

IO/Mesh.h

@@ -24,7 +24,7 @@ enum class VariableType {
 };
 enum class DataType { Double, Float, Int, Null };
 enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown };
-enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO };
+enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO, HDF5 };
 
 
 //! Convert enums to/from strings (more future-proof than static_cast<int>)

IO/MeshDatabase.cpp

@@ -366,7 +366,7 @@ std::vector<MeshDatabase> read( const std::string &filename )
     PROFILE_START( "read" );
     FILE *fid = fopen( filename.c_str(), "rb" );
     if ( fid == NULL )
-        ERROR( "Error opening file" );
+        ERROR( "Error opening file: " + filename );
     char *line = new char[10000];
     while ( std::fgets( line, 1000, fid ) != NULL ) {
         if ( line[0] < 32 ) {

IO/Reader.cpp

@@ -1,13 +1,10 @@
 #include "IO/Reader.h"
+#include "IO/HDF5_IO.h"
 #include "IO/IOHelpers.h"
 #include "IO/Mesh.h"
 #include "IO/MeshDatabase.h"
-#include "common/Utilities.h"
-
-#ifdef USE_SILO
 #include "IO/silo.h"
-#endif
-
+#include "common/Utilities.h"
 
 #include <ProfilerApp.h>
 #include <cstdio>
@@ -62,14 +59,16 @@ std::vector<std::string> IO::readTimesteps( const std::string &path, const std::
         filename += "summary.LBM";
     } else if ( format == "silo" ) {
         filename += "LBM.visit";
+    } else if ( format == "hdf5" ) {
+        filename += "LBM.visit";
     } else if ( format == "auto" ) {
-        bool test_old  = fileExists( path + "/summary.LBM" );
-        bool test_silo = fileExists( path + "/LBM.visit" );
-        if ( test_old && test_silo ) {
+        bool test_old = fileExists( path + "/summary.LBM" );
+        bool test_new = fileExists( path + "/LBM.visit" );
+        if ( test_old && test_new ) {
             ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" );
         } else if ( test_old ) {
             filename += "summary.LBM";
-        } else if ( test_silo ) {
+        } else if ( test_new ) {
             filename += "LBM.visit";
         } else {
             ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" );
@@ -88,6 +87,9 @@ std::vector<std::string> IO::readTimesteps( const std::string &path, const std::
         std::string line( buf );
         line.resize( line.size() - 1 );
         auto pos = line.find( "summary.silo" );
+        if ( pos != std::string::npos )
+            line.resize( pos );
+        pos = line.find( "summary.xmf" );
         if ( pos != std::string::npos )
             line.resize( pos );
         if ( line.empty() )
@@ -170,8 +172,8 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
         if ( count % 3 != 0 )
             ERROR( "Error reading file" );
         if ( meshDatabase.type == IO::MeshType::PointMesh ) {
-            size_t N = count / 3;
-            std::shared_ptr<PointList> pointlist( new PointList( N ) );
+            size_t N              = count / 3;
+            auto pointlist        = std::make_shared<PointList>( N );
             std::vector<Point> &P = pointlist->points;
             for ( size_t i = 0; i < N; i++ ) {
                 P[i].x = data[3 * i + 0];
@@ -182,8 +184,8 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
         } else if ( meshDatabase.type == IO::MeshType::SurfaceMesh ) {
             if ( count % 9 != 0 )
                 ERROR( "Error reading file (2)" );
-            size_t N_tri = count / 9;
-            std::shared_ptr<TriList> trilist( new TriList( N_tri ) );
+            size_t N_tri          = count / 9;
+            auto trilist          = std::make_shared<TriList>( N_tri );
             std::vector<Point> &A = trilist->A;
             std::vector<Point> &B = trilist->B;
             std::vector<Point> &C = trilist->C;
@@ -201,7 +203,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
             mesh = trilist;
         } else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) {
             // this was never supported in the old format
-            mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
+            mesh = std::make_shared<DomainMesh>();
         } else {
             ERROR( "Unknown mesh type" );
         }
@@ -222,13 +224,13 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
         fclose( fid );
         ASSERT( count == bytes );
         if ( meshDatabase.meshClass == "PointList" ) {
-            mesh.reset( new IO::PointList() );
+            mesh = std::make_shared<IO::PointList>();
         } else if ( meshDatabase.meshClass == "TriMesh" ) {
-            mesh.reset( new IO::TriMesh() );
+            mesh = std::make_shared<IO::TriMesh>();
         } else if ( meshDatabase.meshClass == "TriList" ) {
-            mesh.reset( new IO::TriList() );
+            mesh = std::make_shared<IO::TriList>();
         } else if ( meshDatabase.meshClass == "DomainMesh" ) {
-            mesh.reset( new IO::DomainMesh() );
+            mesh = std::make_shared<IO::DomainMesh>();
         } else {
             ERROR( "Unknown mesh class" );
         }
@@ -243,7 +245,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
         if ( meshDatabase.meshClass == "PointList" ) {
             Array<double> coords = silo::readPointMesh<double>( fid, database.name );
             ASSERT( coords.size( 1 ) == 3 );
-            std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size( 0 ) ) );
+            auto mesh2 = std::make_shared<IO::PointList>( coords.size( 0 ) );
             for ( size_t i = 0; i < coords.size( 1 ); i++ ) {
                 mesh2->points[i].x = coords( i, 0 );
                 mesh2->points[i].y = coords( i, 1 );
@@ -257,7 +259,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
             ASSERT( tri.size( 1 ) == 3 && coords.size( 1 ) == 3 );
             int N_tri   = tri.size( 0 );
             int N_point = coords.size( 0 );
-            std::shared_ptr<IO::TriMesh> mesh2( new IO::TriMesh( N_tri, N_point ) );
+            auto mesh2  = std::make_shared<IO::TriMesh>( N_tri, N_point );
             for ( int i = 0; i < N_point; i++ ) {
                 mesh2->vertices->points[i].x = coords( i, 0 );
                 mesh2->vertices->points[i].y = coords( i, 1 );
@@ -280,14 +282,81 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::strin
             silo::readUniformMesh( fid, database.name, range, N );
             auto rankinfo = silo::read<int>( fid, database.name + "_rankinfo" );
             RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
-            mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0],
-                range[3] - range[2], range[5] - range[4] ) );
+            mesh = std::make_shared<IO::DomainMesh>( rank_data, N[0], N[1], N[2],
+                range[1] - range[0], range[3] - range[2], range[5] - range[4] );
         } else {
             ERROR( "Unknown mesh class" );
         }
         silo::close( fid );
 #else
         ERROR( "Build without silo support" );
+#endif
+    } else if ( meshDatabase.format == FileFormat::HDF5 ) {
+        // Reading an hdf5 file
+#ifdef USE_HDF5
+        auto &database = meshDatabase.domains[domain];
+        auto filename  = path + "/" + timestep + "/" + database.file;
+        auto fid       = IO::HDF5::openHDF5( filename, "r" );
+        auto gid       = IO::HDF5::openGroup( fid, database.name );
+        if ( meshDatabase.meshClass == "PointList" ) {
+            std::vector<double> x, y, z;
+            IO::HDF5::readHDF5( gid, "x", x );
+            IO::HDF5::readHDF5( gid, "y", y );
+            IO::HDF5::readHDF5( gid, "z", z );
+            ASSERT( y.size() == x.size() && z.size() == x.size() );
+            auto mesh2 = std::make_shared<IO::PointList>( x.size() );
+            for ( size_t i = 0; i < x.size(); i++ ) {
+                mesh2->points[i].x = x[i];
+                mesh2->points[i].y = y[i];
+                mesh2->points[i].z = z[i];
+            }
+            mesh = mesh2;
+        } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
+            // Read the points
+            std::vector<double> x, y, z;
+            IO::HDF5::readHDF5( gid, "x", x );
+            IO::HDF5::readHDF5( gid, "y", y );
+            IO::HDF5::readHDF5( gid, "z", z );
+            // Read the triangles
+            Array<int> tri;
+            IO::HDF5::readHDF5( gid, "tri", tri );
+            ASSERT( tri.size( 0 ) == 3 );
+            size_t N_tri   = tri.size( 1 );
+            size_t N_point = x.size();
+            auto mesh2     = std::make_shared<IO::TriMesh>( N_tri, N_point );
+            for ( size_t i = 0; i < N_point; i++ ) {
+                mesh2->vertices->points[i].x = x[i];
+                mesh2->vertices->points[i].y = y[i];
+                mesh2->vertices->points[i].z = z[i];
+            }
+            for ( size_t i = 0; i < N_tri; i++ ) {
+                mesh2->A[i] = tri( 0, i );
+                mesh2->B[i] = tri( 1, i );
+                mesh2->C[i] = tri( 2, i );
+            }
+            if ( meshDatabase.meshClass == "TriMesh" ) {
+                mesh = mesh2;
+            } else if ( meshDatabase.meshClass == "TriList" ) {
+                auto trilist = IO::getTriList( std::dynamic_pointer_cast<IO::Mesh>( mesh2 ) );
+                mesh         = trilist;
+            }
+        } else if ( meshDatabase.meshClass == "DomainMesh" ) {
+            std::vector<double> range;
+            std::vector<int> N;
+            std::vector<int> rankinfo;
+            IO::HDF5::readHDF5( gid, "range", range );
+            IO::HDF5::readHDF5( gid, "N", N );
+            IO::HDF5::readHDF5( gid, "rankinfo", rankinfo );
+            RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
+            mesh = std::make_shared<IO::DomainMesh>( rank_data, N[0], N[1], N[2],
+                range[1] - range[0], range[3] - range[2], range[5] - range[4] );
+        } else {
+            ERROR( "Unknown mesh class" );
+        }
+        IO::HDF5::closeGroup( gid );
+        IO::HDF5::closeHDF5( fid );
+#else
+        ERROR( "Build without hdf5 support" );
 #endif
     } else {
         ERROR( "Unknown format" );
@@ -322,7 +391,7 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const st
         size_t N              = atol( values[2].c_str() );
         size_t bytes          = atol( values[3].c_str() );
         std::string precision = values[4];
-        var                   = std::shared_ptr<IO::Variable>( new IO::Variable() );
+        var                   = std::make_shared<IO::Variable>();
         var->dim              = dim;
         var->type             = getVariableType( type );
         var->name             = variable;
@@ -341,7 +410,7 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const st
         auto variableDatabase = meshDatabase.getVariableDatabase( variable );
         std::string filename  = path + "/" + timestep + "/" + database.file;
         auto fid              = silo::open( filename, silo::READ );
-        var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
+        var = std::make_shared<Variable>( variableDatabase.dim, variableDatabase.type, variable );
         if ( meshDatabase.meshClass == "PointList" ) {
             var->data = silo::readPointMeshVariable<double>( fid, variable );
         } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
@@ -355,7 +424,30 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const st
 #else
         ERROR( "Build without silo support" );
 #endif
-
+    } else if ( meshDatabase.format == FileFormat::HDF5 ) {
+        // Reading an hdf5 file
+#ifdef USE_HDF5
+        auto &database   = meshDatabase.domains[domain];
+        auto varDatabase = meshDatabase.getVariableDatabase( variable );
+        auto filename    = path + "/" + timestep + "/" + database.file;
+        var      = std::make_shared<Variable>( varDatabase.dim, varDatabase.type, variable );
+        auto fid = IO::HDF5::openHDF5( filename, "r" );
+        auto gid = IO::HDF5::openGroup( fid, database.name );
+        IO::HDF5::readHDF5( gid, var->name, var->data );
+        IO::HDF5::closeHDF5( fid );
+        if ( meshDatabase.meshClass == "PointList" || meshDatabase.meshClass == "TriMesh" ||
+             meshDatabase.meshClass == "TriList" ) {
+            if ( var->data.ndim() == 2 && var->data.size( 0 ) == 3 )
+                var->data = var->data.permute( { 1, 0 } );
+        } else if ( meshDatabase.meshClass == "DomainMesh" ) {
+            if ( var->data.ndim() == 4 && var->data.size( 0 ) == 3 )
+                var->data = var->data.permute( { 1, 2, 3, 0 } );
+        } else {
+            ERROR( "Unknown mesh class" );
+        }
+#else
+        ERROR( "Build without silo support" );
+#endif
     } else {
         ERROR( "Unknown format" );
     }

IO/Reader.h

@@ -87,7 +87,7 @@ std::shared_ptr<IO::Variable> getVariable( const std::string &path, const std::s
  * @brief Reformat the variable to match the mesh
  * @details  This function modifies the dimensions of the array to match the mesh
  * @param[in] mesh          The underlying mesh
- * @param[in/out] variable  The variable name to read
+ * @param[in,out] var       The variable name to read
  */
 void reformatVariable( const IO::Mesh &mesh, IO::Variable &var );
 

IO/SiloWriter.cpp

@@ -0,0 +1,251 @@
+#include "IO/HDF5_IO.h"
+#include "IO/IOHelpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Writer.h"
+#include "IO/silo.h"
+#include "common/MPI.h"
+#include "common/Utilities.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <sys/stat.h>
+#include <vector>
+
+
+#ifdef USE_SILO
+
+
+// Write a PointList mesh (and variables) to a file
+template<class TYPE>
+static void writeSiloPointMesh(
+    DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
+{
+    const auto &points = mesh.getPoints();
+    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
+    for ( size_t i = 0; i < x.size(); i++ ) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const TYPE *coords[] = { x.data(), y.data(), z.data() };
+    IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
+}
+static void writeSiloPointList(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
+{
+    const IO::PointList &mesh  = dynamic_cast<IO::PointList &>( *meshData.mesh );
+    const std::string meshname = database.domains[0].name;
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeSiloPointMesh<double>( fid, mesh, meshname );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeSiloPointMesh<float>( fid, mesh, meshname );
+    } else {
+        ERROR( "Unsupported format" );
+    }
+    const auto &points = mesh.getPoints();
+    std::vector<double> x( points.size() ), y( points.size() ), z( points.size() );
+    for ( size_t i = 0; i < x.size(); i++ ) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const double *coords[] = { x.data(), y.data(), z.data() };
+    IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        const IO::Variable &var = *meshData.vars[i];
+        if ( var.precision == IO::DataType::Double ) {
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+    }
+}
+// Write a TriMesh mesh (and variables) to a file
+template<class TYPE>
+static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname )
+{
+    const auto &points = mesh.vertices->getPoints();
+    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
+    for ( size_t i = 0; i < x.size(); i++ ) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const TYPE *coords[] = { x.data(), y.data(), z.data() };
+    const int *tri[]     = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
+    IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
+}
+static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
+    const IO::TriMesh &mesh, IO::MeshDatabase database )
+{
+    const std::string meshname = database.domains[0].name;
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeSiloTriMesh<double>( fid, mesh, meshname );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeSiloTriMesh<float>( fid, mesh, meshname );
+    } else {
+        ERROR( "Unsupported format" );
+    }
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        const IO::Variable &var = *meshData.vars[i];
+        if ( var.precision == IO::DataType::Double ) {
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+    }
+}
+static void writeSiloTriMesh(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
+{
+    const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
+    writeSiloTriMesh2( fid, meshData, mesh, database );
+}
+static void writeSiloTriList(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
+{
+    auto mesh = getTriMesh( meshData.mesh );
+    writeSiloTriMesh2( fid, meshData, *mesh, database );
+}
+// Write a DomainMesh mesh (and variables) to a file
+static void writeSiloDomainMesh(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
+{
+    const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
+    RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
+    std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
+        ( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
+        ( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
+        ( info.kz + 1 ) * mesh.Lz / info.nz };
+    std::array<int, 3> N        = { mesh.nx, mesh.ny, mesh.nz };
+    auto meshname               = database.domains[0].name;
+    IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
+    IO::silo::write<int>(
+        fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        const auto &var = *meshData.vars[i];
+        if ( var.precision == IO::DataType::Double ) {
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
+        } else {
+            ERROR( "Unsupported format" );
+        }
+    }
+}
+// Write a mesh (and variables) to a file
+static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
+    const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
+{
+    // Create the MeshDatabase
+    auto database = getDatabase( filename, mesh, format, rank );
+    if ( database.meshClass == "PointList" ) {
+        writeSiloPointList( fid, mesh, database );
+    } else if ( database.meshClass == "TriMesh" ) {
+        writeSiloTriMesh( fid, mesh, database );
+    } else if ( database.meshClass == "TriList" ) {
+        writeSiloTriList( fid, mesh, database );
+    } else if ( database.meshClass == "DomainMesh" ) {
+        writeSiloDomainMesh( fid, mesh, database );
+    } else {
+        ERROR( "Unknown mesh class" );
+    }
+    return database;
+}
+// Write the summary file for silo
+std::pair<int, int> getSiloMeshType( const std::string &meshClass )
+{
+    int meshType = 0;
+    int varType  = 0;
+    if ( meshClass == "PointList" ) {
+        meshType = DB_POINTMESH;
+        varType  = DB_POINTVAR;
+    } else if ( meshClass == "TriMesh" ) {
+        meshType = DB_UCDMESH;
+        varType  = DB_UCDVAR;
+    } else if ( meshClass == "TriList" ) {
+        meshType = DB_UCDMESH;
+        varType  = DB_UCDVAR;
+    } else if ( meshClass == "DomainMesh" ) {
+        meshType = DB_QUAD_RECT;
+        varType  = DB_QUADVAR;
+    } else {
+        ERROR( "Unknown mesh class" );
+    }
+    return std::make_pair( meshType, varType );
+}
+void writeSiloSummary(
+    const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
+{
+    auto fid = IO::silo::open( filename, IO::silo::CREATE );
+    for ( const auto &data : meshes_written ) {
+        auto type = getSiloMeshType( data.meshClass );
+        std::vector<int> meshTypes( data.domains.size(), type.first );
+        std::vector<int> varTypes( data.domains.size(), type.second );
+        std::vector<std::string> meshNames;
+        for ( const auto &tmp : data.domains )
+            meshNames.push_back( tmp.file + ":" + tmp.name );
+        IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
+        for ( const auto &variable : data.variables ) {
+            std::vector<std::string> varnames;
+            for ( const auto &tmp : data.domains )
+                varnames.push_back( tmp.file + ":" + variable.name );
+            IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
+        }
+    }
+    IO::silo::close( fid );
+}
+// Write the mesh data to silo
+std::vector<IO::MeshDatabase> writeMeshesSilo( const std::vector<IO::MeshDataStruct> &meshData,
+    const std::string &path, IO::FileFormat format, int rank )
+{
+    std::vector<IO::MeshDatabase> meshes_written;
+    char filename[100], fullpath[200];
+    sprintf( filename, "%05i.silo", rank );
+    sprintf( fullpath, "%s/%s", path.c_str(), filename );
+    auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
+    for ( size_t i = 0; i < meshData.size(); i++ ) {
+        auto mesh = meshData[i].mesh;
+        meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
+    }
+    IO::silo::close( fid );
+    return meshes_written;
+}
+
+
+#else
+
+
+// Write the mesh data to silo
+std::vector<IO::MeshDatabase> writeMeshesSilo(
+    const std::vector<IO::MeshDataStruct> &, const std::string &, IO::FileFormat, int )
+{
+    return std::vector<IO::MeshDatabase>();
+}
+void writeSiloSummary( const std::vector<IO::MeshDatabase> &, const std::string & );
+
+
+#endif

IO/Writer.cpp

@@ -1,10 +1,13 @@
 #include "IO/Writer.h"
+#include "IO/HDF5_IO.h"
 #include "IO/IOHelpers.h"
 #include "IO/MeshDatabase.h"
-#include "IO/silo.h"
+#include "IO/Xdmf.h"
 #include "common/MPI.h"
 #include "common/Utilities.h"
 
+#include "ProfilerApp.h"
+
 #include <algorithm>
 #include <memory>
 #include <set>
@@ -12,7 +15,17 @@
 #include <vector>
 
 
-enum class Format { OLD, NEW, SILO, UNKNOWN };
+enum class Format { OLD, NEW, SILO, HDF5, UNKNOWN };
+
+
+/****************************************************
+ * External declerations                             *
+ ****************************************************/
+std::vector<IO::MeshDatabase> writeMeshesSilo(
+    const std::vector<IO::MeshDataStruct> &, const std::string &, IO::FileFormat, int );
+void writeSiloSummary( const std::vector<IO::MeshDatabase> &, const std::string & );
+std::vector<IO::MeshDatabase> writeMeshesHDF5(
+    const std::vector<IO::MeshDataStruct> &, const std::string &, IO::FileFormat, int, Xdmf & );
 
 
 /****************************************************
@@ -75,6 +88,8 @@ void IO::initialize( const std::string &path, const std::string &format, bool ap
         global_IO_format = Format::NEW;
     else if ( format == "silo" )
         global_IO_format = Format::SILO;
+    else if ( format == "hdf5" )
+        global_IO_format = Format::HDF5;
     else
         ERROR( "Unknown format" );
     int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
@@ -83,7 +98,7 @@ void IO::initialize( const std::string &path, const std::string &format, bool ap
         std::string filename;
         if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW )
             filename = global_IO_path + "/summary.LBM";
-        else if ( global_IO_format == Format::SILO )
+        else if ( global_IO_format == Format::SILO || global_IO_format == Format::HDF5 )
             filename = global_IO_path + "/LBM.visit";
         else
             ERROR( "Unknown format" );
@@ -116,10 +131,10 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
         domain.file   = filename;
         domain.offset = 0;
         mesh_entry.domains.push_back( domain );
-        if ( !meshData[i].vars.empty() ) {
+        static bool printVariableSupportMsg = true;
+        if ( !meshData[i].vars.empty() && printVariableSupportMsg ) {
+            printVariableSupportMsg = false;
             printf( "Warning: variables are not supported with this format (original)\n" );
-            // for (size_t j=0; j<meshData[i].vars.size(); j++)
-            //    mesh_entry.variables.push_back( meshData[i].vars[j]->name );
         }
         const std::string meshClass = mesh->className();
         if ( meshClass == "PointList" ) {
@@ -170,7 +185,7 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
 
 
 // Create the database entry for the mesh data
-static IO::MeshDatabase getDatabase(
+IO::MeshDatabase IO::getDatabase(
     const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
 {
     char domainname[100];
@@ -212,6 +227,7 @@ static IO::MeshDatabase getDatabase(
 static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
     const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
 {
+    ASSERT( !mesh.meshName.empty() );
     const int level = 0;
     // Create the MeshDatabase
     IO::MeshDatabase database = getDatabase( filename, mesh, format, rank );
@@ -234,6 +250,8 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
         size_t N        = mesh.vars[i]->data.length();
         size_t N_mesh   = mesh.mesh->numberPointsVar( mesh.vars[i]->type );
         ASSERT( N == dim * N_mesh );
+        ASSERT( !type.empty() );
+        ASSERT( !variable.name.empty() );
         fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank,
             variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) );
         fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid );
@@ -243,212 +261,6 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
 }
 
 
-#ifdef USE_SILO
-// Write a PointList mesh (and variables) to a file
-template<class TYPE>
-static void writeSiloPointMesh(
-    DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
-{
-    const auto &points = mesh.getPoints();
-    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
-    for ( size_t i = 0; i < x.size(); i++ ) {
-        x[i] = points[i].x;
-        y[i] = points[i].y;
-        z[i] = points[i].z;
-    }
-    const TYPE *coords[] = { x.data(), y.data(), z.data() };
-    IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
-}
-static void writeSiloPointList(
-    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
-{
-    const IO::PointList &mesh  = dynamic_cast<IO::PointList &>( *meshData.mesh );
-    const std::string meshname = database.domains[0].name;
-    if ( meshData.precision == IO::DataType::Double ) {
-        writeSiloPointMesh<double>( fid, mesh, meshname );
-    } else if ( meshData.precision == IO::DataType::Float ) {
-        writeSiloPointMesh<float>( fid, mesh, meshname );
-    } else {
-        ERROR( "Unsupported format" );
-    }
-    const auto &points = mesh.getPoints();
-    std::vector<double> x( points.size() ), y( points.size() ), z( points.size() );
-    for ( size_t i = 0; i < x.size(); i++ ) {
-        x[i] = points[i].x;
-        y[i] = points[i].y;
-        z[i] = points[i].z;
-    }
-    const double *coords[] = { x.data(), y.data(), z.data() };
-    IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
-    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
-        const IO::Variable &var = *meshData.vars[i];
-        if ( var.precision == IO::DataType::Double ) {
-            IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
-        } else if ( var.precision == IO::DataType::Float ) {
-            Array<float> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
-        } else if ( var.precision == IO::DataType::Int ) {
-            Array<int> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
-        } else {
-            ERROR( "Unsupported format" );
-        }
-    }
-}
-// Write a TriMesh mesh (and variables) to a file
-template<class TYPE>
-static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname )
-{
-    const auto &points = mesh.vertices->getPoints();
-    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
-    for ( size_t i = 0; i < x.size(); i++ ) {
-        x[i] = points[i].x;
-        y[i] = points[i].y;
-        z[i] = points[i].z;
-    }
-    const TYPE *coords[] = { x.data(), y.data(), z.data() };
-    const int *tri[]     = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
-    IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
-}
-static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
-    const IO::TriMesh &mesh, IO::MeshDatabase database )
-{
-    const std::string meshname = database.domains[0].name;
-    if ( meshData.precision == IO::DataType::Double ) {
-        writeSiloTriMesh<double>( fid, mesh, meshname );
-    } else if ( meshData.precision == IO::DataType::Float ) {
-        writeSiloTriMesh<float>( fid, mesh, meshname );
-    } else {
-        ERROR( "Unsupported format" );
-    }
-    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
-        const IO::Variable &var = *meshData.vars[i];
-        if ( var.precision == IO::DataType::Double ) {
-            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
-        } else if ( var.precision == IO::DataType::Float ) {
-            Array<float> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
-        } else if ( var.precision == IO::DataType::Int ) {
-            Array<int> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
-        } else {
-            ERROR( "Unsupported format" );
-        }
-    }
-}
-static void writeSiloTriMesh(
-    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
-{
-    const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
-    writeSiloTriMesh2( fid, meshData, mesh, database );
-}
-static void writeSiloTriList(
-    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
-{
-    auto mesh = getTriMesh( meshData.mesh );
-    writeSiloTriMesh2( fid, meshData, *mesh, database );
-}
-// Write a DomainMesh mesh (and variables) to a file
-static void writeSiloDomainMesh(
-    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
-{
-    const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
-    RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
-    std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
-        ( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
-        ( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
-        ( info.kz + 1 ) * mesh.Lz / info.nz };
-    std::array<int, 3> N        = { mesh.nx, mesh.ny, mesh.nz };
-    auto meshname               = database.domains[0].name;
-    IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
-    IO::silo::write<int>(
-        fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
-    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
-        const auto &var = *meshData.vars[i];
-        if ( var.precision == IO::DataType::Double ) {
-            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
-        } else if ( var.precision == IO::DataType::Float ) {
-            Array<float> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
-        } else if ( var.precision == IO::DataType::Int ) {
-            Array<int> data2( var.data.size() );
-            data2.copy( var.data );
-            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
-        } else {
-            ERROR( "Unsupported format" );
-        }
-    }
-}
-// Write a mesh (and variables) to a file
-static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
-    const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
-{
-    // Create the MeshDatabase
-    auto database = getDatabase( filename, mesh, format, rank );
-    if ( database.meshClass == "PointList" ) {
-        writeSiloPointList( fid, mesh, database );
-    } else if ( database.meshClass == "TriMesh" ) {
-        writeSiloTriMesh( fid, mesh, database );
-    } else if ( database.meshClass == "TriList" ) {
-        writeSiloTriList( fid, mesh, database );
-    } else if ( database.meshClass == "DomainMesh" ) {
-        writeSiloDomainMesh( fid, mesh, database );
-    } else {
-        ERROR( "Unknown mesh class" );
-    }
-    return database;
-}
-// Write the summary file for silo
-std::pair<int, int> getSiloMeshType( const std::string &meshClass )
-{
-    int meshType = 0;
-    int varType  = 0;
-    if ( meshClass == "PointList" ) {
-        meshType = DB_POINTMESH;
-        varType  = DB_POINTVAR;
-    } else if ( meshClass == "TriMesh" ) {
-        meshType = DB_UCDMESH;
-        varType  = DB_UCDVAR;
-    } else if ( meshClass == "TriList" ) {
-        meshType = DB_UCDMESH;
-        varType  = DB_UCDVAR;
-    } else if ( meshClass == "DomainMesh" ) {
-        meshType = DB_QUAD_RECT;
-        varType  = DB_QUADVAR;
-    } else {
-        ERROR( "Unknown mesh class" );
-    }
-    return std::make_pair( meshType, varType );
-}
-void writeSiloSummary(
-    const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
-{
-    auto fid = IO::silo::open( filename, IO::silo::CREATE );
-    for ( const auto &data : meshes_written ) {
-        auto type = getSiloMeshType( data.meshClass );
-        std::vector<int> meshTypes( data.domains.size(), type.first );
-        std::vector<int> varTypes( data.domains.size(), type.second );
-        std::vector<std::string> meshNames;
-        for ( const auto &tmp : data.domains )
-            meshNames.push_back( tmp.file + ":" + tmp.name );
-        IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
-        for ( const auto &variable : data.variables ) {
-            std::vector<std::string> varnames;
-            for ( const auto &tmp : data.domains )
-                varnames.push_back( tmp.file + ":" + variable.name );
-            IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
-        }
-    }
-    IO::silo::close( fid );
-}
-#endif
-
-
 // Write the mesh data in the new format
 static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
     const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
@@ -459,43 +271,14 @@ static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
     sprintf( filename, "%05i", rank );
     sprintf( fullpath, "%s/%s", path.c_str(), filename );
     FILE *fid = fopen( fullpath, "wb" );
-    for ( size_t i = 0; i < meshData.size(); i++ ) {
-        std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
+    ASSERT( fid != nullptr );
+    for ( size_t i = 0; i < meshData.size(); i++ )
         meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) );
-    }
     fclose( fid );
     return meshes_written;
 }
 
 
-// Write the mesh data to silo
-static std::vector<IO::MeshDatabase> writeMeshesSilo(
-    const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
-    int rank )
-{
-#ifdef USE_SILO
-    std::vector<IO::MeshDatabase> meshes_written;
-    char filename[100], fullpath[200];
-    sprintf( filename, "%05i.silo", rank );
-    sprintf( fullpath, "%s/%s", path.c_str(), filename );
-    auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
-    for ( size_t i = 0; i < meshData.size(); i++ ) {
-        auto mesh = meshData[i].mesh;
-        meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
-    }
-    IO::silo::close( fid );
-    return meshes_written;
-#else
-    NULL_USE( meshData );
-    NULL_USE( path );
-    NULL_USE( format );
-    NULL_USE( rank );
-    ERROR( "Application built without silo support" );
-    return std::vector<IO::MeshDatabase>();
-#endif
-}
-
-
 /****************************************************
  * Write the mesh data                               *
  ****************************************************/
@@ -513,6 +296,7 @@ void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStr
     std::string path = global_IO_path + "/" + subdir;
     recursiveMkdir( path, S_IRWXU | S_IRGRP );
     // Write the mesh files
+    Xdmf xmf;
     std::vector<IO::MeshDatabase> meshes_written;
     if ( global_IO_format == Format::OLD ) {
         // Write the original triangle format
@@ -523,24 +307,28 @@ void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStr
     } else if ( global_IO_format == Format::SILO ) {
         // Write silo
         meshes_written = writeMeshesSilo( meshData, path, IO::FileFormat::SILO, rank );
+    } else if ( global_IO_format == Format::HDF5 ) {
+        // Write hdf5
+        meshes_written = writeMeshesHDF5( meshData, path, IO::FileFormat::HDF5, rank, xmf );
     } else {
         ERROR( "Unknown format" );
     }
     // Gather a complete list of files on rank 0
     meshes_written = gatherAll( meshes_written, comm );
+    // Gather xmf file (if applicable)
+    if ( global_IO_format == Format::HDF5 ) {
+        xmf.gather( comm );
+    }
     // Write the summary files
     if ( rank == 0 ) {
         // Write the summary file for the current timestep
-        char filename[200];
-        sprintf( filename, "%s/LBM.summary", path.c_str() );
-        write( meshes_written, filename );
-// Write summary silo file if needed
-#ifdef USE_SILO
+        write( meshes_written, path + "/LBM.summary" );
+        // Write summary file if needed
         if ( global_IO_format == Format::SILO ) {
-            sprintf( filename, "%s/summary.silo", path.c_str() );
-            writeSiloSummary( meshes_written, filename );
+            writeSiloSummary( meshes_written, path + "/summary.silo" );
+        } else if ( global_IO_format == Format::HDF5 ) {
+            xmf.write( path + "/summary.xmf" );
         }
-#endif
         // Add the timestep to the global summary file
         if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) {
             auto filename = global_IO_path + "/summary.LBM";
@@ -552,6 +340,11 @@ void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStr
             FILE *fid     = fopen( filename.c_str(), "ab" );
             fprintf( fid, "%s/summary.silo\n", subdir.c_str() );
             fclose( fid );
+        } else if ( global_IO_format == Format::HDF5 ) {
+            auto filename = global_IO_path + "/LBM.visit";
+            FILE *fid     = fopen( filename.c_str(), "ab" );
+            fprintf( fid, "%s/summary.xmf\n", subdir.c_str() );
+            fclose( fid );
         } else {
             ERROR( "Unknown format" );
         }

IO/Writer.h

@@ -21,11 +21,13 @@ namespace IO {
  * @param[in] format        The data format to use:
  *                              old - Old mesh format
  *                                    (provided for backward compatibility, cannot write variables)
- *                              new - New format, 1 file/process silo - Silo
+ *                              new - New format, 1 file/process
+ *                              silo - Silo
+ *                              hdf5 - HDF5 + XMDF
  * @param[in] append        Append any existing data (default is false)
  */
 void initialize(
-    const std::string &path = "", const std::string &format = "silo", bool append = false );
+    const std::string &path = "", const std::string &format = "hdf5", bool append = false );
 
 
 /*!
@@ -55,6 +57,11 @@ inline void writeData(
 }
 
 
+// Create the database entry for the mesh data
+IO::MeshDatabase getDatabase(
+    const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank );
+
+
 } // namespace IO
 
 #endif

IO/Xdmf.cpp

@@ -0,0 +1,620 @@
+#include "IO/Xdmf.h"
+
+#include "common/Array.h"
+#include "common/UtilityMacros.h"
+
+
+ArraySize squeeze( const ArraySize &x )
+{
+    int Nd      = 0;
+    size_t N[5] = { 1 };
+    for ( size_t i = 0; i < x.maxDim(); i++ ) {
+        if ( x[i] != 1 )
+            N[Nd++] = x[i];
+    }
+    return ArraySize( std::max( Nd, 1 ), N );
+}
+
+
+// Helper functions
+static void addDataItem(
+    FILE *xmf, const std::string &indent, ArraySize size, const std::string &location )
+{
+    size = squeeze( size );
+    if ( size.ndim() == 1 ) {
+        fprintf( xmf, "%s<DataItem Dimensions=\"%lu\"", indent.data(), size[0] );
+    } else if ( size.ndim() == 2 ) {
+        fprintf( xmf, "%s<DataItem Dimensions=\"%lu %lu\"", indent.data(), size[1], size[0] );
+    } else if ( size.ndim() == 3 ) {
+        fprintf( xmf, "%s<DataItem Dimensions=\"%lu %lu %lu\"", indent.data(), size[2], size[1],
+            size[0] );
+    } else if ( size.ndim() == 4 ) {
+        fprintf( xmf, "%s<DataItem Dimensions=\"%lu %lu %lu %lu\"", indent.data(), size[3], size[2],
+            size[1], size[0] );
+    } else {
+        ERROR( "Invalid number of dimensions" );
+    }
+    fprintf( xmf, " Format=\"HDF\">\n" );
+    fprintf( xmf, "%s  %s\n", indent.data(), location.data() );
+    fprintf( xmf, "%s</DataItem>\n", indent.data() );
+}
+template<class TYPE>
+static void addVariable( FILE *xmf, const std::string &indent, const std::string &name,
+    const std::string &type, const std::string &center, ArraySize size,
+    const std::string &location )
+{
+    fprintf( xmf, "%s<Attribute Name=\"%s\" AttributeType=\"%s\" Center=\"%s\">\n", indent.data(),
+        name.data(), type.data(), center.data() );
+    addDataItem( xmf, indent + "  ", size, location );
+    fprintf( xmf, "%s</Attribute>\n", indent.data() );
+}
+
+
+/****************************************************************
+ * Enum functions                                                *
+ ****************************************************************/
+/*template<class TYPE>
+static Xdmf::DataType getDataType()
+{
+    if ( std::is_same<TYPE, char>::value )
+        return Xdmf::DataType::Char;
+    else if ( std::is_same<TYPE, int32_t>::value )
+        return Xdmf::DataType::Int32;
+    else if ( std::is_same<TYPE, int64_t>::value )
+        return Xdmf::DataType::Int64;
+    else if ( std::is_same<TYPE, uint32_t>::value )
+        return Xdmf::DataType::Uint32;
+    else if ( std::is_same<TYPE, uint64_t>::value )
+        return Xdmf::DataType::Uint64;
+    else if ( std::is_same<TYPE, float>::value )
+        return Xdmf::DataType::Float;
+    else if ( std::is_same<TYPE, double>::value )
+        return Xdmf::DataType::Double;
+    else
+        ERROR( "Invalid type" );
+}*/
+static const char *TopologyTypeNames[]  = { "", "Polyvertex", "Polyline", "Polygon", "Triangle",
+    "Quadrilateral", "Tetrahedron", "Pyramid", "Wedge", "Hexahedron", "Edge_3", "Triangle_6",
+    "Quadrilateral_8", "Tetrahedron_10", "Pyramid_13", "Wedge_15", "Hexahedron_20", "Mixed",
+    "CurvilinearMesh2D", "CurvilinearMesh3D", "RectangularMesh2D", "RectangularMesh3D",
+    "UniformMesh2D", "UniformMesh3D" };
+static const uint8_t TopologyTypeDOFs[] = { 0, 1, 2, 0, 3, 4, 4, 5, 6, 8, 3, 6, 8, 10, 13, 15, 20,
+    0, 0, 0, 0, 0, 0, 0 };
+
+
+/****************************************************************
+ * Create a mesh                                                 *
+ ****************************************************************/
+Xdmf::MeshData Xdmf::createPointMesh( const std::string &name, uint8_t NDIM, size_t N,
+    const std::string &x, const std::string &y, const std::string &z )
+{
+    return createUnstructuredMesh( name, NDIM, TopologyType::Polyvertex, N, "", N, x, y, z );
+}
+Xdmf::MeshData Xdmf::createUniformMesh(
+    const std::string &name, const std::vector<double> &range, ArraySize size )
+{
+    ASSERT( range.size() == 2 * size.ndim() );
+    MeshData data;
+    data.name = name;
+    data.size = size;
+    if ( size.ndim() == 2 )
+        data.type = TopologyType::UniformMesh2D;
+    else if ( size.ndim() == 3 )
+        data.type = TopologyType::UniformMesh3D;
+    else
+        ERROR( "# of dimensions != 2 or 3" );
+    for ( int i = 0; i < 2 * size.ndim(); i++ )
+        data.range[i] = range[i];
+    return data;
+}
+Xdmf::MeshData Xdmf::createCurvilinearMesh( const std::string &name, ArraySize size,
+    const std::string &x, const std::string &y, const std::string &z )
+{
+    MeshData data;
+    data.name = name;
+    if ( size.ndim() == 2 )
+        data.type = TopologyType::CurvilinearMesh2D;
+    else if ( size.ndim() == 3 )
+        data.type = TopologyType::CurvilinearMesh3D;
+    else
+        ERROR( "Invalid size for Curvilinear mesh" );
+    data.size = size;
+    data.x    = x;
+    data.y    = y;
+    data.z    = z;
+    return data;
+}
+Xdmf::MeshData Xdmf::createUnstructuredMesh( const std::string &name, uint8_t NDIM,
+    TopologyType type, size_t NumElements, const std::string &dofMap, size_t NumNodes,
+    const std::string &x, const std::string &y, const std::string &z )
+{
+    ASSERT( type != TopologyType::Null );
+    MeshData data;
+    data.name   = name;
+    data.type   = type;
+    data.size   = { NDIM, NumElements, NumNodes };
+    data.dofMap = dofMap;
+    data.x      = x;
+    data.y      = y;
+    data.z      = z;
+    return data;
+}
+
+
+/****************************************************************
+ * Add a variable                                                *
+ ****************************************************************/
+void Xdmf::MeshData::addVariable( const std::string &meshName, const std::string &varName,
+    ArraySize varSize, RankType rank, Center center, const std::string &varData )
+{
+    VarData var;
+    var.name     = varName;
+    var.size     = varSize;
+    var.data     = varData;
+    var.rankType = rank;
+    var.center   = center;
+    vars.push_back( std::move( var ) );
+}
+
+
+/****************************************************************
+ * Add a mesh domain                                             *
+ ****************************************************************/
+void Xdmf::addMesh( const std::string &meshName, const MeshData &domain )
+{
+    auto &domains = d_meshData[meshName];
+    for ( const auto &domain2 : domains )
+        ASSERT( domain2.name != domain.name );
+    domains.push_back( domain );
+}
+
+
+/****************************************************************
+ * Write a variable                                              *
+ ****************************************************************/
+static void writeVariable( FILE *fid, const Xdmf::VarData &var, const std::string &indent )
+{
+    // Write the variable name
+    fprintf( fid, "%s<Attribute Name=\"%s\"", indent.data(), var.name.data() );
+    // Write the variable type
+    if ( var.rankType == Xdmf::RankType::Scalar ) {
+        fprintf( fid, " AttributeType=\"Scalar\"" );
+    } else if ( var.rankType == Xdmf::RankType::Vector ) {
+        fprintf( fid, " AttributeType=\"Vector\"" );
+    } else if ( var.rankType == Xdmf::RankType::Tensor ) {
+        fprintf( fid, " AttributeType=\"Tensor\"" );
+    } else if ( var.rankType == Xdmf::RankType::Tensor6 ) {
+        fprintf( fid, " AttributeType=\"Tensor6\"" );
+    } else if ( var.rankType == Xdmf::RankType::Matrix ) {
+        fprintf( fid, " AttributeType=\"Matrix\"" );
+    } else if ( var.rankType == Xdmf::RankType::GlobalID ) {
+        fprintf( fid, " AttributeType=\"GlobalID\"" );
+    } else {
+        ERROR( "Unknown center type" );
+    }
+    // Write the variable centering
+    if ( var.center == Xdmf::Center::Node ) {
+        fprintf( fid, " Center=\"Node\">\n" );
+    } else if ( var.center == Xdmf::Center::Cell ) {
+        fprintf( fid, " Center=\"Cell\">\n" );
+    } else if ( var.center == Xdmf::Center::Grid ) {
+        fprintf( fid, " Center=\"Grid\">\n" );
+    } else if ( var.center == Xdmf::Center::Face ) {
+        fprintf( fid, " Center=\"Face\">\n" );
+    } else if ( var.center == Xdmf::Center::Edge ) {
+        fprintf( fid, " Center=\"Edge\">\n" );
+    } else if ( var.center == Xdmf::Center::Other ) {
+        fprintf( fid, " Center=\"Other\">\n" );
+    } else {
+        ERROR( "Unknown center type" );
+    }
+    // Write the data item
+    addDataItem( fid, indent + "  ", var.size, var.data );
+    // Finished
+    fprintf( fid, "%s</Attribute>\n", indent.data() );
+}
+
+
+/****************************************************************
+ * Write the mesh grid                                           *
+ ****************************************************************/
+static void writeMeshGrid( FILE *fid, const Xdmf::MeshData &mesh, const std::string &indent )
+{
+    const char *s = indent.data();
+    double x0[3]  = { mesh.range[0], mesh.range[2], mesh.range[4] };
+    double dx[3]  = { ( mesh.range[1] - mesh.range[0] ) / mesh.size[0],
+        ( mesh.range[3] - mesh.range[2] ) / mesh.size[1],
+        ( mesh.range[5] - mesh.range[4] ) / mesh.size[2] };
+    switch ( mesh.type ) {
+    case Xdmf::TopologyType::UniformMesh2D:
+        // Write a uniform 2d mesh
+        fprintf( fid, "%s<Grid Name=\"%s\" GridType=\"Uniform\">\n", s, mesh.name.data() );
+        fprintf( fid,
+            "%s  <Topology TopologyType=\"2DCoRectMesh\" NumberOfElements=\"%lu %lu\"/>\n", s,
+            mesh.size[1] + 1, mesh.size[0] + 1 );
+        fprintf( fid, "%s  <Geometry GeometryType=\"ORIGIN_DXDY\">\n", s );
+        fprintf(
+            fid, "%s    <DataItem  Format=\"XML\" NumberType=\"float\" Dimensions=\"2\">\n", s );
+        fprintf( fid, "%s      %0.12e  %0.12e\n", s, x0[0], x0[1] );
+        fprintf( fid, "%s    </DataItem>\n", s );
+        fprintf(
+            fid, "%s    <DataItem  Format=\"XML\" NumberType=\"float\" Dimensions=\"2\">\n", s );
+        fprintf( fid, "%s       %0.12e  %0.12e\n", s, dx[0], dx[1] );
+        fprintf( fid, "%s    </DataItem>\n", s );
+        fprintf( fid, "%s  </Geometry>\n", s );
+        break;
+    case Xdmf::TopologyType::UniformMesh3D:
+        // Write a uniform 3d mesh
+        fprintf( fid, "%s<Grid Name=\"%s\" GridType=\"Uniform\">\n", s, mesh.name.data() );
+        fprintf( fid,
+            "%s  <Topology TopologyType=\"3DCoRectMesh\" NumberOfElements=\"%lu %lu\"/>\n", s,
+            mesh.size[1] + 1, mesh.size[0] + 1 );
+        fprintf( fid, "%s  <Geometry GeometryType=\"ORIGIN_DXDYDZ\">\n", s );
+        fprintf(
+            fid, "%s    <DataItem  Format=\"XML\" NumberType=\"float\" Dimensions=\"3\">\n", s );
+        fprintf( fid, "%s      %0.12e  %0.12e  %0.12e\n", s, x0[0], x0[1], x0[2] );
+        fprintf( fid, "%s    </DataItem>\n", s );
+        fprintf(
+            fid, "%s    <DataItem  Format=\"XML\" NumberType=\"float\" Dimensions=\"3\">\n", s );
+        fprintf( fid, "%s       %0.12e  %0.12e  %0.12e\n", s, dx[0], dx[1], dx[2] );
+        fprintf( fid, "%s    </DataItem>\n", s );
+        fprintf( fid, "%s  </Geometry>\n", s );
+        break;
+    case Xdmf::TopologyType::CurvilinearMesh2D:
+        // Write a 2D curvillinear mesh
+        fprintf( fid, "%s<Grid Name=\"%s\" GridType=\"Uniform\">\n", s, mesh.name.data() );
+        fprintf( fid, "%s  <Topology TopologyType=\"2DSMesh\" NumberOfElements=\"%lu %lu\"/>\n", s,
+            mesh.size[1] + 1, mesh.size[0] + 1 );
+        fprintf( fid, "%s  <Geometry GeometryType=\"X_Y\">\n", s );
+        addDataItem( fid, indent + "    ", mesh.size + 1, mesh.x );
+        addDataItem( fid, indent + "    ", mesh.size + 1, mesh.y );
+        fprintf( fid, "%s  </Geometry>\n", s );
+        break;
+    case Xdmf::TopologyType::CurvilinearMesh3D:
+        // Write a 3D curvillinear mesh
+        fprintf( fid, "%s<Grid Name=\"%s\" GridType=\"Uniform\">\n", s, mesh.name.data() );
+        fprintf( fid, "%s  <Topology TopologyType=\"3DSMesh\" NumberOfElements=\"%lu %lu %lu\"/>\n",
+            s, mesh.size[2] + 1, mesh.size[1] + 1, mesh.size[0] + 1 );
+        fprintf( fid, "%s  <Geometry GeometryType=\"X_Y_Z\">\n", s );
+        addDataItem( fid, indent + "    ", mesh.size + 1, mesh.x );
+        addDataItem( fid, indent + "    ", mesh.size + 1, mesh.y );
+        addDataItem( fid, indent + "    ", mesh.size + 1, mesh.z );
+        fprintf( fid, "%s  </Geometry>\n", s );
+        break;
+    case Xdmf::TopologyType::Polyvertex:
+    case Xdmf::TopologyType::Polyline:
+    case Xdmf::TopologyType::Polygon:
+    case Xdmf::TopologyType::Triangle:
+    case Xdmf::TopologyType::Quadrilateral:
+    case Xdmf::TopologyType::Tetrahedron:
+    case Xdmf::TopologyType::Pyramid:
+    case Xdmf::TopologyType::Wedge:
+    case Xdmf::TopologyType::Hexahedron:
+    case Xdmf::TopologyType::Edge_3:
+    case Xdmf::TopologyType::Triangle_6:
+    case Xdmf::TopologyType::Quadrilateral_8:
+    case Xdmf::TopologyType::Tetrahedron_10:
+    case Xdmf::TopologyType::Pyramid_13:
+    case Xdmf::TopologyType::Wedge_15:
+    case Xdmf::TopologyType::Hexahedron_20:
+        // Write an unstructured mesh
+        {
+            int NDIM      = mesh.size[0];
+            size_t Nelem  = mesh.size[1];
+            size_t Nnode  = mesh.size[2];
+            uint8_t Ndofs = TopologyTypeDOFs[static_cast<int>( mesh.type )];
+            auto type     = TopologyTypeNames[static_cast<int>( mesh.type )];
+            fprintf( fid, "%s<Grid Name=\"%s\">\n", s, mesh.name.data() );
+            fprintf( fid, "%s  <Topology TopologyType=\"%s\"", s, type );
+            fprintf( fid, " NumberOfElements=\"%lu\">\n", Nelem );
+            if ( !mesh.dofMap.empty() )
+                addDataItem( fid, indent + "    ", { Ndofs, Nelem }, mesh.dofMap );
+            fprintf( fid, "%s  </Topology>\n", s );
+            if ( NDIM == 2 ) {
+                if ( mesh.y.empty() ) {
+                    fprintf( fid, "%s  <Geometry GeometryType=\"XY\">\n", s );
+                    addDataItem( fid, indent + "    ", { 2, Nnode }, mesh.x );
+                } else {
+                    fprintf( fid, "%s  <Geometry GeometryType=\"X_Y\">\n", s );
+                    addDataItem( fid, indent + "    ", Nnode, mesh.x );
+                    addDataItem( fid, indent + "    ", Nnode, mesh.y );
+                }
+            } else if ( NDIM == 3 ) {
+                if ( mesh.y.empty() ) {
+                    fprintf( fid, "%s  <Geometry GeometryType=\"XYZ\">\n", s );
+                    addDataItem( fid, indent + "    ", { 2, Nnode }, mesh.x );
+                } else {
+                    fprintf( fid, "%s  <Geometry GeometryType=\"X_Y_Z\">\n", s );
+                    addDataItem( fid, indent + "    ", Nnode, mesh.x );
+                    addDataItem( fid, indent + "    ", Nnode, mesh.y );
+                    addDataItem( fid, indent + "    ", Nnode, mesh.z );
+                }
+            } else {
+                ERROR( "Dimensions other than 2 or 3 are not supported" );
+            }
+            fprintf( fid, "%s  </Geometry>\n", s );
+        }
+        break;
+    default: {
+        auto msg = "Invalid mesh type: " + std::to_string( static_cast<int>( mesh.type ) ) + " - " +
+                   mesh.name;
+        ERROR( msg );
+    }
+    }
+    // Write the variables
+    for ( const auto &var : mesh.vars )
+        writeVariable( fid, var, indent + "  " );
+    fprintf( fid, "%s  </Grid>\n", s );
+}
+
+
+/****************************************************************
+ * Write the XDMF xml file                                       *
+ ****************************************************************/
+void Xdmf::write( const std::string &filename ) const
+{
+    // Create XDMF file
+    auto fid = fopen( filename.data(), "w" );
+    fprintf( fid, "<?xml version=\"1.0\" ?>\n" );
+    fprintf( fid, "<!DOCTYPE Xdmf SYSTEM \"Xdmf.dtd\" []>\n" );
+    fprintf( fid, "<Xdmf Version=\"2.0\">\n" );
+    fprintf( fid, "<Domain>\n" );
+    // Write an empty mesh to enable collections to work properly
+    fprintf( fid, "  <Grid Name=\"\" GridType=\"Uniform\"></Grid>\n\n" );
+    // Write each mesh
+    for ( const auto &data : d_meshData ) {
+        auto name    = data.first;
+        auto domains = data.second;
+        if ( domains.empty() )
+            continue;
+        if ( domains.size() == 1u && name == domains[0].name ) {
+            writeMeshGrid( fid, domains[0], "  " );
+        } else {
+            fprintf( fid, "  <Grid Name=\"%s\" GridType=\"Collection\">\n", name.data() );
+            for ( const auto &domain : domains )
+                writeMeshGrid( fid, domain, "    " );
+            fprintf( fid, "  </Grid>\n\n" );
+        }
+    }
+    fprintf( fid, "</Domain>\n" );
+    fprintf( fid, "</Xdmf>\n" );
+    fclose( fid );
+}
+
+
+/****************************************************************
+ * Pack/Unpack data                                              *
+ ****************************************************************/
+template<class T>
+typename std::enable_if<std::is_trivially_copyable<T>::value, size_t>::type size( const T & )
+{
+    return sizeof( T );
+}
+template<class T>
+typename std::enable_if<std::is_trivially_copyable<T>::value, char *>::type pack(
+    char *ptr, const T &x )
+{
+    memcpy( ptr, &x, sizeof( T ) );
+    return ptr + sizeof( T );
+}
+template<class T>
+typename std::enable_if<std::is_trivially_copyable<T>::value, char *>::type unpack(
+    char *ptr, T &x )
+{
+    memcpy( &x, ptr, sizeof( T ) );
+    return ptr + sizeof( T );
+}
+static size_t size( const std::string &str ) { return sizeof( int ) + str.size(); }
+static char *pack( char *ptr, const std::string &str )
+{
+    int N = str.size();
+    memcpy( ptr, &N, sizeof( int ) );
+    ptr += sizeof( int );
+    memcpy( ptr, str.data(), str.size() );
+    ptr += str.size();
+    return ptr;
+}
+static char *unpack( char *ptr, std::string &str )
+{
+    int N = 0;
+    memcpy( &N, ptr, sizeof( int ) );
+    ASSERT( N >= 0 && N < 1000 );
+    ptr += sizeof( int );
+    str = std::string( ptr, N );
+    ptr += N;
+    return ptr;
+}
+static size_t size( const Xdmf::VarData &data )
+{
+    size_t bytes = 0;
+    bytes += size( data.name );
+    bytes += size( data.size );
+    bytes += size( data.rankType );
+    bytes += size( data.center );
+    bytes += size( data.data );
+    return bytes;
+}
+static char *pack( char *ptr, const Xdmf::VarData &data )
+{
+    ptr = pack( ptr, data.name );
+    ptr = pack( ptr, data.size );
+    ptr = pack( ptr, data.rankType );
+    ptr = pack( ptr, data.center );
+    ptr = pack( ptr, data.data );
+    return ptr;
+}
+static char *unpack( char *ptr, Xdmf::VarData &data )
+{
+    int rankType = 0, center = 0;
+    ptr           = unpack( ptr, data.name );
+    ptr           = unpack( ptr, data.size );
+    ptr           = unpack( ptr, rankType );
+    ptr           = unpack( ptr, center );
+    ptr           = unpack( ptr, data.data );
+    data.rankType = static_cast<Xdmf::RankType>( rankType );
+    data.center   = static_cast<Xdmf::Center>( center );
+    return ptr;
+}
+static size_t size( const Xdmf::MeshData &data )
+{
+    int N_vars   = data.vars.size();
+    size_t bytes = 0;
+    bytes += size( data.name );
+    bytes += size( data.type );
+    bytes += size( data.size );
+    bytes += size( data.range );
+    bytes += size( data.x );
+    bytes += size( data.y );
+    bytes += size( data.z );
+    bytes += size( N_vars );
+    for ( int i = 0; i < N_vars; i++ )
+        bytes += size( data.vars[i] );
+    return bytes;
+}
+static char *pack( char *ptr, const Xdmf::MeshData &data )
+{
+    int N_vars = data.vars.size();
+    ptr        = pack( ptr, data.name );
+    ptr        = pack( ptr, data.type );
+    ptr        = pack( ptr, data.size );
+    ptr        = pack( ptr, data.range );
+    ptr        = pack( ptr, data.x );
+    ptr        = pack( ptr, data.y );
+    ptr        = pack( ptr, data.z );
+    ptr        = pack( ptr, N_vars );
+    for ( int i = 0; i < N_vars; i++ )
+        ptr = pack( ptr, data.vars[i] );
+    return ptr;
+}
+static char *unpack( char *ptr, Xdmf::MeshData &data )
+{
+    int N_vars = 0;
+    ptr        = unpack( ptr, data.name );
+    ptr        = unpack( ptr, data.type );
+    ptr        = unpack( ptr, data.size );
+    ptr        = unpack( ptr, data.range );
+    ptr        = unpack( ptr, data.x );
+    ptr        = unpack( ptr, data.y );
+    ptr        = unpack( ptr, data.z );
+    ptr        = unpack( ptr, N_vars );
+    data.vars.resize( N_vars );
+    for ( int i = 0; i < N_vars; i++ )
+        ptr = unpack( ptr, data.vars[i] );
+    return ptr;
+}
+static size_t size( const std::vector<Xdmf::MeshData> &data )
+{
+    size_t bytes = 0;
+    int N        = data.size();
+    bytes += size( N );
+    for ( int i = 0; i < N; i++ )
+        bytes += size( data[i] );
+    return bytes;
+}
+static char *pack( char *ptr, const std::vector<Xdmf::MeshData> &data )
+{
+    int N = data.size();
+    ptr   = pack( ptr, N );
+    for ( int i = 0; i < N; i++ )
+        ptr = pack( ptr, data[i] );
+    return ptr;
+}
+static char *unpack( char *ptr, std::vector<Xdmf::MeshData> &data )
+{
+    data.clear();
+    int N = data.size();
+    ptr   = unpack( ptr, N );
+    data.resize( N );
+    for ( int i = 0; i < N; i++ )
+        ptr = unpack( ptr, data[i] );
+    return ptr;
+}
+static size_t size( const std::map<std::string, std::vector<Xdmf::MeshData>> &data )
+{
+    size_t bytes = 0;
+    int N_map    = data.size();
+    bytes += size( N_map );
+    for ( const auto &tmp : data ) {
+        bytes += size( tmp.first );
+        bytes += size( tmp.second );
+    }
+    return bytes;
+}
+static char *pack( char *ptr, const std::map<std::string, std::vector<Xdmf::MeshData>> &data )
+{
+    int N_map = data.size();
+    ptr       = pack( ptr, N_map );
+    for ( const auto &tmp : data ) {
+        ptr = pack( ptr, tmp.first );
+        ptr = pack( ptr, tmp.second );
+    }
+    return ptr;
+}
+static char *unpack( char *ptr, std::map<std::string, std::vector<Xdmf::MeshData>> &data )
+{
+    data.clear();
+    int N_map = data.size();
+    ptr       = unpack( ptr, N_map );
+    for ( int i = 0; i < N_map; i++ ) {
+        std::string name;
+        std::vector<Xdmf::MeshData> data2;
+        ptr        = unpack( ptr, name );
+        ptr        = unpack( ptr, data2 );
+        data[name] = std::move( data2 );
+    }
+    return ptr;
+}
+
+
+/****************************************************************
+ * Gather all data to rank 0                                     *
+ ****************************************************************/
+void Xdmf::gather( const Utilities::MPI &comm )
+{
+    if ( comm.getRank() == 0 ) {
+        for ( int i = 1; i < comm.getSize(); i++ ) {
+            // Recieve the data
+            size_t N_meshes = 0, N_bytes = 0;
+            comm.recv( &N_meshes, 1, i, 717 );
+            comm.recv( &N_bytes, 1, i, 718 );
+            auto buf = new char[N_bytes];
+            comm.recv( buf, N_bytes, i, 719 );
+            // Unpack the data
+            std::map<std::string, std::vector<MeshData>> data;
+            unpack( buf, data );
+            delete[] buf;
+            // Add the meshes
+            for ( auto tmp : data ) {
+                const auto &name    = tmp.first;
+                const auto &domains = tmp.second;
+                if ( domains.size() == 1u && domains[0].name == name ) {
+                    // We are dealing with a single mesh
+                    ASSERT( d_meshData.find( name ) == d_meshData.end() );
+                    d_meshData.insert( tmp );
+                } else {
+                    // Add the domains
+                    auto &meshes = d_meshData[name];
+                    for ( auto domain : domains ) {
+                        for ( const auto &tmp : meshes )
+                            ASSERT( tmp.name != domain.name );
+                        meshes.push_back( domain );
+                    }
+                }
+            }
+        }
+    } else {
+        // Send the number of meshes
+        size_t N_meshes = d_meshData.size();
+        comm.send( &N_meshes, 1, 0, 717 );
+        // Pack the send data
+        size_t N_bytes = size( d_meshData );
+        comm.send( &N_bytes, 1, 0, 718 );
+        auto buf = new char[N_bytes];
+        pack( buf, d_meshData );
+        // Send the data to rank 0
+        comm.send( buf, N_bytes, 0, 719 );
+        delete[] buf;
+        // Clear the internal data
+        d_meshData.clear();
+    }
+}

IO/Xdmf.h

@@ -0,0 +1,131 @@
+#include "IO/HDF5_IO.h"
+#include "common/Array.h"
+#include "common/MPI.h"
+#include "common/Utilities.h"
+
+#include <map>
+#include <stdio.h>
+#include <stdlib.h>
+#include <vector>
+
+
+// Helper class to write/read XDMF files
+class Xdmf
+{
+public:
+    enum class TopologyType {
+        Null = 0,
+        Polyvertex,
+        Polyline,
+        Polygon,
+        Triangle,
+        Quadrilateral,
+        Tetrahedron,
+        Pyramid,
+        Wedge,
+        Hexahedron,
+        Edge_3,
+        Triangle_6,
+        Quadrilateral_8,
+        Tetrahedron_10,
+        Pyramid_13,
+        Wedge_15,
+        Hexahedron_20,
+        Mixed,
+        CurvilinearMesh2D,
+        CurvilinearMesh3D,
+        RectangularMesh2D,
+        RectangularMesh3D,
+        UniformMesh2D,
+        UniformMesh3D,
+    };
+    enum class DataType { Null = 0, Char, Int32, Int64, Uint32, Uint64, Float, Double };
+    enum class RankType { Null = 0, Scalar, Vector, Tensor, Tensor6, Matrix, GlobalID };
+    enum class Center { Null = 0, Node, Edge, Face, Cell, Grid, Other };
+
+    struct VarData {
+        std::string name;  // Variable name
+        ArraySize size;    // Size of variable
+        RankType rankType; // Rank order of data
+        Center center;     // Variable centering
+        std::string data;  // Variable data
+    };
+
+    struct MeshData {
+        std::string name;          // Name of mesh
+        TopologyType type;         // Type of mesh
+        ArraySize size;            // Size of mesh (meaning depends on mesh type)
+        double range[6];           // Range of the mesh (only used for UniformMesh2D/UniformMesh3D)
+        std::string x;             // x coordinates (or xy/xyz coordinates)
+        std::string y;             // y coordinates
+        std::string z;             // z coordinates
+        std::string dofMap;        // mesh connectivity
+        std::vector<VarData> vars; // Variables
+        MeshData() : type( TopologyType::Null ), range{ 0 } {}
+        //! Add a variable
+        void addVariable( const std::string &meshName, const std::string &varName,
+            ArraySize varSize, RankType rank, Center center, const std::string &varData );
+    };
+
+
+public:
+    //! Add a Point mesh
+    static MeshData createPointMesh( const std::string &name, uint8_t NDIM, size_t N,
+        const std::string &x, const std::string &y = "", const std::string &z = "" );
+
+    /*!
+     * @brief  Add a uniform mesh
+     * @details  This function adds a uniform rectangular mesh
+     * @param[in] name          The name of the mesh
+     * @param[in] range         The range of the mesh [ x_min, x_max, y_min, y_max, z_min, z_max ]
+     * @param[in] size          The number of cells in the mesh
+     */
+    static MeshData createUniformMesh(
+        const std::string &name, const std::vector<double> &range, ArraySize size );
+
+    /*!
+     * @brief  Add a Curvilinear mesh
+     * @details  This function adds a curvilinear mesh
+     * @param[in] name          The name of the mesh
+     * @param[in] size          The number of cells in the mesh
+     * @param[in] x             The x coordinates or the xy/xyz coordinates
+     * @param[in] y             The y coordinates (may be null)
+     * @param[in] z             The z coordinates (may be null)
+     */
+    static MeshData createCurvilinearMesh( const std::string &name, ArraySize size,
+        const std::string &x, const std::string &y, const std::string &z = "" );
+
+    /*!
+     * @brief  Add an unstructured mesh
+     * @details  This function adds an unstructerd mesh to the class to write.
+     *    The mesh may be one of several unsupported unstructured mesh types.
+     *    This function does not support mixed elements.
+     * @param[in] name          The name of the mesh
+     * @param[in] NDIM          The number of physical dimensions
+     * @param[in] type          The element type
+     * @param[in] NumElements   The number of elements
+     * @param[in] dofMap        The connectivity information (type x NumElements)
+     * @param[in] NumNodes      The number of nodes
+     * @param[in] x             The x coordinates or the xy/xyz coordinates
+     * @param[in] y             The y coordinates (may be null)
+     * @param[in] z             The z coordinates (may be null)
+     */
+    static MeshData createUnstructuredMesh( const std::string &name, uint8_t NDIM,
+        TopologyType type, size_t NumElements, const std::string &dofMap, size_t NumNodes,
+        const std::string &x, const std::string &y = "", const std::string &z = "" );
+
+
+public:
+    //! Add a sub-domain
+    void addMesh( const std::string &meshName, const MeshData &domain );
+
+    //! Gather all data to rank 0
+    void gather( const Utilities::MPI &comm );
+
+    //! Write the xml file
+    void write( const std::string &filename ) const;
+
+
+private:
+    std::map<std::string, std::vector<MeshData>> d_meshData;
+};

IO/netcdf.cpp

@@ -496,7 +496,7 @@ template void write<double>( int fid, const std::string &var, const std::vector<
     const Array<double> &data, const RankInfoStruct &info );
 
 
-}; // namespace netcdf
+} // namespace netcdf
 
 #else
 

IO/netcdf.h

@@ -123,6 +123,8 @@ Array<TYPE> getAtt( int fid, const std::string &att );
  * @brief  Write the dimensions
  * @details  This function writes the grid dimensions to netcdf.
  * @param fid           Handle to the open file
+ * @param names
+ * @param dims
  */
 std::vector<int> defDim(
     int fid, const std::vector<std::string> &names, const std::vector<int> &dims );
@@ -132,11 +134,17 @@ std::vector<int> defDim(
  * @brief  Write a variable
  * @details  This function writes a variable to netcdf.
  * @param fid           Handle to the open file
+ * @param var           Variable to read
+ * @param dimids
+ * @param data
+ * @param rank_info
  */
 template<class TYPE>
 void write( int fid, const std::string &var, const std::vector<int> &dimids,
     const Array<TYPE> &data, const RankInfoStruct &rank_info );
 
 
-}; // namespace netcdf
+} // namespace netcdf
+
+
 #endif

IO/silo.cpp

@@ -10,7 +10,8 @@
 #include <silo.h>
 
 
-namespace IO::silo {
+namespace IO {
+namespace silo {
 
 
 /****************************************************
@@ -99,7 +100,8 @@ void writeMultiVar( DBfile *fid, const std::string &varname,
 }
 
 
-}; // namespace IO::silo
+} // namespace silo
+} // namespace IO
 
 
 #else

IO/silo.h

@@ -49,7 +49,7 @@ void close( DBfile *fid );
  * @param[in] fid           Handle to the open file
  * @param[in] name          Name of variable
  */
-DataType varDataType( DBfile *dbfile, const std::string &name );
+DataType varDataType( DBfile *fid, const std::string &name );
 
 
 /*!
@@ -250,8 +250,6 @@ void writeMultiMesh( DBfile *fid, const std::string &meshname,
  * @param[in] varname       Mesh name
  * @param[in] subVarNames   Names of the sub meshes in the form "filename:meshname"
  * @param[in] subVarTypes   Type of each submesh
- * @param[in] ndim          Dimension of variable (used to determine suffix)
- * @param[in] nvar          Number of subvariables (used to determine suffix)
  */
 void writeMultiVar( DBfile *fid, const std::string &varname,
     const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );

IO/silo.hpp

@@ -245,11 +245,11 @@ Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname )
         copyData<TYPE>( data2, type, var->vals[i] );
         memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
     }
-    DBFreeQuadvar( var );
     std::vector<size_t> dims( var->ndims + 1, var->nvals );
     for ( int d = 0; d < var->ndims; d++ )
         dims[d] = var->dims[d];
     data.reshape( dims );
+    DBFreeQuadvar( var );
     return data;
 }
 

StackTrace/StackTrace.cpp

@@ -856,7 +856,7 @@ static void getFileAndLineObject( staticVector<StackTrace::stack_info*,blockSize
             char *buf = tmp2;
             if ( buf[0] != '?' && buf[0] != 0 ) {
                 size_t j = 0;
-                for ( j = 0; j < 4095 && buf[j] != ':'; j++ ) {
+                for ( j = 0; j < 1024 && buf[j] != ':'; j++ ) {
                 }
                 buf[j] = 0;
                 copy( buf, info[i]->filename, info[i]->filenamePath );

analysis/ElectroChemistry.cpp

@@ -1,5 +1,6 @@
 #include "analysis/ElectroChemistry.h"
 
+
 ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr <Domain> dm):
 	Dm(dm)
 {
@@ -9,13 +10,25 @@ ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr <Domain> dm):
 	
 	ChemicalPotential.resize(Nx,Ny,Nz);       ChemicalPotential.fill(0);
 	ElectricalPotential.resize(Nx,Ny,Nz);      ElectricalPotential.fill(0);
+	ElectricalField_x.resize(Nx,Ny,Nz);      ElectricalField_x.fill(0);
+	ElectricalField_y.resize(Nx,Ny,Nz);      ElectricalField_y.fill(0);
+	ElectricalField_z.resize(Nx,Ny,Nz);      ElectricalField_z.fill(0);
 	Pressure.resize(Nx,Ny,Nz);     	Pressure.fill(0);
 	Rho.resize(Nx,Ny,Nz);       	Rho.fill(0);
 	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
 	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
 	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
 	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
-	
+    IonFluxDiffusive_x.resize(Nx,Ny,Nz); IonFluxDiffusive_x.fill(0);
+    IonFluxDiffusive_y.resize(Nx,Ny,Nz); IonFluxDiffusive_y.fill(0);
+    IonFluxDiffusive_z.resize(Nx,Ny,Nz); IonFluxDiffusive_z.fill(0);
+    IonFluxAdvective_x.resize(Nx,Ny,Nz); IonFluxAdvective_x.fill(0);
+    IonFluxAdvective_y.resize(Nx,Ny,Nz); IonFluxAdvective_y.fill(0);
+    IonFluxAdvective_z.resize(Nx,Ny,Nz); IonFluxAdvective_z.fill(0);
+    IonFluxElectrical_x.resize(Nx,Ny,Nz); IonFluxElectrical_x.fill(0);
+    IonFluxElectrical_y.resize(Nx,Ny,Nz); IonFluxElectrical_y.fill(0);
+    IonFluxElectrical_z.resize(Nx,Ny,Nz); IonFluxElectrical_z.fill(0);
+
 	if (Dm->rank()==0){
 		bool WriteHeader=false;
 		TIMELOG = fopen("electrokinetic.csv","r");
@@ -158,21 +171,55 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P
 
     visData[0].meshName = "domain";
     visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
-    auto ElectricPotential = std::make_shared<IO::Variable>();
+    //electric potential
+    auto ElectricPotentialVar = std::make_shared<IO::Variable>();
+    //electric field
+    auto ElectricFieldVar_x = std::make_shared<IO::Variable>();
+    auto ElectricFieldVar_y = std::make_shared<IO::Variable>();
+    auto ElectricFieldVar_z = std::make_shared<IO::Variable>();
+
+    //ion concentration
     std::vector<shared_ptr<IO::Variable>> IonConcentration;
     for (size_t ion=0; ion<Ion.number_ion_species; ion++){
         IonConcentration.push_back(std::make_shared<IO::Variable>());
     }
+    //fluid velocity
     auto VxVar = std::make_shared<IO::Variable>();
     auto VyVar = std::make_shared<IO::Variable>();
     auto VzVar = std::make_shared<IO::Variable>();
-    
+    // diffusive ion flux
+    std::vector<shared_ptr<IO::Variable>> IonFluxDiffusive;
+    for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+        //push in x-,y-, and z-component for each ion species
+        IonFluxDiffusive.push_back(std::make_shared<IO::Variable>());
+        IonFluxDiffusive.push_back(std::make_shared<IO::Variable>());
+        IonFluxDiffusive.push_back(std::make_shared<IO::Variable>());
+    }
+    // advective ion flux
+    std::vector<shared_ptr<IO::Variable>> IonFluxAdvective;
+    for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+        //push in x-,y-, and z-component for each ion species
+        IonFluxAdvective.push_back(std::make_shared<IO::Variable>());
+        IonFluxAdvective.push_back(std::make_shared<IO::Variable>());
+        IonFluxAdvective.push_back(std::make_shared<IO::Variable>());
+    }
+    // electro-migrational ion flux
+    std::vector<shared_ptr<IO::Variable>> IonFluxElectrical;
+    for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+        //push in x-,y-, and z-component for each ion species
+        IonFluxElectrical.push_back(std::make_shared<IO::Variable>());
+        IonFluxElectrical.push_back(std::make_shared<IO::Variable>());
+        IonFluxElectrical.push_back(std::make_shared<IO::Variable>());
+    }
+    //--------------------------------------------------------------------------------------------------------------------
+
+    //-------------------------------------Create Names for Variables------------------------------------------------------
     if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
-    	ElectricPotential->name = "ElectricPotential";
-    	ElectricPotential->type = IO::VariableType::VolumeVariable;
-    	ElectricPotential->dim = 1;
-    	ElectricPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
-        visData[0].vars.push_back(ElectricPotential);
+    	ElectricPotentialVar->name = "ElectricPotential";
+    	ElectricPotentialVar->type = IO::VariableType::VolumeVariable;
+    	ElectricPotentialVar->dim = 1;
+    	ElectricPotentialVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricPotentialVar);
     }
 
     if (vis_db->getWithDefault<bool>( "save_concentration", true )){
@@ -203,7 +250,105 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P
         VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
         visData[0].vars.push_back(VzVar);
     }
+
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_diffusive", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            // x-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_x",ion+1);
+    		IonFluxDiffusive[3*ion+0]->name = VisName;
+    		IonFluxDiffusive[3*ion+0]->type = IO::VariableType::VolumeVariable;
+    		IonFluxDiffusive[3*ion+0]->dim = 1;
+    		IonFluxDiffusive[3*ion+0]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxDiffusive[3*ion+0]);
+            // y-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_y",ion+1);
+    		IonFluxDiffusive[3*ion+1]->name = VisName;
+    		IonFluxDiffusive[3*ion+1]->type = IO::VariableType::VolumeVariable;
+    		IonFluxDiffusive[3*ion+1]->dim = 1;
+    		IonFluxDiffusive[3*ion+1]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxDiffusive[3*ion+1]);
+            // z-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_z",ion+1);
+    		IonFluxDiffusive[3*ion+2]->name = VisName;
+    		IonFluxDiffusive[3*ion+2]->type = IO::VariableType::VolumeVariable;
+    		IonFluxDiffusive[3*ion+2]->dim = 1;
+    		IonFluxDiffusive[3*ion+2]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxDiffusive[3*ion+2]);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_advective", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            // x-component of advective flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_x",ion+1);
+    		IonFluxAdvective[3*ion+0]->name = VisName;
+    		IonFluxAdvective[3*ion+0]->type = IO::VariableType::VolumeVariable;
+    		IonFluxAdvective[3*ion+0]->dim = 1;
+    		IonFluxAdvective[3*ion+0]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxAdvective[3*ion+0]);
+            // y-component of advective flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_y",ion+1);
+    		IonFluxAdvective[3*ion+1]->name = VisName;
+    		IonFluxAdvective[3*ion+1]->type = IO::VariableType::VolumeVariable;
+    		IonFluxAdvective[3*ion+1]->dim = 1;
+    		IonFluxAdvective[3*ion+1]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxAdvective[3*ion+1]);
+            // z-component of advective flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_z",ion+1);
+    		IonFluxAdvective[3*ion+2]->name = VisName;
+    		IonFluxAdvective[3*ion+2]->type = IO::VariableType::VolumeVariable;
+    		IonFluxAdvective[3*ion+2]->dim = 1;
+    		IonFluxAdvective[3*ion+2]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxAdvective[3*ion+2]);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_electrical", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            // x-component of electro-migrational flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_x",ion+1);
+    		IonFluxElectrical[3*ion+0]->name = VisName;
+    		IonFluxElectrical[3*ion+0]->type = IO::VariableType::VolumeVariable;
+    		IonFluxElectrical[3*ion+0]->dim = 1;
+    		IonFluxElectrical[3*ion+0]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxElectrical[3*ion+0]);
+            // y-component of electro-migrational flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_y",ion+1);
+    		IonFluxElectrical[3*ion+1]->name = VisName;
+    		IonFluxElectrical[3*ion+1]->type = IO::VariableType::VolumeVariable;
+    		IonFluxElectrical[3*ion+1]->dim = 1;
+    		IonFluxElectrical[3*ion+1]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxElectrical[3*ion+1]);
+            // z-component of electro-migrational flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_z",ion+1);
+    		IonFluxElectrical[3*ion+2]->name = VisName;
+    		IonFluxElectrical[3*ion+2]->type = IO::VariableType::VolumeVariable;
+    		IonFluxElectrical[3*ion+2]->dim = 1;
+    		IonFluxElectrical[3*ion+2]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+    		visData[0].vars.push_back(IonFluxElectrical[3*ion+2]);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_electric_field", false )){
+        ElectricFieldVar_x->name = "ElectricField_x";
+        ElectricFieldVar_x->type = IO::VariableType::VolumeVariable;
+        ElectricFieldVar_x->dim = 1;
+        ElectricFieldVar_x->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricFieldVar_x);
+        ElectricFieldVar_y->name = "ElectricField_y";
+        ElectricFieldVar_y->type = IO::VariableType::VolumeVariable;
+        ElectricFieldVar_y->dim = 1;
+        ElectricFieldVar_y->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricFieldVar_y);
+        ElectricFieldVar_z->name = "ElectricField_z";
+        ElectricFieldVar_z->type = IO::VariableType::VolumeVariable;
+        ElectricFieldVar_z->dim = 1;
+        ElectricFieldVar_z->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(ElectricFieldVar_z);
+    }
+    //--------------------------------------------------------------------------------------------------------------------
     
+    //------------------------------------Save All Variables--------------------------------------------------------------
     if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
     	ASSERT(visData[0].vars[0]->name=="ElectricPotential");
     	Poisson.getElectricPotential(ElectricalPotential);
@@ -214,7 +359,7 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P
     if (vis_db->getWithDefault<bool>( "save_concentration", true )){
     	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
     		sprintf(VisName,"IonConcentration_%zu",ion+1);
-    		IonConcentration[ion]->name = VisName;
+    		//IonConcentration[ion]->name = VisName;
     		ASSERT(visData[0].vars[1+ion]->name==VisName);
     		Array<double>& IonConcentrationData = visData[0].vars[1+ion]->data;
     		Ion.getIonConcentration(Rho,ion);
@@ -234,10 +379,101 @@ void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &P
     	fillData.copy(Vel_y,VelyData);
     	fillData.copy(Vel_z,VelzData);
     }
+
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_diffusive", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            
+            // x-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_x",ion+1);
+    		//IonFluxDiffusive[3*ion+0]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species+3*ion+0]->name==VisName);
+            // y-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_y",ion+1);
+    		//IonFluxDiffusive[3*ion+1]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species+3*ion+1]->name==VisName);
+            // z-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxDiffusive_z",ion+1);
+    		//IonFluxDiffusive[3*ion+2]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species+3*ion+2]->name==VisName);
+
+    		Array<double>& IonFluxData_x = visData[0].vars[4+Ion.number_ion_species+3*ion+0]->data;
+    		Array<double>& IonFluxData_y = visData[0].vars[4+Ion.number_ion_species+3*ion+1]->data;
+    		Array<double>& IonFluxData_z = visData[0].vars[4+Ion.number_ion_species+3*ion+2]->data;
+    		Ion.getIonFluxDiffusive(IonFluxDiffusive_x,IonFluxDiffusive_y,IonFluxDiffusive_z,ion);
+    		fillData.copy(IonFluxDiffusive_x,IonFluxData_x);
+    		fillData.copy(IonFluxDiffusive_y,IonFluxData_y);
+    		fillData.copy(IonFluxDiffusive_z,IonFluxData_z);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_advective", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            
+            // x-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_x",ion+1);
+    		//IonFluxDiffusive[3*ion+0]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+0]->name==VisName);
+            // y-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_y",ion+1);
+    		//IonFluxDiffusive[3*ion+1]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+1]->name==VisName);
+            // z-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxAdvective_z",ion+1);
+    		//IonFluxDiffusive[3*ion+2]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+2]->name==VisName);
+
+    		Array<double>& IonFluxData_x = visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+0]->data;
+    		Array<double>& IonFluxData_y = visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+1]->data;
+    		Array<double>& IonFluxData_z = visData[0].vars[4+Ion.number_ion_species*(1+3)+3*ion+2]->data;
+    		Ion.getIonFluxAdvective(IonFluxAdvective_x,IonFluxAdvective_y,IonFluxAdvective_z,ion);
+    		fillData.copy(IonFluxAdvective_x,IonFluxData_x);
+    		fillData.copy(IonFluxAdvective_y,IonFluxData_y);
+    		fillData.copy(IonFluxAdvective_z,IonFluxData_z);
+    	}
+    }
     
+    if (vis_db->getWithDefault<bool>( "save_ion_flux_electrical", false )){
+    	for (size_t ion=0; ion<Ion.number_ion_species; ion++){
+            
+            // x-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_x",ion+1);
+    		//IonFluxDiffusive[3*ion+0]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+0]->name==VisName);
+            // y-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_y",ion+1);
+    		//IonFluxDiffusive[3*ion+1]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+1]->name==VisName);
+            // z-component of diffusive flux
+    		sprintf(VisName,"Ion%zu_FluxElectrical_z",ion+1);
+    		//IonFluxDiffusive[3*ion+2]->name = VisName;
+    		ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+2]->name==VisName);
+
+    		Array<double>& IonFluxData_x = visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+0]->data;
+    		Array<double>& IonFluxData_y = visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+1]->data;
+    		Array<double>& IonFluxData_z = visData[0].vars[4+Ion.number_ion_species*(1+6)+3*ion+2]->data;
+    		Ion.getIonFluxElectrical(IonFluxElectrical_x,IonFluxElectrical_y,IonFluxElectrical_z,ion);
+    		fillData.copy(IonFluxElectrical_x,IonFluxData_x);
+    		fillData.copy(IonFluxElectrical_y,IonFluxData_y);
+    		fillData.copy(IonFluxElectrical_z,IonFluxData_z);
+    	}
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_electric_field", false )){
+    	ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+9)+0]->name=="ElectricField_x");
+    	ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+9)+1]->name=="ElectricField_y");
+    	ASSERT(visData[0].vars[4+Ion.number_ion_species*(1+9)+2]->name=="ElectricField_z");
+        Poisson.getElectricField(ElectricalField_x, ElectricalField_y, ElectricalField_z);
+    	Array<double>& ElectricalFieldxData = visData[0].vars[4+Ion.number_ion_species*(1+9)+0]->data;
+    	Array<double>& ElectricalFieldyData = visData[0].vars[4+Ion.number_ion_species*(1+9)+1]->data;
+    	Array<double>& ElectricalFieldzData = visData[0].vars[4+Ion.number_ion_species*(1+9)+2]->data;
+    	fillData.copy(ElectricalField_x,ElectricalFieldxData);
+    	fillData.copy(ElectricalField_y,ElectricalFieldyData);
+    	fillData.copy(ElectricalField_z,ElectricalFieldzData);
+    }
+
     if (vis_db->getWithDefault<bool>( "write_silo", true ))
     	IO::writeData( timestep, visData, Dm->Comm );
-
+    //--------------------------------------------------------------------------------------------------------------------
 /*    if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
     	char CurrentIDFilename[40];
     	sprintf(CurrentIDFilename,"id_t%d.raw",timestep);

analysis/ElectroChemistry.h

@@ -35,11 +35,23 @@ public:
 	DoubleArray Rho;	           // density field 
 	DoubleArray ChemicalPotential;	   // density field 
 	DoubleArray ElectricalPotential;	// density field 
+	DoubleArray ElectricalField_x;	// density field 
+	DoubleArray ElectricalField_y;	// density field 
+	DoubleArray ElectricalField_z;	// density field 
 	DoubleArray Pressure; 	// pressure field
 	DoubleArray Vel_x;		// velocity field
 	DoubleArray Vel_y;
 	DoubleArray Vel_z;
 	DoubleArray SDs;
+    DoubleArray IonFluxDiffusive_x; //ion diffusive flux components
+    DoubleArray IonFluxDiffusive_y;
+    DoubleArray IonFluxDiffusive_z;
+    DoubleArray IonFluxAdvective_x; //ion advective flux components
+    DoubleArray IonFluxAdvective_y;
+    DoubleArray IonFluxAdvective_z;
+    DoubleArray IonFluxElectrical_x; //ion electromigration flux components
+    DoubleArray IonFluxElectrical_y;
+    DoubleArray IonFluxElectrical_z;
 
 	ElectroChemistryAnalyzer(std::shared_ptr <Domain> Dm);
 	~ElectroChemistryAnalyzer();

analysis/FlowAdaptor.cpp

@@ -0,0 +1,514 @@
+/* Flow adaptor class for multiphase flow methods */
+
+#include "analysis/FlowAdaptor.h"
+#include "analysis/distance.h"
+#include "analysis/morphology.h"
+
+FlowAdaptor::FlowAdaptor(ScaLBL_ColorModel &M){
+	Nx = M.Dm->Nx;
+	Ny = M.Dm->Ny;
+	Nz = M.Dm->Nz;
+	timestep=-1;
+	timestep_previous=-1;	
+
+	phi.resize(Nx,Ny,Nz);         phi.fill(0);	    // phase indicator field
+	phi_t.resize(Nx,Ny,Nz);       phi_t.fill(0);	// time derivative for the phase indicator field
+}
+
+FlowAdaptor::~FlowAdaptor(){
+
+}
+
+double FlowAdaptor::ImageInit(ScaLBL_ColorModel &M, std::string Filename){
+	int rank = M.rank;
+	int Nx = M.Nx;  int Ny = M.Ny;  int Nz = M.Nz;
+	if (rank==0) printf("Re-initializing fluids from file: %s \n", Filename.c_str());
+	M.Mask->Decomp(Filename);
+	for (int i=0; i<Nx*Ny*Nz; i++) M.id[i] = M.Mask->id[i];  // save what was read
+	for (int i=0; i<Nx*Ny*Nz; i++) M.Dm->id[i] = M.Mask->id[i];  // save what was read
+
+	double *PhaseLabel;
+	PhaseLabel = new double[Nx*Ny*Nz];
+	M.AssignComponentLabels(PhaseLabel);
+
+	double Count = 0.0;
+	double PoreCount = 0.0;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				if (M.id[Nx*Ny*k+Nx*j+i] == 2){
+					PoreCount++;
+					Count++;
+				}
+				else if (M.id[Nx*Ny*k+Nx*j+i] == 1){
+					PoreCount++;						
+				}
+			}
+		}
+	}
+
+	Count=M.Dm->Comm.sumReduce(  Count);
+	PoreCount=M.Dm->Comm.sumReduce(  PoreCount);
+	
+	if (rank==0) printf("   new saturation: %f (%f / %f) \n", Count / PoreCount, Count, PoreCount);
+	ScaLBL_CopyToDevice(M.Phi, PhaseLabel, Nx*Ny*Nz*sizeof(double));
+	M.Dm->Comm.barrier();
+	
+	ScaLBL_D3Q19_Init(M.fq, M.Np);
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, 0, M.ScaLBL_Comm->LastExterior(), M.Np);
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, M.ScaLBL_Comm->FirstInterior(), M.ScaLBL_Comm->LastInterior(), M.Np);
+	M.Dm->Comm.barrier();
+	
+	ScaLBL_CopyToHost(M.Averages->Phi.data(),M.Phi,Nx*Ny*Nz*sizeof(double));
+
+	double saturation = Count/PoreCount;
+	return saturation;
+}
+
+
+double FlowAdaptor::UpdateFractionalFlow(ScaLBL_ColorModel &M){	
+
+	double MASS_FRACTION_CHANGE = 0.006;
+	double FRACTIONAL_FLOW_EPSILON = 5e-6;
+	if (M.db->keyExists( "FlowAdaptor" )){
+		auto flow_db = M.db->getDatabase( "FlowAdaptor" );
+		MASS_FRACTION_CHANGE = flow_db->getWithDefault<double>( "mass_fraction_factor", 0.006);
+		FRACTIONAL_FLOW_EPSILON = flow_db->getWithDefault<double>( "fractional_flow_epsilon", 5e-6);
+	}
+	int Np = M.Np;
+	double dA, dB, phi;
+	double vx,vy,vz;
+	double mass_a, mass_b, mass_a_global, mass_b_global;
+
+	double *Aq_tmp, *Bq_tmp;
+	double *Vel_x, *Vel_y, *Vel_z, *Phase;
+
+	Aq_tmp = new double [7*Np];
+	Bq_tmp = new double [7*Np];
+	Phase = new double [Np];
+	Vel_x = new double [Np];
+	Vel_y = new double [Np];
+	Vel_z = new double [Np];
+
+	ScaLBL_CopyToHost(Aq_tmp, M.Aq, 7*Np*sizeof(double));
+	ScaLBL_CopyToHost(Bq_tmp, M.Bq, 7*Np*sizeof(double));
+	ScaLBL_CopyToHost(Vel_x, &M.Velocity[0], Np*sizeof(double));
+	ScaLBL_CopyToHost(Vel_y, &M.Velocity[Np], Np*sizeof(double));
+	ScaLBL_CopyToHost(Vel_z, &M.Velocity[2*Np], Np*sizeof(double));
+
+	int Nx = M.Nx;  int Ny = M.Ny;  int Nz = M.Nz;
+
+	mass_a = mass_b = 0.0;
+	double maxSpeed = 0.0;
+	double localMaxSpeed = 0.0;
+	/* compute mass change based on weights */
+	double sum_weights_A = 0.0;
+	double sum_weights_B = 0.0;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				int n=M.Map(i,j,k);
+				//double distance = M.Averages->SDs(i,j,k);
+				if (!(n<0) ){
+					dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
+					dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
+					phi = (dA - dB) / (dA + dB);
+					Phase[n] = phi;
+					mass_a += dA;
+					mass_b += dB;
+					vx = Vel_x[n];
+					vy = Vel_y[n];
+					vz = Vel_z[n];
+					double local_momentum = sqrt(vx*vx+vy*vy+vz*vz);
+					double local_weight = (FRACTIONAL_FLOW_EPSILON + local_momentum);
+					if (phi > 0.0){
+						sum_weights_A += local_weight*dA;
+					}
+					else {
+						sum_weights_B += local_weight*dB;						
+					}
+					if ( local_momentum > localMaxSpeed){
+						localMaxSpeed = local_momentum;
+					}
+				}
+			}
+		}
+	}
+	maxSpeed = M.Dm->Comm.maxReduce(localMaxSpeed);
+	mass_a_global = M.Dm->Comm.sumReduce(mass_a);
+	mass_b_global = M.Dm->Comm.sumReduce(mass_b);
+	double sum_weights_A_global = M.Dm->Comm.sumReduce(sum_weights_A);
+	double sum_weights_B_global = M.Dm->Comm.sumReduce(sum_weights_B);
+	sum_weights_A_global /= (FRACTIONAL_FLOW_EPSILON + maxSpeed);
+	sum_weights_B_global /= (FRACTIONAL_FLOW_EPSILON + maxSpeed);
+
+	//double  total_momentum_A = sqrt(vax_global*vax_global+vay_global*vay_global+vaz_global*vaz_global);
+	//double  total_momentum_B = sqrt(vbx_global*vbx_global+vby_global*vby_global+vbz_global*vbz_global);
+	/* compute the total mass change */
+	double TOTAL_MASS_CHANGE = MASS_FRACTION_CHANGE*(mass_a_global + mass_b_global);
+	if (fabs(TOTAL_MASS_CHANGE) > 0.1*mass_a_global )
+		TOTAL_MASS_CHANGE = 0.1*mass_a_global;
+	if (fabs(TOTAL_MASS_CHANGE) > 0.1*mass_b_global )
+		TOTAL_MASS_CHANGE = 0.1*mass_b_global;
+	
+	double MASS_FACTOR_A = TOTAL_MASS_CHANGE / sum_weights_A_global;
+	double MASS_FACTOR_B = TOTAL_MASS_CHANGE / sum_weights_B_global;
+
+	double LOCAL_MASS_CHANGE = 0.0;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				int n=M.Map(i,j,k);
+				if (!(n<0)){
+					phi = Phase[n];
+					vx = Vel_x[n];
+					vy = Vel_y[n];
+					vz = Vel_z[n];
+					double local_momentum = sqrt(vx*vx+vy*vy+vz*vz);
+					double local_weight = (FRACTIONAL_FLOW_EPSILON + local_momentum)/(FRACTIONAL_FLOW_EPSILON + maxSpeed);
+					/* impose ceiling for spurious currents */
+					//if (local_momentum > maxSpeed) local_momentum =  maxSpeed;
+					if (phi > 0.0){
+						LOCAL_MASS_CHANGE = MASS_FACTOR_A*local_weight;
+						Aq_tmp[n] -= 0.3333333333333333*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+2*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+3*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+4*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+5*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Aq_tmp[n+6*Np] -= 0.1111111111111111*LOCAL_MASS_CHANGE;
+						//DebugMassA[n] = (-1.0)*LOCAL_MASS_CHANGE;
+					}
+					else{
+						LOCAL_MASS_CHANGE = MASS_FACTOR_B*local_weight;
+						Bq_tmp[n] += 0.3333333333333333*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+2*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+3*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+4*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+5*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						Bq_tmp[n+6*Np] += 0.1111111111111111*LOCAL_MASS_CHANGE;
+						//DebugMassB[n] = LOCAL_MASS_CHANGE;
+					}
+				}
+			}
+		}
+	}
+
+	if (M.rank == 0) printf("Update Fractional Flow: change mass of fluid B by %f \n",TOTAL_MASS_CHANGE/mass_b_global);
+
+	// Need to initialize Aq, Bq, Den, Phi directly
+	//ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double));
+	ScaLBL_CopyToDevice(M.Aq, Aq_tmp, 7*Np*sizeof(double));
+	ScaLBL_CopyToDevice(M.Bq, Bq_tmp, 7*Np*sizeof(double));
+
+	return(TOTAL_MASS_CHANGE);
+}
+
+void FlowAdaptor::Flatten(ScaLBL_ColorModel &M){	
+
+	ScaLBL_D3Q19_Init(M.fq, M.Np);
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, 0, M.ScaLBL_Comm->LastExterior(), M.Np);
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, M.ScaLBL_Comm->FirstInterior(), M.ScaLBL_Comm->LastInterior(), M.Np);
+}
+
+double FlowAdaptor::MoveInterface(ScaLBL_ColorModel &M){	
+
+	double INTERFACE_CUTOFF = M.color_db->getWithDefault<double>( "move_interface_cutoff", 0.1 );
+	double MOVE_INTERFACE_FACTOR = M.color_db->getWithDefault<double>( "move_interface_factor", 10.0 );
+
+	ScaLBL_CopyToHost( phi.data(), M.Phi, Nx*Ny*Nz* sizeof( double ) );
+	/* compute the local derivative of phase indicator field */
+	double beta = M.beta;
+	double factor = 0.5/beta;
+	double total_interface_displacement = 0.0;
+	double total_interface_sites = 0.0;
+	for (int n=0; n<Nx*Ny*Nz; n++){
+		/* compute the distance to the interface */
+		double value1 = M.Averages->Phi(n);
+		double dist1 = factor*log((1.0+value1)/(1.0-value1));
+		double value2 = phi(n);
+		double dist2 = factor*log((1.0+value2)/(1.0-value2));
+		phi_t(n) = value2;
+		if (value1 < INTERFACE_CUTOFF && value1 > -1*INTERFACE_CUTOFF && value2 < INTERFACE_CUTOFF && value2 > -1*INTERFACE_CUTOFF ){
+			/* time derivative of distance */
+			double dxdt = 0.125*(dist2-dist1);
+			/* extrapolate to move the distance further */
+			double dist3 = dist2 + MOVE_INTERFACE_FACTOR*dxdt;
+			/* compute the new phase interface */
+			phi_t(n) = (2.f*(exp(-2.f*beta*(dist3)))/(1.f+exp(-2.f*beta*(dist3))) - 1.f);
+			total_interface_displacement += fabs(MOVE_INTERFACE_FACTOR*dxdt);
+			total_interface_sites += 1.0;
+		}
+	}
+	ScaLBL_CopyToDevice( M.Phi, phi_t.data(), Nx*Ny*Nz* sizeof( double ) );	
+	return total_interface_sites;
+}
+
+double FlowAdaptor::ShellAggregation(ScaLBL_ColorModel &M, const double target_delta_volume){
+
+	const RankInfoStruct rank_info(M.rank,M.nprocx,M.nprocy,M.nprocz);
+	auto rank = M.rank;
+	auto Nx = M.Nx;  auto Ny = M.Ny; auto Nz = M.Nz;
+	auto N = Nx*Ny*Nz;
+	double vF = 0.f;
+	double vS = 0.f;
+	double delta_volume;
+	double WallFactor = 1.0;
+	bool USE_CONNECTED_NWP = false;
+
+	DoubleArray phase(Nx,Ny,Nz);
+	IntArray phase_label(Nx,Ny,Nz);;
+	DoubleArray phase_distance(Nx,Ny,Nz);
+	Array<char> phase_id(Nx,Ny,Nz);
+	fillHalo<double> fillDouble(M.Dm->Comm,M.Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
+
+	// Basic algorithm to 
+	// 1. Copy phase field to CPU
+	ScaLBL_CopyToHost(phase.data(), M.Phi, N*sizeof(double));
+
+	double count = 0.f;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				if (phase(i,j,k) > 0.f && M.Averages->SDs(i,j,k) > 0.f) count+=1.f;
+			}
+		}
+	}
+	double volume_initial = M.Dm->Comm.sumReduce(  count);
+	double PoreVolume = M.Dm->Volume*M.Dm->Porosity();
+	/*ensure target isn't an absurdly small fraction of pore volume */
+	if (volume_initial < target_delta_volume*PoreVolume){
+		volume_initial = target_delta_volume*PoreVolume;
+	}
+
+	// 2. Identify connected components of phase field -> phase_label
+
+	double volume_connected = 0.0;
+	double second_biggest = 0.0;
+	if (USE_CONNECTED_NWP){
+		ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,M.Averages->SDs,vF,vS,phase_label,M.Dm->Comm);
+		M.Dm->Comm.barrier();
+
+		// only operate on component "0"ScaLBL_ColorModel &M, 
+		count = 0.0;
+
+		for (int k=0; k<Nz; k++){
+			for (int j=0; j<Ny; j++){
+				for (int i=0; i<Nx; i++){
+					int label = phase_label(i,j,k);
+					if (label == 0 ){
+						phase_id(i,j,k) = 0;
+						count += 1.0;
+					}
+					else 		
+						phase_id(i,j,k) = 1;
+					if (label == 1 ){
+						second_biggest += 1.0;
+					}
+				}
+			}
+		}	
+		volume_connected = M.Dm->Comm.sumReduce(  count);
+		second_biggest = M.Dm->Comm.sumReduce(  second_biggest);
+	}
+	else {
+		// use the whole NWP 
+		for (int k=0; k<Nz; k++){
+			for (int j=0; j<Ny; j++){
+				for (int i=0; i<Nx; i++){
+					if (M.Averages->SDs(i,j,k) > 0.f){
+						if (phase(i,j,k) > 0.f ){
+							phase_id(i,j,k) = 0;
+						}
+						else {
+							phase_id(i,j,k) = 1;
+						}
+					}
+					else {
+						phase_id(i,j,k) = 1;
+					}
+				}
+			}
+		}
+	}
+
+	// 3. Generate a distance map to the largest object -> phase_distance
+	CalcDist(phase_distance,phase_id,*M.Dm);
+
+	double temp,value;
+	double factor=0.5/M.beta;
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if (phase_distance(i,j,k) < 3.f ){
+					value = phase(i,j,k);
+					if (value > 1.f)   value=1.f;
+					if (value < -1.f)  value=-1.f;
+					// temp -- distance based on analytical form McClure, Prins et al, Comp. Phys. Comm.
+					temp = -factor*log((1.0+value)/(1.0-value));
+					/// use this approximation close to the object
+					if (fabs(value) < 0.8 && M.Averages->SDs(i,j,k) > 1.f ){
+						phase_distance(i,j,k) = temp;
+					}
+					// erase the original object
+					phase(i,j,k) = -1.0;
+				}
+			}
+		}
+	}
+
+
+	if (rank==0) printf("Pathway volume / next largest ganglion %f \n",volume_connected/second_biggest );
+
+	if (rank==0) printf("MorphGrow with target volume fraction change %f \n", target_delta_volume/volume_initial);
+	double target_delta_volume_incremental = target_delta_volume;
+	if (fabs(target_delta_volume) > 0.01*volume_initial)  
+		target_delta_volume_incremental = 0.01*volume_initial*target_delta_volume/fabs(target_delta_volume);
+	
+	delta_volume = MorphGrow(M.Averages->SDs,phase_distance,phase_id,M.Averages->Dm, target_delta_volume_incremental, WallFactor);
+
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				if (phase_distance(i,j,k) < 0.0 ) phase_id(i,j,k) = 0;
+				else 		     				  phase_id(i,j,k) = 1;
+				//if (phase_distance(i,j,k) < 0.0 ) phase(i,j,k) = 1.0;
+			}
+		}
+	}	
+
+	CalcDist(phase_distance,phase_id,*M.Dm); // re-calculate distance
+
+	// 5. Update phase indicator field based on new distnace
+	for (int k=0; k<Nz; k++){
+		for (int j=0; j<Ny; j++){
+			for (int i=0; i<Nx; i++){
+				double d = phase_distance(i,j,k);
+				if (M.Averages->SDs(i,j,k) > 0.f){
+					if (d < 3.f){
+						//phase(i,j,k) = -1.0;
+						phase(i,j,k) = (2.f*(exp(-2.f*M.beta*d))/(1.f+exp(-2.f*M.beta*d))-1.f);	
+					}
+				}
+			} 
+		}
+	}
+	fillDouble.fill(phase);
+
+	count = 0.f;
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				if (phase(i,j,k) > 0.f && M.Averages->SDs(i,j,k) > 0.f){
+					count+=1.f;
+				}
+			}
+		}
+	}
+	double volume_final= M.Dm->Comm.sumReduce(  count);
+
+	delta_volume = (volume_final-volume_initial);
+	if (rank == 0)  printf("Shell Aggregation: change fluid volume fraction by %f \n", delta_volume/volume_initial);
+	if (rank == 0)  printf("   new saturation =  %f \n", volume_final/(M.Mask->Porosity()*double((Nx-2)*(Ny-2)*(Nz-2)*M.nprocs)));
+
+	// 6. copy back to the device
+	//if (rank==0)  printf("MorphInit: copy data  back to device\n");
+	ScaLBL_CopyToDevice(M.Phi,phase.data(),N*sizeof(double));
+
+	// 7. Re-initialize phase field and density
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, 0, M.ScaLBL_Comm->LastExterior(), M.Np);
+	ScaLBL_PhaseField_Init(M.dvcMap, M.Phi, M.Den, M.Aq, M.Bq, M.ScaLBL_Comm->FirstInterior(), M.ScaLBL_Comm->LastInterior(), M.Np);
+	auto BoundaryCondition = M.BoundaryCondition;
+	if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){
+		if (M.Dm->kproc()==0){
+			ScaLBL_SetSlice_z(M.Phi,1.0,Nx,Ny,Nz,0);
+			ScaLBL_SetSlice_z(M.Phi,1.0,Nx,Ny,Nz,1);
+			ScaLBL_SetSlice_z(M.Phi,1.0,Nx,Ny,Nz,2);
+		}
+		if (M.Dm->kproc() == M.nprocz-1){
+			ScaLBL_SetSlice_z(M.Phi,-1.0,Nx,Ny,Nz,Nz-1);
+			ScaLBL_SetSlice_z(M.Phi,-1.0,Nx,Ny,Nz,Nz-2);
+			ScaLBL_SetSlice_z(M.Phi,-1.0,Nx,Ny,Nz,Nz-3);
+		}
+	}
+	return delta_volume;
+}
+
+
+double FlowAdaptor::SeedPhaseField(ScaLBL_ColorModel &M, const double seed_water_in_oil){
+  srand(time(NULL));
+  auto rank = M.rank;
+  auto Np = M.Np; 
+  double mass_loss =0.f;
+  double count =0.f;
+  double *Aq_tmp, *Bq_tmp;
+  
+  Aq_tmp = new double [7*Np];
+  Bq_tmp = new double [7*Np];
+
+  ScaLBL_CopyToHost(Aq_tmp, M.Aq, 7*Np*sizeof(double));
+  ScaLBL_CopyToHost(Bq_tmp, M.Bq, 7*Np*sizeof(double));
+  
+ 
+   for (int n=0; n < M.ScaLBL_Comm->LastExterior(); n++){
+    double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
+    double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
+    double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
+    double phase_id = (dA - dB) / (dA + dB);
+    if (phase_id > 0.0){
+      Aq_tmp[n] -= 0.3333333333333333*random_value;
+      Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
+      
+      Bq_tmp[n] += 0.3333333333333333*random_value;
+      Bq_tmp[n+Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
+    }
+    mass_loss += random_value*seed_water_in_oil;
+  }
+
+  for (int n=M.ScaLBL_Comm->FirstInterior(); n < M.ScaLBL_Comm->LastInterior(); n++){
+    double random_value = seed_water_in_oil*double(rand())/ RAND_MAX;
+    double dA = Aq_tmp[n] + Aq_tmp[n+Np]  + Aq_tmp[n+2*Np] + Aq_tmp[n+3*Np] + Aq_tmp[n+4*Np] + Aq_tmp[n+5*Np] + Aq_tmp[n+6*Np];
+    double dB = Bq_tmp[n] + Bq_tmp[n+Np]  + Bq_tmp[n+2*Np] + Bq_tmp[n+3*Np] + Bq_tmp[n+4*Np] + Bq_tmp[n+5*Np] + Bq_tmp[n+6*Np];
+    double phase_id = (dA - dB) / (dA + dB);
+    if (phase_id > 0.0){
+      Aq_tmp[n] -= 0.3333333333333333*random_value;
+      Aq_tmp[n+Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+2*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+3*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+4*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+5*Np] -= 0.1111111111111111*random_value;
+      Aq_tmp[n+6*Np] -= 0.1111111111111111*random_value;
+      
+      Bq_tmp[n] += 0.3333333333333333*random_value;
+      Bq_tmp[n+Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+2*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+3*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+4*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+5*Np] += 0.1111111111111111*random_value;
+      Bq_tmp[n+6*Np] += 0.1111111111111111*random_value;
+    }
+    mass_loss += random_value*seed_water_in_oil;
+  }
+
+  count= M.Dm->Comm.sumReduce(  count);
+  mass_loss= M.Dm->Comm.sumReduce(  mass_loss);
+  if (rank == 0) printf("Remove mass %f from %f voxels \n",mass_loss,count);
+
+  // Need to initialize Aq, Bq, Den, Phi directly
+  //ScaLBL_CopyToDevice(Phi,phase.data(),7*Np*sizeof(double));
+  ScaLBL_CopyToDevice(M.Aq, Aq_tmp, 7*Np*sizeof(double));
+  ScaLBL_CopyToDevice(M.Bq, Bq_tmp, 7*Np*sizeof(double));
+
+  return(mass_loss);
+}

analysis/FlowAdaptor.h

@@ -0,0 +1,90 @@
+/* Flow adaptor class for multiphase flow methods */
+
+#ifndef ScaLBL_FlowAdaptor_INC
+#define ScaLBL_FlowAdaptor_INC
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <iostream>
+#include <exception>
+#include <stdexcept>
+#include <fstream>
+
+#include "models/ColorModel.h"
+
+
+/**
+ * \class FlowAdaptor
+ * @brief 
+ * The FlowAdaptor class operates on a lattice Boltzmann model to alter the flow conditions
+ * 
+ */
+
+class FlowAdaptor{
+public:
+	
+
+    /**
+    * \brief Create a flow adaptor to operate on the LB model
+    * @param M        ScaLBL_ColorModel 
+    */
+	FlowAdaptor(ScaLBL_ColorModel &M);
+	
+    /**
+	* \brief Destructor
+	*/
+	~FlowAdaptor();
+	
+    /**
+    * \brief   Fast-forward interface motion
+    * \details Accelerate the movement of interfaces based on the time derivative
+    *       Optional keys to control behavior can be specified in the input database:
+    *          move_interface_cutoff  -- identifies the diffuse interface region
+    *          move_interface_factor -- determines how much to ``fast forward" 
+    * @param M        ScaLBL_ColorModel 
+    */
+	double MoveInterface(ScaLBL_ColorModel &M);
+	
+    /**
+     * \brief Image re-initialization
+     * \details Re-initialize LB simulation from image data
+     * @param M        ScaLBL_ColorModel 
+     * @param Filename    name of input file to be used to read image 
+    */
+	double ImageInit(ScaLBL_ColorModel &M, std::string Filename);
+	
+    /**
+     * \details Update volume fraction based on morphological algorithm. Dilation / erosion algorithm will be applied to 
+     * grow / shrink the phase regions
+     * @param M        ScaLBL_ColorModel 
+     * @param delta_volume    target change in volume fraction 
+    */
+	double ShellAggregation(ScaLBL_ColorModel &M, const double delta_volume);
+	
+    /**
+     * \details  Update fractional flow condition. Mass will be preferentially added or removed from 
+     * phase regions based on where flow is occurring
+     * @param M        ScaLBL_ColorModel 
+    */	double UpdateFractionalFlow(ScaLBL_ColorModel &M);
+
+    /**
+     * \brief image re-initialization
+     * \details Re-initialize LB simulation from image data  
+     * @param M        ScaLBL_ColorModel 
+     * @param seed_water_in_oil    controls amount of mass to randomly seed into fluids 
+    */
+	double SeedPhaseField(ScaLBL_ColorModel &M, const double seed_water_in_oil);
+	
+    /**
+     * \brief Re-initialize LB simulation
+     * @param M        ScaLBL_ColorModel 
+    */
+	void Flatten(ScaLBL_ColorModel &M);
+	DoubleArray phi;
+	DoubleArray phi_t;
+private:
+	int Nx, Ny, Nz;
+	int timestep;
+	int timestep_previous;
+};
+#endif

analysis/FreeEnergy.h

@@ -19,6 +19,14 @@
 #include "IO/Writer.h"
 #include "models/FreeLeeModel.h"
 
+/**
+ * \class FreeEnergyAnalyzer
+ *
+ * @brief 
+ * The FreeEnergyAnalyzer class is constructed to analyze the LBPM free energy model for liquid-gas systems
+ * 
+ */
+
 class FreeEnergyAnalyzer{
 public:
 	std::shared_ptr <Domain> Dm;

analysis/GreyPhase.cpp

@@ -19,6 +19,7 @@ GreyPhaseAnalysis::GreyPhaseAnalysis(std::shared_ptr <Domain> dm):
 	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
 	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
 	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	MobilityRatio.resize(Nx,Ny,Nz); MobilityRatio.fill(0);
 	//.........................................
 	
 	if (Dm->rank()==0){
@@ -89,14 +90,17 @@ void GreyPhaseAnalysis::Basic(){
 					double nB = Rho_w(n);				
 					double phi = (nA-nB)/(nA+nB);					
 					double porosity = Porosity(n);
-					Water_local.M += rho_w*nB*porosity;
-					Water_local.Px += porosity*rho_w*nB*Vel_x(n);
-					Water_local.Py += porosity*rho_w*nB*Vel_y(n);
-					Water_local.Pz += porosity*rho_w*nB*Vel_z(n);
-					Oil_local.M += rho_n*nA*porosity;
-					Oil_local.Px += porosity*rho_n*nA*Vel_x(n);
-					Oil_local.Py += porosity*rho_n*nA*Vel_y(n);
-					Oil_local.Pz += porosity*rho_n*nA*Vel_z(n);
+					double mobility_ratio = MobilityRatio(n);
+					
+					Water_local.M += nB*porosity;
+					Water_local.Px += porosity*(nA+nB)*Vel_x(n)*0.5*(1.0-mobility_ratio);
+					Water_local.Py += porosity*(nA+nB)*Vel_y(n)*0.5*(1.0-mobility_ratio);
+					Water_local.Pz += porosity*(nA+nB)*Vel_z(n)*0.5*(1.0-mobility_ratio);
+					
+					Oil_local.M += nA*porosity;
+					Oil_local.Px += porosity*(nA+nB)*Vel_x(n)*0.5*(1.0+mobility_ratio);
+					Oil_local.Py += porosity*(nA+nB)*Vel_y(n)*0.5*(1.0+mobility_ratio);
+					Oil_local.Pz += porosity*(nA+nB)*Vel_z(n)*0.5*(1.0+mobility_ratio);
 
 					if ( phi > 0.99 ){
 						Oil_local.p += Pressure(n);

analysis/GreyPhase.h

@@ -15,6 +15,14 @@
 #include "IO/Reader.h"
 #include "IO/Writer.h"
 
+
+/**
+ * \class GreyPhase
+ *
+ * @brief 
+ * The GreyPhase class tracks pressure, mass and momentum within a grey phase 
+ * 
+ */
 class GreyPhase{
  public:
   double p;
@@ -26,6 +34,14 @@ class GreyPhase{
  private:
 };
 
+
+/**
+ * \class GreyPhaseAnalysis
+ *
+ * @brief 
+ * The GreyPhaseAnalysis class is constructed to analyze the LBPM greyscale model
+ * 
+ */
 class GreyPhaseAnalysis{
 public:
 	std::shared_ptr <Domain> Dm;
@@ -55,6 +71,7 @@ public:
 	DoubleArray Vel_x;		// velocity field
 	DoubleArray Vel_y;
 	DoubleArray Vel_z;
+	DoubleArray MobilityRatio;
 
 	GreyPhaseAnalysis(std::shared_ptr <Domain> Dm);
 	~GreyPhaseAnalysis();

analysis/Minkowski.cpp

@@ -50,7 +50,7 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
 {
     PROFILE_START("ComputeScalar");
 	Xi = Ji = Ai = 0.0;
-	DECL object;
+	DCEL object;
 	int e1,e2,e3;
 	double s,s1,s2,s3;
 	double a1,a2,a3;

analysis/Minkowski.h

@@ -18,6 +18,14 @@
 #include "IO/Reader.h"
 #include "IO/Writer.h"
 
+/**
+ * \class Minkowski
+ *
+ * @brief 
+ * The Minkowski class is constructed to analyze the geometric properties of structures based on the Minkowski functionals
+ * 
+ */
+
 
 class Minkowski{
 	//...........................................................................
@@ -45,6 +53,7 @@ public:
 	int n_connected_components;
 	//...........................................................................
 	int Nx,Ny,Nz;
+
 	double V(){
 		return Vi;
 	}
@@ -59,15 +68,56 @@ public:
 	}
 		
 	//..........................................................................
+    /**
+    * \brief   Null constructor
+    */
 	Minkowski(){};//NULL CONSTRUCTOR
+	
+    /**
+    * \brief   Constructor based on an existing Domain
+    * @param Dm    - Domain structure 
+    */
 	Minkowski(std::shared_ptr <Domain> Dm);
 	~Minkowski();
+	
+    /**
+    * \brief   Compute scalar minkowski functionals	
+	 *  step 1. compute the distance to an object 
+	 *  step 2. construct dcel to represent the isosurface 
+	 *  step 3. compute the scalar Minkowski functionals
+	 * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+	 *    0 - labels the object
+	 *    1 - labels everything else
+	 */    
 	void MeasureObject();
+	
 	void MeasureObject(double factor, const DoubleArray &Phi);
+	
+    /**
+    * \details   Compute scalar minkowski functionals for connected part of a structure
+    *   step 1. compute connected components and extract largest region by volume
+    *  step 2. compute the distance to the connected part of the structure 
+    *  step 3. construct dcel to represent the isosurface 
+    *  step 4. compute the scalar Minkowski functionals
+    * THIS ALGORITHM ASSUMES THAT id() is populated with phase id to distinguish objects
+    *    0 - labels the object
+    *    1 - labels everything else
+    */    
 	int MeasureConnectedPathway();
+
 	int MeasureConnectedPathway(double factor, const DoubleArray &Phi);
+	
+    /**
+    * \brief   Compute scalar minkowski functionals
+    * \details Construct an isosurface and return the geometric invariants based on the triangulated list
+    * @param isovalue        - threshold value to use to determine iso-surface
+    * @param Field           - DoubleArray containing the field to threshold
+    */
 	void ComputeScalar(const DoubleArray& Field, const double isovalue);
 
+    /**
+    * \brief   print the scalar invariants
+    */
 	void PrintAll();
 
 };

analysis/SubPhase.cpp

@@ -23,6 +23,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
 	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
 	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
 	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	Dissipation.resize(Nx,Ny,Nz);   Dissipation.fill(0);
 	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
 	//.........................................
 
@@ -42,11 +43,12 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
 			//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
 			fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");				
 			fprintf(SUBPHASE,"pwc pwd pnc pnd ");						// pressures 
-			fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni ");				// mass 
-			fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x ");	// momentum 
-			fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y ");			
-			fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z ");			
+			fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni Msw Msn ");				// mass 
+			fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x Psw_x Psn_x ");	// momentum 
+			fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y Psw_y Psn_y ");			
+			fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z Psw_z Psn_z ");			
 			fprintf(SUBPHASE,"Kwc Kwd Kwi Knc Knd Kni ");				// kinetic energy
+			fprintf(SUBPHASE,"Dwc Dwd Dnc Dnd ");						// viscous dissipation
 			fprintf(SUBPHASE,"Vwc Awc Hwc Xwc ");					 	// wc region 
 			fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd ");					// wd region
 			fprintf(SUBPHASE,"Vnc Anc Hnc Xnc ");					 	// nc region
@@ -67,11 +69,12 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
 		//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
 		fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");				
 		fprintf(SUBPHASE,"pwc pwd pnc pnd ");						// pressures 
-		fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni ");				// mass 
-		fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x ");	// momentum 
-		fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y ");			
-		fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z ");			
+		fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni Msw Msn ");				// mass 
+		fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x Psw_x Psn_x ");	// momentum 
+		fprintf(SUBPHASE,"Pwc_y Pwd_y Pwi_y Pnc_y Pnd_y Pni_y Psw_y Psn_y ");			
+		fprintf(SUBPHASE,"Pwc_z Pwd_z Pwi_z Pnc_z Pnd_z Pni_z Psw_z Psn_z ");				
 		fprintf(SUBPHASE,"Kwc Kwd Kwi Knc Knd Kni ");				// kinetic energy
+		fprintf(SUBPHASE,"Dwc Dwd Dnc Dnd ");						// viscous dissipation		
 		fprintf(SUBPHASE,"Vwc Awc Hwc Xwc ");					 	// wc region 
 		fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd ");					// wd region
 		fprintf(SUBPHASE,"Vnc Anc Hnc Xnc ");					 	// nc region
@@ -111,11 +114,12 @@ void SubPhase::Write(int timestep)
 	if (Dm->rank()==0){
 		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction_global); 
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Py, gwd.Py, giwn.Pwy, gnc.Py, gnd.Py, giwn.Pny);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Pz, gwd.Pz, giwn.Pwz, gnc.Pz, gnd.Pz, giwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn, gifs.Mw, gifs.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx, gifs.Pwx, gifs.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Py, gwd.Py, giwn.Pwy, gnc.Py, gnd.Py, giwn.Pny, gifs.Pwy, gifs.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Pz, gwd.Pz, giwn.Pwz, gnc.Pz, gnd.Pz, giwn.Pnz, gifs.Pwz, gifs.Pnz);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.K, gwd.K, giwn.Kw, gnc.K, gnd.K, giwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.visc, gwd.visc, gnc.visc, gnd.visc);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.V, gwc.A, gwc.H, gwc.X);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",gwd.V, gwd.A, gwd.H, gwd.X, gwd.Nc);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gnc.V, gnc.A, gnc.H, gnc.X);
@@ -127,11 +131,12 @@ void SubPhase::Write(int timestep)
 	else{
 		fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Py, wd.Py, iwn.Pwy, nc.Py, nd.Py, iwn.Pny);
-		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Pz, wd.Pz, iwn.Pwz, nc.Pz, nd.Pz, iwn.Pnz);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn, ifs.Mw, ifs.Mn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx, ifs.Pwx, ifs.Pnx);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",wc.Py, wd.Py, iwn.Pwy, nc.Py, nd.Py, iwn.Pny, ifs.Pwy, ifs.Pny);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",wc.Pz, wd.Pz, iwn.Pwz, nc.Pz, nd.Pz, iwn.Pnz, ifs.Pwz, ifs.Pnz);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.K, wd.K, iwn.Kw, nc.K, nd.K, iwn.Kn);
+		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.visc, wd.visc, nc.visc, nd.visc);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.V, wc.A, wc.H, wc.X);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %i ",wd.V, wd.A, wd.H, wd.X, wd.Nc);
 		fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",nc.V, nc.A, nc.H, nc.X);
@@ -481,7 +486,7 @@ void SubPhase::Full(){
 	if (Dm->inlet_layers_z > 0 && Dm->kproc() == 0) kmin += Dm->inlet_layers_z; 
 	if (Dm->outlet_layers_z > 0 && Dm->kproc() == Dm->nprocz()-1) kmax -= Dm->outlet_layers_z; 
 	*/
-	nd.reset();	nc.reset(); wd.reset();	wc.reset();	iwn.reset();	iwnc.reset();
+	nd.reset();	nc.reset(); wd.reset();	wc.reset();	iwn.reset(); iwnc.reset(); ifs.reset();
 
  	Dm->CommunicateMeshHalo(Phi);
 	for (int k=1; k<Nz-1; k++){
@@ -496,6 +501,33 @@ void SubPhase::Full(){
 		}
 	}
  	Dm->CommunicateMeshHalo(DelPhi);
+ 	
+ 	
+ 	Dm->CommunicateMeshHalo(Vel_x);
+ 	Dm->CommunicateMeshHalo(Vel_y);
+ 	Dm->CommunicateMeshHalo(Vel_z);
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				// Compute velocity gradients using finite differences
+				double phi = Phi(i,j,k);
+				double nu = nu_n + 0.5*(1.0-phi)*(nu_w-nu_n);
+				double rho = rho_n + 0.5*(1.0-phi)*(rho_w-rho_n);
+				double ux = 0.5*(Vel_x(i+1,j,k) - Vel_x(i-1,j,k));
+				double uy = 0.5*(Vel_x(i,j+1,k) - Vel_x(i,j-1,k));
+				double uz = 0.5*(Vel_x(i,j,k+1) - Vel_x(i,j,k-1));
+				double vx = 0.5*(Vel_y(i+1,j,k) - Vel_y(i-1,j,k));
+				double vy = 0.5*(Vel_y(i,j+1,k) - Vel_y(i,j-1,k));
+				double vz = 0.5*(Vel_y(i,j,k+1) - Vel_y(i,j,k-1));
+				double wx = 0.5*(Vel_z(i+1,j,k) - Vel_z(i-1,j,k));
+				double wy = 0.5*(Vel_z(i,j+1,k) - Vel_z(i,j-1,k));
+				double wz = 0.5*(Vel_z(i,j,k+1) - Vel_z(i,j,k-1));
+				if (SDs(i,j,k) > 2.0){
+					Dissipation(i,j,k) = 2*rho*nu*( ux*ux + vy*vy + wz*wz + 0.5*(vx + uy)*(vx + uy)+ 0.5*(vz + wy)*(vz + wy)+ 0.5*(uz + wx)*(uz + wx));					
+				}
+			}
+		}
+	}
 
  	/*  Set up geometric analysis of each region */
 	
@@ -654,13 +686,33 @@ void SubPhase::Full(){
 					double ux = Vel_x(n);
 					double uy = Vel_y(n);
 					double uz = Vel_z(n);
+					double visc = Dissipation(n);
 					
-					if (DelPhi(n) > 1e-3 && SDs(n) < 3.0 ){
-						// film region
+					if (DelPhi(n) > 1e-3 ){
+						// get the normal vector 
 						double nx = 0.5*(Phi(i+1,j,k)-Phi(i-1,j,k));
 						double ny = 0.5*(Phi(i,j+1,k)-Phi(i,j-1,k));
 						double nz = 0.5*(Phi(i,j,k+1)-Phi(i,j,k-1));
-						InterfaceTransportMeasures(  beta,  rho_w,  rho_n,  nA, nB, nx, ny, nz, ux, uy, uz, iwn);
+						if (SDs(n) > 2.5){
+							// not a film region
+							InterfaceTransportMeasures(  beta,  rho_w,  rho_n,  nA, nB, nx, ny, nz, ux, uy, uz, iwn);
+						}
+						else{
+							// films that are close to the wetting fluid
+							if ( morph_w->distance(i,j,k) < 2.5 && phi > 0.0){
+								ifs.Mw += rho_w;
+								ifs.Pwx += rho_w*ux;
+								ifs.Pwy += rho_w*uy;
+								ifs.Pwz += rho_w*uz;
+							}
+							// films that are close to the NWP 
+							if ( morph_n->distance(i,j,k) < 2.5 && phi < 0.0){
+								ifs.Mn += rho_n;
+								ifs.Pnx += rho_n*ux;
+								ifs.Pny += rho_n*uy;
+								ifs.Pnz += rho_n*uz;
+							}
+						}
 					}
 					else if ( phi > 0.0){
 						if (morph_n->label(i,j,k) > 0 ){
@@ -691,6 +743,7 @@ void SubPhase::Full(){
 							nd.Py += nA*rho_n*uy;
 							nd.Pz += nA*rho_n*uz;
 							nd.K += nA*rho_n*(ux*ux + uy*uy + uz*uz);
+							nd.visc += visc;
 						}
 						else{
 							nA = 1.0;
@@ -699,6 +752,7 @@ void SubPhase::Full(){
 							nc.Py += nA*rho_n*uy;
 							nc.Pz += nA*rho_n*uz;
 							nc.K += nA*rho_n*(ux*ux + uy*uy + uz*uz);
+							nc.visc += visc;
 						}
 					}
 					else{
@@ -710,6 +764,7 @@ void SubPhase::Full(){
 							wd.Py += nB*rho_w*uy;
 							wd.Pz += nB*rho_w*uz;
 							wd.K += nB*rho_w*(ux*ux + uy*uy + uz*uz);
+							wd.visc += visc;
 						}
 						else{
 							nB = 1.0;
@@ -718,6 +773,7 @@ void SubPhase::Full(){
 							wc.Py += nB*rho_w*uy;
 							wc.Pz += nB*rho_w*uz;
 							wc.K += nB*rho_w*(ux*ux + uy*uy + uz*uz);
+							wc.visc += visc;
 						}
 					}
 				}
@@ -730,25 +786,29 @@ void SubPhase::Full(){
 	gnd.Py=Dm->Comm.sumReduce(  nd.Py);
 	gnd.Pz=Dm->Comm.sumReduce(  nd.Pz);
 	gnd.K=Dm->Comm.sumReduce(  nd.K);
+	gnd.visc=Dm->Comm.sumReduce(  nd.visc);
 
 	gwd.M=Dm->Comm.sumReduce(  wd.M);
 	gwd.Px=Dm->Comm.sumReduce(  wd.Px);
 	gwd.Py=Dm->Comm.sumReduce(  wd.Py);
 	gwd.Pz=Dm->Comm.sumReduce(  wd.Pz);
 	gwd.K=Dm->Comm.sumReduce(  wd.K);
+	gwd.visc=Dm->Comm.sumReduce(  wd.visc);
 	
 	gnc.M=Dm->Comm.sumReduce(  nc.M);
 	gnc.Px=Dm->Comm.sumReduce(  nc.Px);
 	gnc.Py=Dm->Comm.sumReduce(  nc.Py);
 	gnc.Pz=Dm->Comm.sumReduce(  nc.Pz);
 	gnc.K=Dm->Comm.sumReduce(  nc.K);
+	gnc.visc=Dm->Comm.sumReduce(  nc.visc);
 
 	gwc.M=Dm->Comm.sumReduce(  wc.M);
 	gwc.Px=Dm->Comm.sumReduce(  wc.Px);
 	gwc.Py=Dm->Comm.sumReduce(  wc.Py);
 	gwc.Pz=Dm->Comm.sumReduce(  wc.Pz);
 	gwc.K=Dm->Comm.sumReduce(  wc.K);
-	
+	gwc.visc=Dm->Comm.sumReduce(  wc.visc);
+
 	giwn.Mn=Dm->Comm.sumReduce(  iwn.Mn);
 	giwn.Pnx=Dm->Comm.sumReduce(  iwn.Pnx);
 	giwn.Pny=Dm->Comm.sumReduce(  iwn.Pny);
@@ -760,6 +820,15 @@ void SubPhase::Full(){
 	giwn.Pwz=Dm->Comm.sumReduce(  iwn.Pwz);
 	giwn.Kw=Dm->Comm.sumReduce(  iwn.Kw);
 	
+	gifs.Mn= Dm->Comm.sumReduce(  ifs.Mn);
+	gifs.Pnx=Dm->Comm.sumReduce(  ifs.Pnx);
+	gifs.Pny=Dm->Comm.sumReduce(  ifs.Pny);
+	gifs.Pnz=Dm->Comm.sumReduce(  ifs.Pnz);
+	gifs.Mw= Dm->Comm.sumReduce(  ifs.Mw);
+	gifs.Pwx=Dm->Comm.sumReduce(  ifs.Pwx);
+	gifs.Pwy=Dm->Comm.sumReduce(  ifs.Pwy);
+	gifs.Pwz=Dm->Comm.sumReduce(  ifs.Pwz);	
+	
 	// pressure averaging
 	gnc.p=Dm->Comm.sumReduce(  nc.p);
 	gnd.p=Dm->Comm.sumReduce(  nd.p);

analysis/SubPhase.h

@@ -22,10 +22,11 @@ class phase{
 public:
 	int Nc;
 	double p;
-	double M,Px,Py,Pz,K;
+	double M,Px,Py,Pz,K,visc;
 	double V,A,H,X;
 	void reset(){
 		p=M=Px=Py=Pz=K=0.0;
+		visc=0.0;
 		V=A=H=X=0.0;
 		Nc=1;
 	}
@@ -70,10 +71,12 @@ public:
 	// local entities
 	phase wc,wd,wb,nc,nd,nb,solid;
 	interface iwn,iwnc;
+	interface ifs;
 	
 	// global entities
 	phase gwc,gwd,gwb,gnc,gnd,gnb,gsolid;
 	interface giwn,giwnc;
+	interface gifs;
 	/* fluid-solid wetting interaction */
 	double total_wetting_interaction, count_wetting_interaction;
 	double total_wetting_interaction_global, count_wetting_interaction_global;
@@ -92,6 +95,7 @@ public:
 	DoubleArray Vel_x;		// velocity field
 	DoubleArray Vel_y;
 	DoubleArray Vel_z;
+	DoubleArray Dissipation;
 	DoubleArray SDs;
 
 	std::shared_ptr<Minkowski> morph_w;

analysis/analysis.h

@@ -22,8 +22,8 @@ typedef Array<BlobIDType> BlobIDArray;
  * @param[in] SignDist      SignDist
  * @param[in] vF            vF
  * @param[in] vS            vS
- * @param[in] S             S
  * @param[out] LocalBlobID  The ids of the blobs
+ * @param[in] periodic      Optional value
  * @return  Returns the number of blobs
  */
 int ComputeLocalBlobIDs( const DoubleArray& Phase, const DoubleArray& SignDist, 
@@ -48,12 +48,13 @@ int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &C
  * @param[in] nx            Number of elements in the x-direction
  * @param[in] ny            Number of elements in the y-direction
  * @param[in] nz            Number of elements in the z-direction
+ * @param[in] rank_info     MPI communication info
  * @param[in] Phase         Phase
  * @param[in] SignDist      SignDist
  * @param[in] vF            vF
  * @param[in] vS            vS
- * @param[in] S             S
- * @param[out] LocalBlobID  The ids of the blobs
+ * @param[out] GlobalBlobID The ids of the blobs
+ * @param[in] comm          MPI communicator
  * @return  Returns the number of blobs
  */
 int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, 
@@ -68,10 +69,11 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
  * @param[in] nx            Number of elements in the x-direction
  * @param[in] ny            Number of elements in the y-direction
  * @param[in] nz            Number of elements in the z-direction
- * @param[in] rank_in       MPI communication info
+ * @param[in] rank_info     MPI communication info
  * @param[in] PhaseID       Array that identifies the phases
  * @param[in] VALUE         Identifier for the phase to decompose
  * @param[out] GlobalBlobID The ids of the blobs for the phase
+ * @param[in] comm          The communicator to use
  * @return Return the number of components in the specified phase
  */
 int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
@@ -82,10 +84,8 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
  * @brief  Reorder the blobs
  * @details  Reorder the blobs based on the number of cells they contain
  *    largest first.
- * @param[in] nx            Number of elements in the x-direction
- * @param[in] ny            Number of elements in the y-direction
- * @param[in] nz            Number of elements in the z-direction
- * @param[in/out] ID        The ids of the blobs
+ * @param[in,out] ID        The ids of the blobs
+ * @param[in] comm          MPI communicator
  */
 void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm );
 
@@ -117,8 +117,12 @@ struct ID_map_struct {
  * @details  This functions computes the map of blob ids between iterations
  * @return  Returns the map of the blob ids.  Each final blob may have no source
  *    ids, one parent, or multiple parents.  Each src id may be a parent for multiple blobs.
+ * @param[in] nx            Number of elements in the x-direction
+ * @param[in] ny            Number of elements in the y-direction
+ * @param[in] nz            Number of elements in the z-direction
  * @param[in] ID1           The blob ids at the first timestep
  * @param[in] ID2           The blob ids at the second timestep
+ * @param[in] comm          The communicator to use
  */
 ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm );
 
@@ -127,7 +131,7 @@ ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, cons
  * @brief  Compute the new global ids based on the map
  * @details  This functions computes the time-consistent global ids for the
  *     current global id index
- * @param[in/out] map       The timestep mapping for the ids
+ * @param[in,out] map       The timestep mapping for the ids
  * @param[in] id_max        The globally largest id used previously
  * @param[out] new_ids      The newly renumbered blob ids (0:ids.max())
  */
@@ -139,9 +143,9 @@ void getNewIDs( ID_map_struct& map, BlobIDType& id_max, std::vector<BlobIDType>&
  * @details  This functions computes the map of blob ids between iterations.
  *    Note: we also update the map to reflect the new ids
  * @param[out] new_ids          The newly renumbered blob ids (0:ids.max())
- * @param[in/out] IDs           The blob ids to renumber
+ * @param[in,out] IDs           The blob ids to renumber
  */
-void renumberIDs( const std::vector<BlobIDType>& new_id_list, BlobIDArray& IDs );
+void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDArray& IDs );
 
 
 /*!

analysis/dcel.cpp

@@ -1,19 +1,19 @@
 #include "analysis/dcel.h"
 
-DECL::DECL(){
+DCEL::DCEL(){
 }
 
-DECL::~DECL(){
+DCEL::~DCEL(){
 	TriangleCount=0;
 	VertexCount=0;
 	
 }
 
-int DECL::Face(int index){
+int DCEL::Face(int index){
 	return FaceData[index];
 }
 
-void DECL::Write(){
+void DCEL::Write(){
 	int e1,e2,e3;
 	FILE *TRIANGLES;
 	TRIANGLES = fopen("triangles.stl","w");
@@ -32,7 +32,7 @@ void DECL::Write(){
 	fclose(TRIANGLES);
 }
 
-void DECL::LocalIsosurface(const DoubleArray& A, double value, const int i, const int j, const int k){
+void DCEL::LocalIsosurface(const DoubleArray& A, double value, const int i, const int j, const int k){
 	Point P,Q;
 	Point PlaceHolder;
 	Point C0,C1,C2,C3,C4,C5,C6,C7;
@@ -174,7 +174,7 @@ void DECL::LocalIsosurface(const DoubleArray& A, double value, const int i, cons
 	}
 	int nTris = TriangleCount;
 
-	// Now add the local values to the DECL data structure
+	// Now add the local values to the DCEL data structure
 	if (nTris>0){
 		FaceData.resize(TriangleCount);
 	    //printf("Construct halfedge structure... \n");
@@ -250,7 +250,7 @@ void DECL::LocalIsosurface(const DoubleArray& A, double value, const int i, cons
 	}
 }
 
-Point DECL::TriNormal(int edge)
+Point DCEL::TriNormal(int edge)
 {
 	Point P,Q,R;
 	Point U,V,W;
@@ -294,7 +294,7 @@ Point DECL::TriNormal(int edge)
 	return W;
 }
 
-double DECL::EdgeAngle(int edge)
+double DCEL::EdgeAngle(int edge)
 {
 	double angle;
 	double dotprod;
@@ -369,7 +369,7 @@ double DECL::EdgeAngle(int edge)
 
 void iso_surface(const Array<double>&Field, const double isovalue)
 {
-  DECL object;
+  DCEL object;
   int e1,e2,e3;
   FILE *TRIANGLES;
   TRIANGLES = fopen("isosurface.stl","w");

analysis/dcel.h

@@ -4,8 +4,9 @@
 #include <vector>
 #include "analysis/pmmc.h"
 
-/* 
-Doubly-connected edge list (DECL) 
+/**
+ * \class Vertex
+ * @brief store vertex for DCEL data structure
 */
 
 // Vertex structure
@@ -34,8 +35,10 @@ private:
 };
 
 
-// Halfedge structure
-// Face
+/**
+ * \class Halfedge
+ * @brief store half edge for DCEL data structure
+*/
 class Halfedge{
 public:
 	Halfedge() = default;
@@ -60,11 +63,14 @@ private:
 	std::vector<std::array<int,6>> d_data;
 };
 
-// DECL
-class DECL{
+/**
+ * \class DCEL
+ * @details doubly connected edge list data structure 
+*/
+class DCEL{
 public:
-	DECL();
-	~DECL();
+	DCEL();
+	~DCEL();
 	
 	int face();
 	Vertex vertex;

analysis/distance.h

@@ -24,6 +24,7 @@ inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.
  * @param[in] ID            Segmentation id
  * @param[in] Dm            Domain information
  * @param[in] periodic      Directions that are periodic
+ * @param[in] dx            Cell size
  */
 template<class TYPE>
 void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
@@ -36,6 +37,7 @@ void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
  * @param[in] ID            Domain id
  * @param[in] Dm            Domain information
  * @param[in] periodic      Directions that are periodic
+ * @param[in] dx            Cell size
  */
 void CalcVecDist( Array<Vec> &Distance, const Array<int> &ID, const Domain &Dm,
     const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );

analysis/filters.h

@@ -25,7 +25,10 @@ void Med3D( const Array<float> &Input, Array<float> &Output );
  * @details  This routine performs a non-linear local means filter
  * @param[in] Input     Input image
  * @param[in] Mean      Mean value
+ * @param[in] Distance  Distance
  * @param[out] Output   Output image
+ * @param[in] d
+ * @param[in] h
  */
 int NLM3D( const Array<float> &Input, Array<float> &Mean, 
     const Array<float> &Distance, Array<float> &Output, const int d, const float h);

analysis/imfilter.h

@@ -89,7 +89,7 @@ Array<TYPE> imfilter_separable( const Array<TYPE>& A, const std::vector<Array<TY
  *    multidimensional filter H.  The result B has the same size and class as A.
  *    This version works with separable filters and is more efficient than a single filter.
  * @param[in]  A            The input array (Nx,Ny,Nz)
- * @param[in]  H            The filter [2*Nhx+1,2*Nhy+1,...]
+ * @param[in]  Nh           The filter size
  * @param[in]  boundary     The boundary conditions to apply (ndim):
  *                          fixed     - Input array values outside the bounds of the array are
  *                                      implicitly assumed to have the value X
@@ -114,6 +114,7 @@ Array<TYPE> imfilter_separable( const Array<TYPE>& A, const std::vector<int>& Nh
  *    This version works with separable filters and is more efficient than a single filter.
  * @param[in]  A            The input array (Nx,Ny,Nz)
  * @param[in]  H            The filter [2*Nhx+1,2*Nhy+1,...]
+ * @param[in]  Nh           The filter size
  * @param[in]  boundary     The boundary conditions to apply (ndim):
  *                          fixed     - Input array values outside the bounds of the array are
  *                                      implicitly assumed to have the value X

analysis/morphology.cpp

@@ -24,6 +24,409 @@ inline void UnpackID(const int *list, int count, signed char *recvbuf, signed ch
 	}
 }
 
+Morphology::Morphology(){
+	
+	/* MPI tags*/
+	sendtag = recvtag = 1381;
+}
+
+Morphology::~Morphology(){
+	
+}
+
+void Morphology::Initialize(std::shared_ptr <Domain> Dm, DoubleArray &Distance){
+	/* Loop over all faces and determine overlaps */
+	size_t Nx = Dm->Nx;
+	size_t Ny = Dm->Ny;
+	size_t Nz = Dm->Nz;
+	size_t N = Nx*Ny*Nz;
+
+	int *tmpShift_x, *tmpShift_y, *tmpShift_z;
+	double *tmpDistance;
+	tmpShift_x = new int [N];
+	tmpShift_y= new int [N];
+	tmpShift_z = new int [N];
+	tmpDistance = new double [N];
+	
+	double distance, boundary_distance;
+	
+	/* Loop over the local sub-domain and create overlap lists for each neighboring sub-domain */
+	int sendLoc = 0;   // counter for the local sub-domain send values
+	int recvLoc = 0;   // counter for the local recv
+	//...................................................
+	/* x face */
+	sendCount = recvCount = 0;	
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to x boundary
+				boundary_distance = double(i-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = Nx + i-2;
+					tmpShift_y[sendCount] = j;
+					tmpShift_z[sendCount] = k;
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_X(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_x(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_x = sendLoc;
+	recvOffset_X = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_x = sendCount;
+	recvCount_X = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_X],recvCount,Dm->rank_X(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_x(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_X],recvCount,Dm->rank_X(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_x(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_X],recvCount,Dm->rank_X(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_x(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_X],recvCount,Dm->rank_X(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_x(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	//...................................................
+	/* X face */
+	sendCount = recvCount = 0;
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to x boundary
+				boundary_distance = double(Nx-i-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = (i)-(Nx-2);
+					tmpShift_y[sendCount] = j;
+					tmpShift_z[sendCount] = k;
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_x(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_X(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_X = sendLoc;
+	recvOffset_x = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_X = sendCount;
+	recvCount_x = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_x],recvCount,Dm->rank_x(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_X(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_x],recvCount,Dm->rank_x(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_X(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_x],recvCount,Dm->rank_x(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_X(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_x],recvCount,Dm->rank_x(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_X(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	
+	//...................................................
+	/* y face */
+	sendCount = recvCount = 0;
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to y boundary
+				boundary_distance = double(j-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = i;
+					tmpShift_y[sendCount] = Ny + j-2;
+					tmpShift_z[sendCount] = k;
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_Y(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_y(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_y = sendLoc;
+	recvOffset_Y = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_y = sendCount;
+	recvCount_Y = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_Y],recvCount,Dm->rank_Y(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_y(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_Y],recvCount,Dm->rank_Y(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_y(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_Y],recvCount,Dm->rank_Y(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_y(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_Y],recvCount,Dm->rank_Y(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_y(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	//...................................................
+	/* X face */
+	sendCount = recvCount = 0;
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to x boundary
+				boundary_distance = double(Ny-j-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = i;
+					tmpShift_y[sendCount] = j-(Ny-2);
+					tmpShift_z[sendCount] = k;
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_y(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_Y(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_Y = sendLoc;
+	recvOffset_y = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_Y = sendCount;
+	recvCount_y = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_y],recvCount,Dm->rank_y(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_Y(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_y],recvCount,Dm->rank_y(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_Y(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_y],recvCount,Dm->rank_y(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_Y(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_y],recvCount,Dm->rank_y(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_Y(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	
+	//...................................................
+	/* z face */
+	sendCount = recvCount = 0;
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to z boundary
+				boundary_distance = double(k-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = i;
+					tmpShift_y[sendCount] = j;
+					tmpShift_z[sendCount] = (Nz-2) + k;
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_Z(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_z(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_z = sendLoc;
+	recvOffset_Z = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_z = sendCount;
+	recvCount_Z = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_Z],recvCount,Dm->rank_Z(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_z(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_Z],recvCount,Dm->rank_Z(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_z(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_Z],recvCount,Dm->rank_Z(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_z(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_Z],recvCount,Dm->rank_Z(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_z(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	/* Z face */
+	sendCount = recvCount = 0;
+	for (size_t k=1; k<Nz-1; k++){
+		for (size_t j=1; j<Ny-1; j++){
+			for (size_t i=1; i<Nx-1; i++){
+				distance = Distance(i,j,k);
+				// Distance to x boundary
+				boundary_distance = double(Nz-k-1);
+				if (distance > boundary_distance){
+					tmpShift_x[sendCount] = i;
+					tmpShift_y[sendCount] = j;
+					tmpShift_z[sendCount] = k-(Nz-2);
+					tmpDistance[sendCount++] = distance;
+					int n = k*Nx*Ny + j*Nx + i;
+					sendID.push_back(n);
+				}
+			}
+		}
+	}
+	Dm->Comm.Irecv(&recvCount,1,Dm->rank_z(),recvtag+0);
+	Dm->Comm.send(&sendCount,1,Dm->rank_Z(),sendtag+0);	
+	Dm->Comm.barrier();
+	sendOffset_Z = sendLoc;
+	recvOffset_z = recvLoc;
+	sendLoc += sendCount;
+	recvLoc += recvCount;
+	sendCount_Z = sendCount;
+	recvCount_z = recvCount;
+	/* grow the arrays */
+	xShift.resize(recvLoc);
+	yShift.resize(recvLoc);
+	zShift.resize(recvLoc);
+	morphRadius.resize(recvLoc);
+	//..............................
+	/* send the morphological radius */
+	Dm->Comm.Irecv(&morphRadius[recvOffset_z],recvCount,Dm->rank_z(),recvtag+0);
+	Dm->Comm.send(&tmpDistance[0],sendCount,Dm->rank_Z(),sendtag+0);	
+	/* send the shift values */
+	Dm->Comm.Irecv(&xShift[recvOffset_z],recvCount,Dm->rank_z(),recvtag+1);
+	Dm->Comm.send(&tmpShift_x[0],sendCount,Dm->rank_Z(),sendtag+1);
+	Dm->Comm.Irecv(&yShift[recvOffset_z],recvCount,Dm->rank_z(),recvtag+2);
+	Dm->Comm.send(&tmpShift_y[0],sendCount,Dm->rank_Z(),sendtag+2);
+	Dm->Comm.Irecv(&zShift[recvOffset_z],recvCount,Dm->rank_z(),recvtag+3);
+	Dm->Comm.send(&tmpShift_z[0],sendCount,Dm->rank_Z(),sendtag+3);
+	Dm->Comm.barrier();	
+	//...................................................
+	
+	/* resize the send / recv lists */
+	sendCount = sendLoc;
+	recvCount = recvLoc;
+	sendList.resize(sendLoc);
+	recvList.resize(recvLoc);
+	localID.resize(sendCount);
+	nonlocalID.resize(recvCount);
+	
+	/*printf("  offset %i for send (x) %i \n", sendOffset_x, sendCount_x);
+	printf("  offset %i for send (X) %i \n", sendOffset_X, sendCount_X);
+	printf("  offset %i for send (y) %i \n", sendOffset_y, sendCount_y);
+	printf("  offset %i for send (Y) %i \n", sendOffset_Y, sendCount_Y);
+	printf("  offset %i for send (z) %i \n", sendOffset_z, sendCount_z);
+	printf("  offset %i for send (Z) %i \n", sendOffset_Z, sendCount_Z);
+	*/
+
+}
+
+int Morphology::GetOverlaps(std::shared_ptr <Domain> Dm, signed char *id, const signed char ErodeLabel, const signed char NewLabel){
+	
+	int Nx = Dm->Nx;
+	int Ny = Dm->Ny;
+	int Nz = Dm->Nz;
+	int LocalNumber=0;
+	int i,j,k,ii,jj,kk;
+	int imin,jmin,kmin,imax,jmax,kmax;
+
+	for (int idx=0; idx<sendCount; idx++){
+		int n = sendID[idx];
+		localID[idx] = id[n]; 
+	}
+	//printf("send x -- offset: %i, count: %i \n",sendOffset_x,sendCount_x);
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_X],recvCount_X,Dm->rank_x(),recvtag+2);
+	Dm->Comm.send(&localID[sendOffset_x],sendCount_x,Dm->rank_X(),sendtag+2);
+
+	//printf("send X \n");	
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_x],recvCount_x,Dm->rank_X(),recvtag+3);
+	Dm->Comm.send(&localID[sendOffset_X],sendCount_X,Dm->rank_x(),sendtag+3);
+	
+	//printf("send y \n");
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_Y],recvCount_Y,Dm->rank_y(),recvtag+4);
+	Dm->Comm.send(&localID[sendOffset_y],sendCount_y,Dm->rank_Y(),sendtag+4);
+	
+	//printf("send Y \n");
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_y],recvCount_y,Dm->rank_Y(),recvtag+5);
+	Dm->Comm.send(&localID[sendOffset_Y],sendCount_Y,Dm->rank_y(),sendtag+5);
+	
+	//printf("send z \n");
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_Z],recvCount_Z,Dm->rank_z(),recvtag+6);
+	Dm->Comm.send(&localID[sendOffset_z],sendCount_z,Dm->rank_Z(),sendtag+6);
+	
+	//printf("send Z \n");
+	Dm->Comm.Irecv(&nonlocalID[recvOffset_z],recvCount_z,Dm->rank_Z(),recvtag+7);
+	Dm->Comm.send(&localID[sendOffset_Z],sendCount_Z,Dm->rank_z(),sendtag+7);
+
+	for (int idx=0; idx<recvCount; idx++){
+		double radius = morphRadius[idx];
+		signed char label = nonlocalID[idx];
+		/* get the neighboring site index */
+		i = xShift[idx];
+		j = yShift[idx];
+		k = zShift[idx];
+		int Window = int(radius);
+		// loop over the window and update
+		if (label == NewLabel){
+			imin=max(1,i-Window);
+			jmin=max(1,j-Window);
+			kmin=max(1,k-Window);
+			imax=min(Nx-1,i+Window);
+			jmax=min(Ny-1,j+Window);
+			kmax=min(Nz-1,k+Window);
+			for (kk=kmin; kk<kmax; kk++){
+				for (jj=jmin; jj<jmax; jj++){
+					for (ii=imin; ii<imax; ii++){
+						int nn = kk*Nx*Ny+jj*Nx+ii;
+						double dsq = double((ii-i)*(ii-i)+(jj-j)*(jj-j)+(kk-k)*(kk-k));
+						if (id[nn] == ErodeLabel && dsq <= radius*radius){
+							LocalNumber+=1.0;
+							id[nn]=NewLabel;
+						}
+					}
+				}
+			}
+		}
+	}
+	Dm->Comm.barrier();
+
+	return LocalNumber;
+}
+
 //***************************************************************************************
 double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction, signed char ErodeLabel, signed char NewLabel){
 	// SignDist is the distance to the object that you want to constaing the morphological opening
@@ -60,6 +463,9 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	}
 	Dm->Comm.barrier();
 	
+	Morphology Structure;
+	Structure.Initialize(Dm,SignDist);
+	
 	// total Global is the number of nodes in the pore-space
 	totalGlobal = Dm->Comm.sumReduce( count );
 	maxdistGlobal = Dm->Comm.sumReduce( maxdist );
@@ -69,56 +475,6 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	if (rank==0) printf("Maximum pore size: %f \n",maxdistGlobal);
 	final_void_fraction = volume_fraction; //initialize
 
-	// Communication buffers
-	signed char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
-	signed char *sendID_xy, *sendID_yz, *sendID_xz, *sendID_Xy, *sendID_Yz, *sendID_xZ;
-	signed char *sendID_xY, *sendID_yZ, *sendID_Xz, *sendID_XY, *sendID_YZ, *sendID_XZ;
-	signed char *recvID_x, *recvID_y, *recvID_z, *recvID_X, *recvID_Y, *recvID_Z;
-	signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
-	signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
-	// send buffers
-	sendID_x = new signed char [Dm->sendCount("x")];
-	sendID_y = new signed char [Dm->sendCount("y")];
-	sendID_z = new signed char [Dm->sendCount("z")];
-	sendID_X = new signed char [Dm->sendCount("X")];
-	sendID_Y = new signed char [Dm->sendCount("Y")];
-	sendID_Z = new signed char [Dm->sendCount("Z")];
-	sendID_xy = new signed char [Dm->sendCount("xy")];
-	sendID_yz = new signed char [Dm->sendCount("yz")];
-	sendID_xz = new signed char [Dm->sendCount("xz")];
-	sendID_Xy = new signed char [Dm->sendCount("Xy")];
-	sendID_Yz = new signed char [Dm->sendCount("Yz")];
-	sendID_xZ = new signed char [Dm->sendCount("xZ")];
-	sendID_xY = new signed char [Dm->sendCount("xY")];
-	sendID_yZ = new signed char [Dm->sendCount("yZ")];
-	sendID_Xz = new signed char [Dm->sendCount("Xz")];
-	sendID_XY = new signed char [Dm->sendCount("XY")];
-	sendID_YZ = new signed char [Dm->sendCount("YZ")];
-	sendID_XZ = new signed char [Dm->sendCount("XZ")];
-	//......................................................................................
-	// recv buffers
-	recvID_x = new signed char [Dm->recvCount("x")];
-	recvID_y = new signed char [Dm->recvCount("y")];
-	recvID_z = new signed char [Dm->recvCount("z")];
-	recvID_X = new signed char [Dm->recvCount("X")];
-	recvID_Y = new signed char [Dm->recvCount("Y")];
-	recvID_Z = new signed char [Dm->recvCount("Z")];
-	recvID_xy = new signed char [Dm->recvCount("xy")];
-	recvID_yz = new signed char [Dm->recvCount("yz")];
-	recvID_xz = new signed char [Dm->recvCount("xz")];
-	recvID_Xy = new signed char [Dm->recvCount("Xy")];
-	recvID_xZ = new signed char [Dm->recvCount("xZ")];
-	recvID_xY = new signed char [Dm->recvCount("xY")];
-	recvID_yZ = new signed char [Dm->recvCount("yZ")];
-	recvID_Yz = new signed char [Dm->recvCount("Yz")];
-	recvID_Xz = new signed char [Dm->recvCount("Xz")];
-	recvID_XY = new signed char [Dm->recvCount("XY")];
-	recvID_YZ = new signed char [Dm->recvCount("YZ")];
-	recvID_XZ = new signed char [Dm->recvCount("XZ")];
-	//......................................................................................
-	int sendtag,recvtag;
-	sendtag = recvtag = 7;
-
 	int ii,jj,kk;
 	int imin,jmin,kmin,imax,jmax,kmax;
 	int Nx = nx;
@@ -138,12 +494,18 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	double Rcrit_old = maxdistGlobal;
 	double Rcrit_new = maxdistGlobal;
 
-	while (void_fraction_new > VoidFraction)
+	int numTry = 0;
+	int maxTry = 100;
+	while (void_fraction_new > VoidFraction && numTry < maxTry)
 	{
+		numTry++;
 		void_fraction_diff_old = void_fraction_diff_new;
 		void_fraction_old = void_fraction_new;
 		Rcrit_old = Rcrit_new;
 		Rcrit_new -= deltaR*Rcrit_old;
+		if (Rcrit_new < 0.5 ){
+			numTry = maxTry;
+		}
 		int Window=round(Rcrit_new);
 		if (Window == 0) Window = 1; // If Window = 0 at the begining, after the following process will have sw=1.0
 		// and sw<Sw will be immediately broken
@@ -178,66 +540,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 				}
 			}
 		}
-		// Pack and send the updated ID values
-		PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
-		PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
-		PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
-		PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
-		PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
-		PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
-		PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
-		PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
-		PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
-		PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
-		PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
-		PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
-		PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
-		PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
-		PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
-		PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
-		PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
-		PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
-		//......................................................................................
-		Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
-		Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
-		Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
-		Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
-		Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
-		Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
-		Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
-		Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
-		Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
-		Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
-		Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
-		Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
-		Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
-		Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
-		//......................................................................................
-		UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
-		UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
-		UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
-		UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
-		UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
-		UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
-		UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
-		UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
-		UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
-		UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
-		UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
-		UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
-		UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
-		UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
-		UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
-		UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
-		UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
-		UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
-		//......................................................................................
-
-		//double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
+		LocalNumber += Structure.GetOverlaps(Dm,id,ErodeLabel,NewLabel);
 
 		count = 0.f;
 		for (int k=1; k<Nz-1; k++){
@@ -253,10 +556,10 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 		countGlobal = Dm->Comm.sumReduce( count );
 		void_fraction_new = countGlobal/totalGlobal;
 		void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
-	/*	if (rank==0){
+		if (rank==0){
 			printf("     %f ",void_fraction_new);
 			printf("     %f\n",Rcrit_new);
-		} */
+		}
 	}
 
 	if (void_fraction_diff_new<void_fraction_diff_old){
@@ -275,30 +578,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
 	}
 	return final_void_fraction;
 }
-/*
-double morph_open()
-{
 
-	fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
-
-
-	GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
-
-	count = 0.f;
-	for (int k=1; k<Nz-1; k++){
-		for (int j=1; j<Ny-1; j++){
-			for (int i=1; i<Nx-1; i++){
-				n=k*Nx*Ny+j*Nx+i;
-				if (id[n] == 2){
-					count+=1.0;
-				}
-			}
-		}
-	}
-	countGlobal = Dm->Comm.sumReduce( count );
-	return countGlobal;
-}
-*/
 
 //***************************************************************************************
 double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction){
@@ -307,6 +587,9 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	// id is a labeled map
 	// Dm contains information about the domain structure
 	
+	signed char ErodeLabel = 2;
+	signed char NewLabel = 1;
+	
 	int nx = Dm->Nx;
 	int ny = Dm->Ny;
 	int nz = Dm->Nz;
@@ -319,6 +602,9 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	IntArray phase_label(nx,ny,nz);
 	Array<char> ID(nx,ny,nz);
 	fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{nx-2,ny-2,nz-2},{1,1,1},0,1);
+	
+	Morphology Structure;
+	Structure.Initialize(Dm,SignDist);
 
 	int n;
 	double final_void_fraction;
@@ -334,7 +620,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 				if ( SignDist(i,j,k) > maxdist) maxdist=SignDist(i,j,k);
 				if ( SignDist(i,j,k) > 0.0 ){
 					count += 1.0;
-					id[n]  = 2;
+					id[n]  = ErodeLabel;
 				}
 				ID(i,j,k) = id[n];
 			}
@@ -351,57 +637,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
 	if (rank==0) printf("Maximum pore size: %f \n",maxdistGlobal);
 
-	
-/*	// Communication buffers
-	signed char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
-	signed char *sendID_xy, *sendID_yz, *sendID_xz, *sendID_Xy, *sendID_Yz, *sendID_xZ;
-	signed char *sendID_xY, *sendID_yZ, *sendID_Xz, *sendID_XY, *sendID_YZ, *sendID_XZ;
-	signed char *recvID_x, *recvID_y, *recvID_z, *recvID_X, *recvID_Y, *recvID_Z;
-	signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
-	signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
-	// send buffers
-	sendID_x = new signed char [Dm->sendCount("x")];
-	sendID_y = new signed char [Dm->sendCount("y")];
-	sendID_z = new signed char [Dm->sendCount("z")];
-	sendID_X = new signed char [Dm->sendCount("X")];
-	sendID_Y = new signed char [Dm->sendCount("Y")];
-	sendID_Z = new signed char [Dm->sendCount("Z")];
-	sendID_xy = new signed char [Dm->sendCount("xy")];
-	sendID_yz = new signed char [Dm->sendCount("yz")];
-	sendID_xz = new signed char [Dm->sendCount("xz")];
-	sendID_Xy = new signed char [Dm->sendCount("Xy")];
-	sendID_Yz = new signed char [Dm->sendCount("Yz")];
-	sendID_xZ = new signed char [Dm->sendCount("xZ")];
-	sendID_xY = new signed char [Dm->sendCount("xY")];
-	sendID_yZ = new signed char [Dm->sendCount("yZ")];
-	sendID_Xz = new signed char [Dm->sendCount("Xz")];
-	sendID_XY = new signed char [Dm->sendCount("XY")];
-	sendID_YZ = new signed char [Dm->sendCount("YZ")];
-	sendID_XZ = new signed char [Dm->sendCount("XZ")];
-	//......................................................................................
-	// recv buffers
-	recvID_x = new signed char [Dm->recvCount("x")];
-	recvID_y = new signed char [Dm->recvCount("y")];
-	recvID_z = new signed char [Dm->recvCount("z")];
-	recvID_X = new signed char [Dm->recvCount("X")];
-	recvID_Y = new signed char [Dm->recvCount("Y")];
-	recvID_Z = new signed char [Dm->recvCount("Z")];
-	recvID_xy = new signed char [Dm->recvCount("xy")];
-	recvID_yz = new signed char [Dm->recvCount("yz")];
-	recvID_xz = new signed char [Dm->recvCount("xz")];
-	recvID_Xy = new signed char [Dm->recvCount("Xy")];
-	recvID_xZ = new signed char [Dm->recvCount("xZ")];
-	recvID_xY = new signed char [Dm->recvCount("xY")];
-	recvID_yZ = new signed char [Dm->recvCount("yZ")];
-	recvID_Yz = new signed char [Dm->recvCount("Yz")];
-	recvID_Xz = new signed char [Dm->recvCount("Xz")];
-	recvID_XY = new signed char [Dm->recvCount("XY")];
-	recvID_YZ = new signed char [Dm->recvCount("YZ")];
-	recvID_XZ = new signed char [Dm->recvCount("XZ")];
-	//......................................................................................
-	int sendtag,recvtag;
-	sendtag = recvtag = 7;
-*/
 	int ii,jj,kk;
 	int imin,jmin,kmin,imax,jmax,kmax;
 	int Nx = nx;
@@ -422,7 +657,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 	//	if (rank==0) printf("Max. distance =%f, Initial critical radius = %f \n",maxdistGlobal,Rcrit_new);
 	//}
 	Dm->Comm.barrier();
-
 	
 	FILE *DRAIN = fopen("morphdrain.csv","w");
 	fprintf(DRAIN,"sw radius\n");				
@@ -453,86 +687,36 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 							for (jj=jmin; jj<jmax; jj++){
 								for (ii=imin; ii<imax; ii++){
 									double dsq = double((ii-i)*(ii-i)+(jj-j)*(jj-j)+(kk-k)*(kk-k));
-									if (ID(ii,jj,kk) == 2 && dsq <= (Rcrit_new+1)*(Rcrit_new+1)){
+									if (ID(ii,jj,kk) == ErodeLabel && dsq <= (Rcrit_new+1)*(Rcrit_new+1)){
 										LocalNumber+=1.0;
 										//id[nn]=1;
-										ID(ii,jj,kk)=1;
+										ID(ii,jj,kk)=NewLabel;
+										id[kk*Nx*Ny+jj*Nx+ii] = NewLabel;
 									}
 								}
 							}
 						}
-
 					}
 					// move on
 				}
 			}
 		}
+		LocalNumber += Structure.GetOverlaps(Dm,id,ErodeLabel,NewLabel);
+
+		for(int k=1; k<Nz-1; k++){
+			for(int j=1; j<Ny-1; j++){
+				for(int i=1; i<Nx-1; i++){
+					ID(i,j,k) = id[k*Nx*Ny+j*Nx+i];
+				}
+			}
+		}
 		fillChar.fill(ID);
-		// Pack and send the updated ID values
-/*		PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
-		PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
-		PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
-		PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
-		PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
-		PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
-		PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
-		PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
-		PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
-		PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
-		PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
-		PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
-		PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
-		PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
-		PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
-		PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
-		PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
-		PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
-		//......................................................................................
-		Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
-		Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
-		Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
-		Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
-		Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
-		Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
-		Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
-		Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
-		Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
-		Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
-		Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
-		Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
-		Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
-		Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
-		Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
-		//......................................................................................
-		UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
-		UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
-		UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
-		UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
-		UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
-		UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
-		UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
-		UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
-		UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
-		UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
-		UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
-		UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
-		UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
-		UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
-		UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
-		UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
-		UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
-		UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
-		//......................................................................................
-		// double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
-		*/
+		Dm->Comm.barrier();
+
 		for (int k=0; k<nz; k++){
 			for (int j=0; j<ny; j++){
 				for (int i=0; i<nx; i++){
-					n=k*nx*ny+j*nx+i;
-					if (ID(i,j,k) == 1){
+					if (ID(i,j,k) == NewLabel){
 						phase(i,j,k) = 1.0;
 					}
 					else
@@ -551,50 +735,12 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
 				for (int i=0; i<nx; i++){
 					n=k*nx*ny+j*nx+i;
 					if (ID(i,j,k) == 1 && phase_label(i,j,k) > 1){
-						ID(i,j,k) = 2;
+						ID(i,j,k) = ErodeLabel;
 					}
 					id[n] = ID(i,j,k);
 				}
 			}
 		}
-		
-		
-		/*
-		* Extract only the connected part of NWP
-		for (int k=1; k<nz-1; k++){
-			for (int j=1; j<ny-1; j++){
-				for (int i=1; i<nx-1; i++){
-					n=k*nx*ny+j*nx+i;
-					if (id[n] == 2){
-						phase(i,j,k) = 1.0;
-					}
-					else if (id[n] == 1){
-						// nwp
-						phase(i,j,k) = -1.0;
-					}
-					else{i
-						// treat solid as WP since films can connect 
-						phase(i,j,k) = 1.0;
-					}
-				}
-			}
-		}
-		
-		ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
-		Dm->Comm.barrier();
-		
-		for (int k=1; k<nz-1; k++){
-			for (int j=1; j<ny-1; j++){
-				for (int i=1; i<nx-1; i++){
-					n=k*nx*ny+j*nx+i;
-					if (id[n] == 2 && phase_label(i,j,k) > 1){
-						id[n] = 20;
-					}
-				}
-			}
-		}
-		// done
-		*/
 
 		count = 0.f;
 		for (int k=1; k<nz-1; k++){

analysis/morphology.h

@@ -6,3 +6,93 @@
 double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction, signed char ErodeLabel, signed char ReplaceLabel);
 double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain> Dm, double VoidFraction);
 double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetVol, double WallFactor);
+
+#ifndef MORPHOLOGY_INC
+#define MORPHOLOGY_INC
+/**
+ * \class Morphology
+ * @brief 
+ * The Morphology class supports morphological operations on complex structures
+ * 
+ */
+class Morphology{
+public:    
+	/**
+	* \brief Create a flow adaptor to operate on the LB model
+	*/
+	Morphology();
+
+	/**
+	* \brief Destructor
+	*/
+	~Morphology();
+	
+	/**
+	* \brief Initialize morphology structure from distance map
+	* @param Dm       Domain structure 
+	* @param Distance   Signed distance to boundary of structure
+	*/
+	void Initialize(std::shared_ptr <Domain> Dm, DoubleArray &Distance);
+	
+	/**
+	* \brief  Find all sites such that the reach of the signed distance at the site  overlaps with a sub-domain boundary
+	* @param Dm    Domain structure 
+	* @param id    image labels
+	* @param ErodeLabel label to erode based on morphological operation
+	* @param NewLabel   label to assign based on morphological operation
+	*/
+	int GetOverlaps(std::shared_ptr <Domain> Dm, signed char *id, const signed char ErodeLabel, const signed char NewLabel);
+	
+    /*
+     *  data structures to store non-local morphological information 
+    */
+    std::vector<int> xShift, yShift, zShift;
+    std::vector<int> sendID;
+    std::vector<double> morphRadius; 
+    std::vector<unsigned char> localID;
+    std::vector<unsigned char> nonlocalID;
+    
+private:
+	int sendtag,recvtag;
+    
+    //......................................................................................
+    int sendCount, recvCount;
+    //......................................................................................
+    int sendOffset_x, sendOffset_y, sendOffset_z, sendOffset_X, sendOffset_Y, sendOffset_Z;
+    int sendOffset_xy, sendOffset_yz, sendOffset_xz, sendOffset_Xy, sendOffset_Yz, sendOffset_xZ;
+    int sendOffset_xY, sendOffset_yZ, sendOffset_Xz, sendOffset_XY, sendOffset_YZ, sendOffset_XZ;
+    int sendOffset_xyz, sendOffset_XYZ, sendOffset_xYz, sendOffset_XyZ;
+    int sendOffset_Xyz, sendOffset_xYZ, sendOffset_xyZ, sendOffset_XYz;
+    //......................................................................................
+    int recvOffset_x, recvOffset_y, recvOffset_z, recvOffset_X, recvOffset_Y, recvOffset_Z;
+    int recvOffset_xy, recvOffset_yz, recvOffset_xz, recvOffset_Xy, recvOffset_Yz, recvOffset_xZ;
+    int recvOffset_xY, recvOffset_yZ, recvOffset_Xz, recvOffset_XY, recvOffset_YZ, recvOffset_XZ;
+    int recvOffset_xyz, recvOffset_XYZ, recvOffset_xYz, recvOffset_XyZ;
+    int recvOffset_Xyz, recvOffset_xYZ, recvOffset_xyZ, recvOffset_XYz;    
+    //......................................................................................
+    int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
+    int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
+    int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
+    int sendCount_xyz, sendCount_XYZ, sendCount_xYz, sendCount_XyZ;
+    int sendCount_Xyz, sendCount_xYZ, sendCount_xyZ, sendCount_XYz;
+    //......................................................................................
+    int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
+    int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
+    int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
+    int recvCount_xyz, recvCount_XYZ, recvCount_xYz, recvCount_XyZ;
+    int recvCount_Xyz, recvCount_xYZ, recvCount_xyZ, recvCount_XYz;    
+    //......................................................................................
+    std::vector<char> sendList;
+    std::vector<char> recvList;
+    //......................................................................................
+    
+	// Communication buffers
+	signed char *sendID_x, *sendID_y, *sendID_z, *sendID_X, *sendID_Y, *sendID_Z;
+	signed char *sendID_xy, *sendID_yz, *sendID_xz, *sendID_Xy, *sendID_Yz, *sendID_xZ;
+	signed char *sendID_xY, *sendID_yZ, *sendID_Xz, *sendID_XY, *sendID_YZ, *sendID_XZ;
+	signed char *recvID_x, *recvID_y, *recvID_z, *recvID_X, *recvID_Y, *recvID_Z;
+	signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
+	signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
+};
+
+#endif

analysis/runAnalysis.cpp

@@ -296,15 +296,21 @@ public:
             fillData.copy( Averages.Vel_z, VelzData );
         }
 
+        if ( vis_db->getWithDefault<bool>( "save_dissipation", false ) ) {
+            ASSERT( visData[0].vars[5]->name == "ViscousDissipation" );
+            Array<double> &ViscousDissipation = visData[0].vars[5]->data;
+            fillData.copy( Averages.Dissipation, ViscousDissipation );
+        }
+
         if ( vis_db->getWithDefault<bool>( "save_distance", false ) ) {
-            ASSERT( visData[0].vars[5]->name == "SignDist" );
-            Array<double> &SignData = visData[0].vars[5]->data;
+            ASSERT( visData[0].vars[6]->name == "SignDist" );
+            Array<double> &SignData = visData[0].vars[6]->data;
             fillData.copy( Averages.SDs, SignData );
         }
 
         if ( vis_db->getWithDefault<bool>( "save_connected_components", false ) ) {
-            ASSERT( visData[0].vars[6]->name == "BlobID" );
-            Array<double> &BlobData = visData[0].vars[6]->data;
+            ASSERT( visData[0].vars[7]->name == "BlobID" );
+            Array<double> &BlobData = visData[0].vars[7]->data;
             fillData.copy( Averages.morph_n->label, BlobData );
         }
 
@@ -618,8 +624,8 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStru
     d_N[1] = Dm->Ny;
     d_N[2] = Dm->Nz;
 
-    d_restart_interval           = db->getScalar<int>( "restart_interval" );
-    d_analysis_interval          = db->getScalar<int>( "analysis_interval" );
+    d_restart_interval           = db->getWithDefault<int>( "restart_interval", 100000 );
+    d_analysis_interval          = db->getWithDefault<int>( "analysis_interval", 1000 );
     d_subphase_analysis_interval = INT_MAX;
     d_visualization_interval     = INT_MAX;
     d_blobid_interval            = INT_MAX;
@@ -633,7 +639,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStru
         d_subphase_analysis_interval = db->getScalar<int>( "subphase_analysis_interval" );
     }
 
-    auto restart_file = db->getScalar<std::string>( "restart_file" );
+    auto restart_file = db->getWithDefault<std::string>( "restart_file", "Restart");
     d_restartFile     = restart_file + "." + rankString;
 
 
@@ -652,6 +658,7 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStru
     auto VxVar       = std::make_shared<IO::Variable>();
     auto VyVar       = std::make_shared<IO::Variable>();
     auto VzVar       = std::make_shared<IO::Variable>();
+    auto ViscousDissipationVar = std::make_shared<IO::Variable>();
     auto SignDistVar = std::make_shared<IO::Variable>();
     auto BlobIDVar   = std::make_shared<IO::Variable>();
 
@@ -688,6 +695,14 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStru
         VzVar->data.resize( d_n[0], d_n[1], d_n[2] );
         d_meshData[0].vars.push_back( VzVar );
     }
+    
+    if ( vis_db->getWithDefault<bool>( "save_dissipation", false ) ) {
+        ViscousDissipationVar->name = "ViscousDissipation";
+    	ViscousDissipationVar->type = IO::VariableType::VolumeVariable;
+    	ViscousDissipationVar->dim  = 1;
+    	ViscousDissipationVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( ViscousDissipationVar );
+    }
 
     if ( vis_db->getWithDefault<bool>( "save_distance", false ) ) {
         SignDistVar->name = "SignDist";
@@ -750,8 +765,8 @@ runAnalysis::runAnalysis(  ScaLBL_ColorModel &ColorModel)
     d_N[1] = ColorModel.Dm->Ny;
     d_N[2] = ColorModel.Dm->Nz;
 
-    d_restart_interval           = db->getScalar<int>( "restart_interval" );
-    d_analysis_interval          = db->getScalar<int>( "analysis_interval" );
+    d_restart_interval           = db->getWithDefault<int>( "restart_interval", 100000 );
+    d_analysis_interval          = db->getWithDefault<int>( "analysis_interval", 1000 );
     d_subphase_analysis_interval = INT_MAX;
     d_visualization_interval     = INT_MAX;
     d_blobid_interval            = INT_MAX;
@@ -765,10 +780,9 @@ runAnalysis::runAnalysis(  ScaLBL_ColorModel &ColorModel)
         d_subphase_analysis_interval = db->getScalar<int>( "subphase_analysis_interval" );
     }
 
-    auto restart_file = db->getScalar<std::string>( "restart_file" );
+    auto restart_file = db->getWithDefault<std::string>( "restart_file", "Restart");
     d_restartFile     = restart_file + "." + rankString;
-
-
+    
     d_rank = d_comm.getRank();
     writeIDMap( ID_map_struct(), 0, id_map_filename );
     // Initialize IO for silo

common/Array.cpp

@@ -1,117 +1,74 @@
+// clang-format off
 #include "common/Array.h"
 #include "common/Array.hpp"
+#include "common/Utilities.h"
 
 #include <complex>
 
 
-/********************************************************
- *  ArraySize                                            *
- ********************************************************/
-ArraySize::ArraySize( const std::vector<size_t>& N )
-{
-    d_ndim = N.size();
-    d_N[0] = 0;
-    d_N[1] = 1;
-    d_N[2] = 1;
-    d_N[3] = 1;
-    d_N[4] = 1;
-    for ( size_t i = 0; i < d_ndim; i++ )
-        d_N[i] = N[i];
-    d_length = 1;
-    for ( unsigned long i : d_N )
-        d_length *= i;
-    if ( d_ndim == 0 )
-        d_length = 0;
-}
-
-
 /********************************************************
  *  Explicit instantiations of Array                     *
  ********************************************************/
-template class Array<char, FunctionTable>;
-template class Array<uint8_t, FunctionTable>;
-template class Array<uint16_t, FunctionTable>;
-template class Array<uint32_t, FunctionTable>;
-template class Array<uint64_t, FunctionTable>;
-template class Array<int8_t, FunctionTable>;
-template class Array<int16_t, FunctionTable>;
-template class Array<int32_t, FunctionTable>;
-template class Array<int64_t, FunctionTable>;
-template class Array<float, FunctionTable>;
-template class Array<double, FunctionTable>;
+template class Array<char,FunctionTable>;
+template class Array<uint8_t,FunctionTable>;
+template class Array<uint16_t,FunctionTable>;
+template class Array<uint32_t,FunctionTable>;
+template class Array<uint64_t,FunctionTable>;
+template class Array<int8_t,FunctionTable>;
+template class Array<int16_t,FunctionTable>;
+template class Array<int32_t,FunctionTable>;
+template class Array<int64_t,FunctionTable>;
+template class Array<float,FunctionTable>;
+template class Array<double,FunctionTable>;
+template class Array<long double,FunctionTable>;
 
 
 /********************************************************
  *  Explicit instantiations of Array<bool>               *
  ********************************************************/
-// clang-format off
-template Array<bool, FunctionTable>::Array();
-template Array<bool, FunctionTable>::~Array();
-template Array<bool, FunctionTable>::Array( size_t );
-template Array<bool, FunctionTable>::Array( size_t, size_t );
-template Array<bool, FunctionTable>::Array( size_t, size_t, size_t );
-template Array<bool, FunctionTable>::Array( size_t, size_t, size_t, size_t );
-template Array<bool, FunctionTable>::Array( size_t, size_t, size_t, size_t, size_t );
-template Array<bool, FunctionTable>::Array( const std::vector<size_t>&, const bool* );
-template Array<bool, FunctionTable>::Array( std::string );
-template Array<bool, FunctionTable>::Array( std::initializer_list<bool> );
-template Array<bool, FunctionTable>::Array( const Array<bool, FunctionTable>& );
-template Array<bool, FunctionTable>::Array( Array<bool, FunctionTable>&& );
-template Array<bool, FunctionTable>& Array<bool, FunctionTable>::operator=( const Array<bool, FunctionTable>& );
-template Array<bool, FunctionTable>& Array<bool, FunctionTable>::operator=( Array<bool, FunctionTable>&& );
-template Array<bool, FunctionTable>& Array<bool, FunctionTable>::operator=( const std::vector<bool>& );
-template void Array<bool, FunctionTable>::fill(bool const&);
-template void Array<bool, FunctionTable>::clear();
-template bool Array<bool, FunctionTable>::operator==(Array<bool, FunctionTable> const&) const;
-template void Array<bool, FunctionTable>::resize( ArraySize const& );
-// clang-format on
+instantiateArrayConstructors( bool )
+template Array<bool,FunctionTable>& Array<bool,FunctionTable>::operator=( const std::vector<bool>& );
+template void Array<bool,FunctionTable>::clear();
+template bool Array<bool,FunctionTable>::operator==(Array<bool,FunctionTable> const&) const;
+template void Array<bool,FunctionTable>::resize( ArraySize const& );
 
 
 /********************************************************
  *  Explicit instantiations of Array<std::complex>       *
  ********************************************************/
-// clang-format off
-template Array<std::complex<double>, FunctionTable>::Array();
-template Array<std::complex<double>, FunctionTable>::~Array();
-template Array<std::complex<double>, FunctionTable>::Array( size_t );
-template Array<std::complex<double>, FunctionTable>::Array( size_t, size_t );
-template Array<std::complex<double>, FunctionTable>::Array( size_t, size_t, size_t );
-template Array<std::complex<double>, FunctionTable>::Array( size_t, size_t, size_t, size_t );
-template Array<std::complex<double>, FunctionTable>::Array( size_t, size_t, size_t, size_t, size_t );
-template Array<std::complex<double>, FunctionTable>::Array( const std::vector<size_t>&, const std::complex<double>* );
-template Array<std::complex<double>, FunctionTable>::Array( std::initializer_list<std::complex<double>> );
-template Array<std::complex<double>, FunctionTable>::Array( const Range<std::complex<double>>& range );
-template Array<std::complex<double>, FunctionTable>::Array( const Array<std::complex<double>, FunctionTable>& );
-template Array<std::complex<double>, FunctionTable>::Array( Array<std::complex<double>, FunctionTable>&& );
-template Array<std::complex<double>, FunctionTable>& Array<std::complex<double>, FunctionTable>::operator=( const Array<std::complex<double>, FunctionTable>& );
-template Array<std::complex<double>, FunctionTable>& Array<std::complex<double>, FunctionTable>::operator=( Array<std::complex<double>, FunctionTable>&& );
-template Array<std::complex<double>, FunctionTable>& Array<std::complex<double>, FunctionTable>::operator=( const std::vector<std::complex<double>>& );
-template void Array<std::complex<double>, FunctionTable>::resize( ArraySize const& );
-template void Array<std::complex<double>, FunctionTable>::viewRaw( ArraySize const&, std::complex<double>*, bool, bool );
-template void Array<std::complex<double>, FunctionTable>::fill(std::complex<double> const&);
-template void Array<std::complex<double>, FunctionTable>::clear();
-template bool Array<std::complex<double>, FunctionTable>::operator==(Array<std::complex<double>, FunctionTable> const&) const;
-template Array<std::complex<double>, FunctionTable> Array<std::complex<double>, FunctionTable>::repmat(std::vector<unsigned long, std::allocator<unsigned long> > const&) const;
-// clang-format on
+instantiateArrayConstructors( std::complex<float> )
+instantiateArrayConstructors( std::complex<double> )
+template void Array<std::complex<float>,FunctionTable>::resize( ArraySize const& );
+template void Array<std::complex<double>,FunctionTable>::resize( ArraySize const& );
+template Array<std::complex<double>,FunctionTable>& Array<std::complex<double>,FunctionTable>::operator=(std::vector<std::complex<double>> const&);
+template Array<std::complex<float>,FunctionTable>& Array<std::complex<float>,FunctionTable>::operator=(std::vector<std::complex<float>> const&);
+template void Array<std::complex<float>,FunctionTable>::clear();
+template void Array<std::complex<double>,FunctionTable>::clear();
+template bool Array<std::complex<float>,FunctionTable>::operator==(Array<std::complex<float>,FunctionTable> const&) const;
+template bool Array<std::complex<double>,FunctionTable>::operator==(Array<std::complex<double>,FunctionTable> const&) const;
+template Array<std::complex<float>,FunctionTable> Array<std::complex<float>,FunctionTable>::repmat(std::vector<unsigned long> const&) const;
+template Array<std::complex<double>,FunctionTable> Array<std::complex<double>,FunctionTable>::repmat(std::vector<unsigned long> const&) const;
+template void Array<std::complex<float>,FunctionTable>::copySubset(std::vector<unsigned long> const&, Array<std::complex<float>,FunctionTable> const&);
+template void Array<std::complex<double>,FunctionTable>::copySubset(std::vector<unsigned long> const&, Array<std::complex<double>,FunctionTable> const&);
+template Array<std::complex<float>,FunctionTable> Array<std::complex<float>,FunctionTable>::subset(std::vector<unsigned long> const&) const;
+template Array<std::complex<double>,FunctionTable> Array<std::complex<double>,FunctionTable>::subset(std::vector<unsigned long> const&) const;
+template bool Array<std::complex<float>,FunctionTable>::NaNs() const;
+template bool Array<std::complex<double>,FunctionTable>::NaNs() const;
 
 
 /********************************************************
  *  Explicit instantiations of Array<std::string>        *
  ********************************************************/
-// clang-format off
-template Array<std::string, FunctionTable>::Array();
-template Array<std::string, FunctionTable>::~Array();
-template Array<std::string, FunctionTable>::Array( size_t );
-template Array<std::string, FunctionTable>::Array( size_t, size_t );
-template Array<std::string, FunctionTable>::Array( size_t, size_t, size_t );
-template Array<std::string, FunctionTable>::Array( size_t, size_t, size_t, size_t );
-template Array<std::string, FunctionTable>::Array( size_t, size_t, size_t, size_t, size_t );
-template Array<std::string, FunctionTable>::Array( const std::vector<size_t>&, const std::string* );
-template Array<std::string, FunctionTable>::Array( std::initializer_list<std::string> );
-template Array<std::string, FunctionTable>::Array( const Array<std::string, FunctionTable>& );
-template Array<std::string, FunctionTable>::Array( Array<std::string, FunctionTable>&& );
-template Array<std::string, FunctionTable>& Array<std::string, FunctionTable>::operator=( const Array<std::string, FunctionTable>& );
-template Array<std::string, FunctionTable>& Array<std::string, FunctionTable>::operator=( Array<std::string, FunctionTable>&& );
-template Array<std::string, FunctionTable>& Array<std::string, FunctionTable>::operator=( const std::vector<std::string>& );
-template void Array<std::string, FunctionTable>::resize( ArraySize const& );
-// clang-format on
+instantiateArrayConstructors( std::string )
+template void Array<std::string,FunctionTable>::resize( ArraySize const& );
+template void Array<std::string,FunctionTable>::clear();
+template Array<std::string, FunctionTable> &Array<std::string, FunctionTable>::
+operator=( const std::vector<std::string> & );
+template bool Array<std::string>::operator==(Array<std::string> const&) const;
+
+
+#if defined( USING_ICC )
+ENABLE_WARNINGS
+#endif
+
+

common/Array.h

@@ -10,6 +10,7 @@
 #include <iostream>
 #include <memory>
 #include <stdint.h>
+#include <string>
 #include <vector>
 
 
@@ -76,13 +77,7 @@ public: // Constructors / assignment operators
      * @param N             Number of elements in each dimension
      * @param data          Optional raw array to copy the src data
      */
-    explicit Array( const std::vector<size_t> &N, const TYPE *data = NULL );
-
-    /*!
-     * Create a 1D Array with the range
-     * @param range         Range of the data
-     */
-    explicit Array( const Range<TYPE> &range );
+    explicit Array( const std::vector<size_t> &N, const TYPE *data = nullptr );
 
     /*!
      * Create a 1D Array using a string that mimic's MATLAB
@@ -96,6 +91,12 @@ public: // Constructors / assignment operators
      */
     Array( std::initializer_list<TYPE> data );
 
+    /*!
+     * Create a 2D Array with the given initializer lists
+     * @param data          Input data
+     */
+    Array( std::initializer_list<std::initializer_list<TYPE>> data );
+
 
     /*!
      * Copy constructor
@@ -146,7 +147,7 @@ public: // Views/copies/subset
      * @param N             Number of elements in each dimension
      * @param data          Pointer to the data
      */
-    static std::unique_ptr<Array> view( const ArraySize &N, std::shared_ptr<TYPE> &data );
+    static std::unique_ptr<Array> view( const ArraySize &N, std::shared_ptr<TYPE> data );
 
 
     /*!
@@ -154,8 +155,8 @@ public: // Views/copies/subset
      * @param N             Number of elements in each dimension
      * @param data          Pointer to the data
      */
-    static std::unique_ptr<const Array> constView(
-        const ArraySize &N, std::shared_ptr<const TYPE> const &data );
+    static std::unique_ptr<const Array> constView( const ArraySize &N,
+                                                   std::shared_ptr<const TYPE> const &data );
 
 
     /*!
@@ -169,7 +170,7 @@ public: // Views/copies/subset
      * @param N             Number of elements in each dimension
      * @param data          Pointer to the data
      */
-    void view2( const ArraySize &N, std::shared_ptr<TYPE> const &data );
+    void view2( const ArraySize &N, std::shared_ptr<TYPE> data );
 
     /*!
      * Make this object a view of the raw data (expert use only).
@@ -204,14 +205,30 @@ public: // Views/copies/subset
      */
     void viewRaw( const ArraySize &N, TYPE *data, bool isCopyable = true, bool isFixedSize = true );
 
+    /*!
+     * Create an array view of the given data (expert use only).
+     * Use view2( N, shared_ptr(data,[](TYPE*){}) ) instead.
+     *   Note: this interface is not recommended as it does not protect from
+     *   the src data being deleted while still being used by the Array.
+     *   Additionally for maximum performance it does not set the internal shared_ptr
+     *   so functions like getPtr and resize will not work correctly.
+     * @param N             Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    static inline Array staticView( const ArraySize &N, TYPE *data )
+    {
+        Array x;
+        x.viewRaw( N, data, true, true );
+        return x;
+    }
 
     /*!
      * Convert an array of one type to another.  This may or may not allocate new memory.
      * @param array         Input array
      */
     template<class TYPE2>
-    static inline std::unique_ptr<Array<TYPE2, FUN, Allocator>> convert(
-        std::shared_ptr<Array<TYPE, FUN, Allocator>> array )
+    static inline std::unique_ptr<Array<TYPE2, FUN, Allocator>>
+    convert( std::shared_ptr<Array<TYPE, FUN, Allocator>> array )
     {
         auto array2 = std::make_unique<Array<TYPE2>>( array->size() );
         array2.copy( *array );
@@ -224,8 +241,8 @@ public: // Views/copies/subset
      * @param array         Input array
      */
     template<class TYPE2>
-    static inline std::unique_ptr<const Array<TYPE2, FUN, Allocator>> convert(
-        std::shared_ptr<const Array<TYPE, FUN, Allocator>> array )
+    static inline std::unique_ptr<const Array<TYPE2, FUN, Allocator>>
+    convert( std::shared_ptr<const Array<TYPE, FUN, Allocator>> array )
     {
         auto array2 = std::make_unique<Array<TYPE2>>( array->size() );
         array2.copy( *array );
@@ -237,8 +254,8 @@ public: // Views/copies/subset
      * Copy and convert data from another array to this array
      * @param array         Source array
      */
-    template<class TYPE2>
-    void inline copy( const Array<TYPE2, FUN, Allocator> &array )
+    template<class TYPE2, class FUN2, class Allocator2>
+    void inline copy( const Array<TYPE2, FUN2, Allocator2> &array )
     {
         resize( array.size() );
         copy( array.data() );
@@ -247,51 +264,55 @@ public: // Views/copies/subset
     /*!
      * Copy and convert data from a raw vector to this array.
      *    Note: The current array must be allocated to the proper size first.
-     * @param array         Source array
+     * @param data          Source data
      */
     template<class TYPE2>
-    void inline copy( const TYPE2 *data )
-    {
-        for ( size_t i = 0; i < d_size.length(); i++ )
-            d_data[i] = static_cast<TYPE>( data[i] );
-    }
+    inline void copy( const TYPE2 *data );
 
     /*!
      * Copy and convert data from this array to a raw vector.
-     * @param array         Source array
+     * @param data          Source data
      */
     template<class TYPE2>
-    void inline copyTo( TYPE2 *data ) const
-    {
-        for ( size_t i = 0; i < d_size.length(); i++ )
-            data[i] = static_cast<TYPE2>( d_data[i] );
-    }
+    inline void copyTo( TYPE2 *data ) const;
 
     /*!
      * Copy and convert data from this array to a new array
      */
     template<class TYPE2>
-    Array<TYPE2, FUN, Allocator> inline cloneTo() const
+    Array<TYPE2, FUN, std::allocator<TYPE2>> inline cloneTo() const
     {
-        Array<TYPE2, FUN> dst( this->size() );
+        Array<TYPE2, FUN, std::allocator<TYPE2>> dst( this->size() );
         copyTo( dst.data() );
         return dst;
     }
 
+
     /*! swap the raw data pointers for the Arrays after checking for compatibility */
     void swap( Array &other );
 
+
     /*!
      * Fill the array with the given value
-     * @param value         Value to fill
+     * @param y         Value to fill
      */
-    void fill( const TYPE &value );
+    inline void fill( const TYPE &y )
+    {
+        for ( auto &x : *this )
+            x = y;
+    }
 
     /*!
      * Scale the array by the given value
-     * @param scale         Value to scale by
+     * @param y         Value to scale by
      */
-    void scale( const TYPE &scale );
+    template<class TYPE2>
+    inline void scale( const TYPE2 &y )
+    {
+        for ( auto &x : *this )
+            x *= y;
+    }
+
 
     /*!
      * Set the values of this array to pow(base, exp)
@@ -300,6 +321,7 @@ public: // Views/copies/subset
      */
     void pow( const Array &base, const TYPE &exp );
 
+
     //! Destructor
     ~Array();
 
@@ -328,6 +350,10 @@ public: // Views/copies/subset
     inline bool empty() const { return d_size.length() == 0; }
 
 
+    //! Return true if the Array is not empty
+    inline operator bool() const { return d_size.length() != 0; }
+
+
     /*!
      * Resize the Array
      * @param N             NUmber of elements
@@ -373,6 +399,12 @@ public: // Views/copies/subset
     void reshape( const ArraySize &N );
 
 
+    /*!
+     * Remove singleton dimensions.
+     */
+    void squeeze();
+
+
     /*!
      * Reshape the Array so that the number of dimensions is the
      *    max of ndim and the largest dim>1.
@@ -501,8 +533,8 @@ public: // Accessors
      * @param i3            The third index
      * @param i4            The fourth index
      */
-    ARRAY_ATTRIBUTE inline const TYPE &operator()(
-        size_t i1, size_t i2, size_t i3, size_t i4 ) const
+    ARRAY_ATTRIBUTE inline const TYPE &
+    operator()( size_t i1, size_t i2, size_t i3, size_t i4 ) const
     {
         return d_data[d_size.index( i1, i2, i3, i4 )];
     }
@@ -528,8 +560,8 @@ public: // Accessors
      * @param i4            The fourth index
      * @param i5            The fifth index
      */
-    ARRAY_ATTRIBUTE inline const TYPE &operator()(
-        size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const
+    ARRAY_ATTRIBUTE inline const TYPE &
+    operator()( size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const
     {
         return d_data[d_size.index( i1, i2, i3, i4, i5 )];
     }
@@ -602,11 +634,17 @@ public: // Math operations
     //! Concatenates the arrays along the dimension dim.
     static Array cat( const std::vector<Array> &x, int dim = 0 );
 
+    //! Concatenates the arrays along the dimension dim.
+    static Array cat( const std::initializer_list<Array> &x, int dim = 0 );
+
+    //! Concatenates the arrays along the dimension dim.
+    static Array cat( size_t N_array, const Array *x, int dim );
+
     //! Concatenates a given array with the current array
     void cat( const Array &x, int dim = 0 );
 
     //! Initialize the array with random values (defined from the function table)
-    void rand();
+    //void rand();
 
     //! Return true if NaNs are present
     bool NaNs() const;
@@ -657,20 +695,37 @@ public: // Math operations
     TYPE mean( const std::vector<Range<size_t>> &index ) const;
 
     //! Find all elements that match the operator
-    std::vector<size_t> find(
-        const TYPE &value, std::function<bool( const TYPE &, const TYPE & )> compare ) const;
+    std::vector<size_t> find( const TYPE &value,
+                              std::function<bool( const TYPE &, const TYPE & )> compare ) const;
 
 
     //! Print an array
-    void print(
-        std::ostream &os, const std::string &name = "A", const std::string &prefix = "" ) const;
-
-    //! Multiply two arrays
-    static Array multiply( const Array &a, const Array &b );
+    void
+    print( std::ostream &os, const std::string &name = "A", const std::string &prefix = "" ) const;
 
     //! Transpose an array
     Array reverseDim() const;
 
+    /*!
+     * @brief  Shift dimensions
+     * @details  Shifts the dimensions of the array by N.  When N is positive,
+     *    shiftDim shifts the dimensions to the left and wraps the
+     *    N leading dimensions to the end.  When N is negative,
+     *    shiftDim shifts the dimensions to the right and pads with singletons.
+     * @param N             Desired shift
+     */
+    Array shiftDim( int N ) const;
+
+    /*!
+     * @brief   Permute array dimensions
+     * @details  Rearranges the dimensions of the array so that they
+     *    are in the order specified by the vector index.
+     *    The array produced has the same values as A but the order of the subscripts
+     *    needed to access any particular element are rearranged as specified.
+     * @param index        Desired order of the subscripts
+     */
+    Array permute( const std::vector<uint8_t> &index ) const;
+
     //! Replicate an array a given number of times in each direction
     Array repmat( const std::vector<size_t> &N ) const;
 
@@ -678,8 +733,8 @@ public: // Math operations
     Array coarsen( const Array &filter ) const;
 
     //! Coarsen an array using the given filter
-    Array coarsen(
-        const std::vector<size_t> &ratio, std::function<TYPE( const Array & )> filter ) const;
+    Array coarsen( const std::vector<size_t> &ratio,
+                   std::function<TYPE( const Array & )> filter ) const;
 
     /*!
      * Perform a element-wise operation y = f(x)
@@ -694,8 +749,9 @@ public: // Math operations
      * @param[in] x             The first array
      * @param[in] y             The second array
      */
-    static Array transform(
-        std::function<TYPE( const TYPE &, const TYPE & )> fun, const Array &x, const Array &y );
+    static Array transform( std::function<TYPE( const TYPE &, const TYPE & )> fun,
+                            const Array &x,
+                            const Array &y );
 
     /*!
      * axpby operation: this = alpha*x + beta*this
@@ -709,7 +765,13 @@ public: // Math operations
      * Linear interpolation
      * @param[in] x             Position as a decimal index
      */
-    TYPE interp( const std::vector<double> &x ) const;
+    inline TYPE interp( const std::vector<double> &x ) const { return interp( x.data() ); }
+
+    /*!
+     * Linear interpolation
+     * @param[in] x             Position as a decimal index
+     */
+    TYPE interp( const double *x ) const;
 
     /**
      * \fn equals (Array & const rhs, TYPE tol )
@@ -732,8 +794,10 @@ private:
     inline void checkSubsetIndex( const std::vector<Range<size_t>> &range ) const;
     inline std::vector<Range<size_t>> convert( const std::vector<size_t> &index ) const;
     static inline void getSubsetArrays( const std::vector<Range<size_t>> &range,
-        std::array<size_t, 5> &first, std::array<size_t, 5> &last, std::array<size_t, 5> &inc,
-        std::array<size_t, 5> &N );
+                                        std::array<size_t, 5> &first,
+                                        std::array<size_t, 5> &last,
+                                        std::array<size_t, 5> &inc,
+                                        std::array<size_t, 5> &N );
 };
 
 
@@ -758,8 +822,8 @@ inline Array<TYPE, FUN, Allocator> operator+(
     const Array<TYPE, FUN, Allocator> &a, const Array<TYPE, FUN, Allocator> &b )
 {
     Array<TYPE, FUN, Allocator> c;
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a + b; };
-    FUN::transform( fun, a, b, c );
+    const auto &op = []( const TYPE &a, const TYPE &b ) { return a + b; };
+    FUN::transform( op, a, b, c );
     return c;
 }
 template<class TYPE, class FUN, class Allocator>
@@ -767,30 +831,78 @@ inline Array<TYPE, FUN, Allocator> operator-(
     const Array<TYPE, FUN, Allocator> &a, const Array<TYPE, FUN, Allocator> &b )
 {
     Array<TYPE, FUN, Allocator> c;
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a - b; };
-    FUN::transform( fun, a, b, c );
+    const auto &op = []( const TYPE &a, const TYPE &b ) { return a - b; };
+    FUN::transform( op, a, b, c );
     return c;
 }
 template<class TYPE, class FUN, class Allocator>
 inline Array<TYPE, FUN, Allocator> operator*(
     const Array<TYPE, FUN, Allocator> &a, const Array<TYPE, FUN, Allocator> &b )
 {
-    return Array<TYPE, FUN, Allocator>::multiply( a, b );
+    Array<TYPE, FUN, Allocator> c;
+    FUN::multiply( a, b, c );
+    return c;
 }
 template<class TYPE, class FUN, class Allocator>
 inline Array<TYPE, FUN, Allocator> operator*(
     const Array<TYPE, FUN, Allocator> &a, const std::vector<TYPE> &b )
 {
-    Array<TYPE, FUN, Allocator> b2;
+    Array<TYPE, FUN, Allocator> b2, c;
     b2.viewRaw( { b.size() }, const_cast<TYPE *>( b.data() ) );
-    return Array<TYPE, FUN, Allocator>::multiply( a, b2 );
+    FUN::multiply( a, b2, c );
+    return c;
+}
+template<class TYPE, class FUN, class Allocator>
+inline Array<TYPE, FUN, Allocator> operator*( const TYPE &a,
+                                              const Array<TYPE, FUN, Allocator> &b )
+{
+    auto c = b;
+    c.scale( a );
+    return c;
+}
+template<class TYPE, class FUN, class Allocator>
+inline Array<TYPE, FUN, Allocator> operator*( const Array<TYPE, FUN, Allocator> &a,
+                                              const TYPE &b )
+{
+    auto c = a;
+    c.scale( b );
+    return c;
+}
+
+
+/********************************************************
+ *  Copy array                                           *
+ ********************************************************/
+template<class TYPE, class FUN, class Allocator>
+template<class TYPE2>
+inline void Array<TYPE, FUN, Allocator>::copy( const TYPE2 *data )
+{
+    if ( std::is_same<TYPE, TYPE2>::value ) {
+        std::copy( data, data + d_size.length(), d_data );
+    } else {
+        for ( size_t i = 0; i < d_size.length(); i++ )
+            d_data[i] = static_cast<TYPE>( data[i] );
+    }
+}
+template<class TYPE, class FUN, class Allocator>
+template<class TYPE2>
+inline void Array<TYPE, FUN, Allocator>::copyTo( TYPE2 *data ) const
+{
+    if ( std::is_same<TYPE, TYPE2>::value ) {
+        std::copy( d_data, d_data + d_size.length(), data );
+    } else {
+        for ( size_t i = 0; i < d_size.length(); i++ )
+            data[i] = static_cast<TYPE2>( d_data[i] );
+    }
 }
 
 
 /********************************************************
  *  Convience typedefs                                   *
+ *  Copy array                                           *
  ********************************************************/
 typedef Array<double> DoubleArray;
 typedef Array<int> IntArray;
 
+
 #endif

common/Array.hpp

@@ -17,65 +17,46 @@
 /********************************************************
  *  External instantiations                              *
  ********************************************************/
-extern template class Array<char, FunctionTable>;
-extern template class Array<uint8_t, FunctionTable>;
-extern template class Array<uint16_t, FunctionTable>;
-extern template class Array<uint32_t, FunctionTable>;
-extern template class Array<uint64_t, FunctionTable>;
-extern template class Array<int8_t, FunctionTable>;
-extern template class Array<int16_t, FunctionTable>;
-extern template class Array<int32_t, FunctionTable>;
-extern template class Array<int64_t, FunctionTable>;
-extern template class Array<double, FunctionTable>;
-extern template class Array<float, FunctionTable>;
+extern template class Array<char>;
+extern template class Array<uint8_t>;
+extern template class Array<uint16_t>;
+extern template class Array<uint32_t>;
+extern template class Array<uint64_t>;
+extern template class Array<int8_t>;
+extern template class Array<int16_t>;
+extern template class Array<int32_t>;
+extern template class Array<int64_t>;
+extern template class Array<double>;
+extern template class Array<float>;
 
 
 /********************************************************
- *  Helper functions                                     *
+ *  Macros to help instantiate functions                 *
  ********************************************************/
-template<class TYPE>
-inline typename std::enable_if<std::is_integral<TYPE>::value, size_t>::type getRangeSize(
-    const Range<TYPE> &range )
-{
-    return ( static_cast<int64_t>( range.j ) - static_cast<int64_t>( range.i ) ) /
-           static_cast<int64_t>( range.k );
-}
-template<class TYPE>
-inline typename std::enable_if<std::is_floating_point<TYPE>::value, size_t>::type getRangeSize(
-    const Range<TYPE> &range )
-{
-    double tmp = static_cast<double>( ( range.j - range.i ) ) / static_cast<double>( range.k );
-    return static_cast<size_t>( floor( tmp + 1e-12 ) + 1 );
-}
-template<class TYPE>
-inline typename std::enable_if<std::is_same<TYPE, std::complex<float>>::value ||
-                                   std::is_same<TYPE, std::complex<double>>::value,
-    size_t>::type
-getRangeSize( const Range<TYPE> &range )
-{
-    double tmp = std::real( ( range.j - range.i ) / ( range.k ) );
-    return static_cast<size_t>( floor( tmp + 1e-12 ) + 1 );
-}
-template<class TYPE>
-inline typename std::enable_if<std::is_integral<TYPE>::value, TYPE>::type getRangeValue(
-    const Range<TYPE> &range, size_t index )
-{
-    return range.i + index * range.k;
-}
-template<class TYPE>
-inline typename std::enable_if<std::is_floating_point<TYPE>::value, TYPE>::type getRangeValue(
-    const Range<TYPE> &range, size_t index )
-{
-    return range.k * ( range.i / range.k + index );
-}
-template<class TYPE>
-inline typename std::enable_if<std::is_same<TYPE, std::complex<float>>::value ||
-                                   std::is_same<TYPE, std::complex<double>>::value,
-    TYPE>::type
-getRangeValue( const Range<TYPE> &range, size_t index )
-{
-    return range.k * ( range.i / range.k + static_cast<TYPE>( index ) );
-}
+// clang-format off
+#define instantiateArrayConstructors( TYPE )                                    \
+    template Array<TYPE>::Array();                                              \
+    template Array<TYPE>::~Array();                                             \
+    template Array<TYPE>::Array( const ArraySize & );                           \
+    template Array<TYPE>::Array( size_t );                                      \
+    template Array<TYPE>::Array( size_t, size_t );                              \
+    template Array<TYPE>::Array( size_t, size_t, size_t );                      \
+    template Array<TYPE>::Array( size_t, size_t, size_t, size_t );              \
+    template Array<TYPE>::Array( size_t, size_t, size_t, size_t, size_t );      \
+    template Array<TYPE>::Array( const std::vector<size_t> &, const TYPE * );   \
+    template Array<TYPE>::Array( std::initializer_list<TYPE> );                 \
+    template Array<TYPE>::Array( std::initializer_list<std::initializer_list<TYPE>> ); \
+    template Array<TYPE>::Array( const Array<TYPE> & );                         \
+    template Array<TYPE>::Array( Array<TYPE> && );                              \
+    template void Array<TYPE>::reshape( ArraySize const& );                     \
+    template void Array<TYPE>::squeeze();                                       \
+    template std::unique_ptr<const Array<TYPE>>                                 \
+        Array<TYPE>::constView(ArraySize const&, std::shared_ptr<TYPE const> const&); \
+    template void Array<TYPE>::viewRaw( ArraySize const&, TYPE*, bool, bool );  \
+    template void Array<TYPE>::view2(ArraySize const&, std::shared_ptr<TYPE> ); \
+    template Array<TYPE> &Array<TYPE>::operator=( const Array<TYPE> & );        \
+    template Array<TYPE> &Array<TYPE>::operator=( Array<TYPE> && );
+// clang-format on
 
 
 /********************************************************
@@ -126,19 +107,8 @@ Array<TYPE, FUN, Allocator>::Array( const std::vector<size_t> &N, const TYPE *da
     : d_isCopyable( true ), d_isFixedSize( false )
 {
     allocate( N );
-    if ( data ) {
-        for ( size_t i = 0; i < d_size.length(); i++ )
-            d_data[i] = data[i];
-    }
-}
-template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator>::Array( const Range<TYPE> &range )
-    : d_isCopyable( true ), d_isFixedSize( false )
-{
-    size_t N = getRangeSize( range );
-    allocate( { N } );
-    for ( size_t i = 0; i < N; i++ )
-        d_data[i] = getRangeValue( range, i );
+    if ( data )
+        copy( data );
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator>::Array( std::string str ) : d_isCopyable( true ), d_isFixedSize( false )
@@ -216,8 +186,12 @@ Array<TYPE, FUN, Allocator>::Array( std::string str ) : d_isCopyable( true ), d_
             i2 = str.length();
     }
     allocate( data.size() );
-    for ( size_t i = 0; i < data.size(); i++ )
-        d_data[i] = data[i];
+    if ( std::is_same<TYPE, bool>::value ) {
+        for ( size_t i = 0; i < data.size(); i++ )
+            d_data[i] = data[i];
+    } else {
+        copy( data.data() );
+    }
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator>::Array( std::initializer_list<TYPE> x )
@@ -229,19 +203,38 @@ Array<TYPE, FUN, Allocator>::Array( std::initializer_list<TYPE> x )
         d_data[i] = *it;
 }
 template<class TYPE, class FUN, class Allocator>
+Array<TYPE, FUN, Allocator>::Array( std::initializer_list<std::initializer_list<TYPE>> x )
+    : d_isCopyable( true ), d_isFixedSize( false )
+{
+    size_t Nx = x.size();
+    size_t Ny = 0;
+    for ( const auto y : x )
+        Ny = std::max<size_t>( Ny, y.size() );
+    allocate( { Nx, Ny } );
+    auto itx = x.begin();
+    for ( size_t i = 0; i < x.size(); ++i, ++itx ) {
+        auto ity = itx->begin();
+        for ( size_t j = 0; j < itx->size(); ++j, ++ity ) {
+            d_data[i + j * Nx] = *ity;
+        }
+    }
+}
+template<class TYPE, class FUN, class Allocator>
 void Array<TYPE, FUN, Allocator>::allocate( const ArraySize &N )
 {
     if ( d_isFixedSize )
         throw std::logic_error( "Array cannot be resized" );
     d_size      = N;
     auto length = d_size.length();
-    if ( length == 0 )
-        d_ptr.reset();
-    else
-        d_ptr.reset( new ( std::nothrow ) TYPE[length], []( TYPE *p ) { delete[] p; } );
-    d_data = d_ptr.get();
-    if ( length > 0 && d_data == nullptr )
-        throw std::logic_error( "Failed to allocate array" );
+    d_data      = nullptr;
+    if ( length > 0 ) {
+        try {
+            d_data = new TYPE[length];
+        } catch ( ... ) {
+            throw std::logic_error( "Failed to allocate array" );
+        }
+    }
+    d_ptr.reset( d_data, []( TYPE *p ) { delete[] p; } );
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator>::Array( const Array &rhs )
@@ -250,18 +243,19 @@ Array<TYPE, FUN, Allocator>::Array( const Array &rhs )
     if ( !rhs.d_isCopyable )
         throw std::logic_error( "Array cannot be copied" );
     allocate( rhs.size() );
-    for ( size_t i = 0; i < d_size.length(); i++ )
-        d_data[i] = rhs.d_data[i];
+    copy( rhs.d_data );
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator>::Array( Array &&rhs )
     : d_isCopyable( rhs.d_isCopyable ),
       d_isFixedSize( rhs.d_isFixedSize ),
       d_size( rhs.d_size ),
-      d_data( rhs.d_data )
+      d_data( rhs.d_data ),
+      d_ptr( std::move( rhs.d_ptr ) )
 {
+    rhs.d_size = ArraySize();
     rhs.d_data = nullptr;
-    d_ptr      = std::move( rhs.d_ptr );
+    rhs.d_ptr  = nullptr;
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator=( const Array &rhs )
@@ -270,9 +264,8 @@ Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator=( const Array
         return *this;
     if ( !rhs.d_isCopyable )
         throw std::logic_error( "Array cannot be copied" );
-    this->allocate( rhs.size() );
-    for ( size_t i = 0; i < d_size.length(); i++ )
-        this->d_data[i] = rhs.d_data[i];
+    allocate( rhs.size() );
+    copy( rhs.d_data );
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
@@ -285,15 +278,17 @@ Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator=( Array &&rhs
     d_size        = rhs.d_size;
     d_data        = rhs.d_data;
     d_ptr         = std::move( rhs.d_ptr );
+    rhs.d_size    = ArraySize();
     rhs.d_data    = nullptr;
+    rhs.d_ptr     = nullptr;
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator=( const std::vector<TYPE> &rhs )
 {
-    this->allocate( ArraySize( rhs.size() ) );
+    allocate( ArraySize( rhs.size() ) );
     for ( size_t i = 0; i < rhs.size(); i++ )
-        this->d_data[i] = rhs[i];
+        d_data[i] = rhs[i];
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
@@ -331,9 +326,9 @@ static inline void moveValues( const ArraySize &N1, const ArraySize &N2, TYPE *d
         }
     }
 }
-template<bool test, class TYPE>
-static inline typename std::enable_if<test, void>::type copyValues(
-    const ArraySize &N1, const ArraySize &N2, const TYPE *data1, TYPE *data2 )
+template<class TYPE>
+static inline void
+copyValues( const ArraySize &N1, const ArraySize &N2, const TYPE *data1, TYPE *data2 )
 {
     for ( size_t i5 = 0; i5 < std::min( N1[4], N2[4] ); i5++ ) {
         for ( size_t i4 = 0; i4 < std::min( N1[3], N2[3] ); i4++ ) {
@@ -349,12 +344,6 @@ static inline typename std::enable_if<test, void>::type copyValues(
         }
     }
 }
-template<bool test, class TYPE>
-static inline typename std::enable_if<!test, void>::type copyValues(
-    const ArraySize &, const ArraySize &, const TYPE *, TYPE * )
-{
-    throw std::logic_error( "No copy constructor" );
-}
 
 
 /********************************************************
@@ -381,9 +370,11 @@ void Array<TYPE, FUN, Allocator>::resize( const ArraySize &N )
         if ( data0.use_count() <= 1 ) {
             // We own the data, use std:move
             moveValues( N0, N, data0.get(), d_data );
-        } else {
+        } else if ( std::is_copy_constructible<TYPE>::value ) {
             // We do not own the data, copy
-            copyValues<std::is_copy_constructible<TYPE>::value, TYPE>( N0, N, data0.get(), d_data );
+            copyValues( N0, N, data0.get(), d_data );
+        } else {
+            throw std::logic_error( "No copy constructor" );
         }
     }
 }
@@ -412,7 +403,7 @@ void Array<TYPE, FUN, Allocator>::resizeDim( int dim, size_t N, const TYPE &valu
 
 
 /********************************************************
- *  Reshape the array                                     *
+ *  Reshape/squeeze the array                            *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
 void Array<TYPE, FUN, Allocator>::reshape( const ArraySize &N )
@@ -421,6 +412,85 @@ void Array<TYPE, FUN, Allocator>::reshape( const ArraySize &N )
         throw std::logic_error( "reshape is not allowed to change the array size" );
     d_size = N;
 }
+template<class TYPE, class FUN, class Allocator>
+void Array<TYPE, FUN, Allocator>::squeeze()
+{
+    d_size.squeeze();
+}
+
+
+/********************************************************
+ *  Shift/permute the array                              *
+ ********************************************************/
+template<class TYPE, class FUN, class Allocator>
+Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::shiftDim( int N ) const
+{
+    if ( N > 0 )
+        N = N % d_size.ndim();
+    if ( N == 0 ) {
+        // No shift required
+        return *this;
+    } else if ( N > 0 ) {
+        // Shift to the left and wrap
+        std::vector<uint8_t> index( d_size.ndim() );
+        size_t i = 0;
+        for ( size_t j=N; j<index.size(); j++, i++)
+            index[i] = j;
+        for ( size_t j=0; i<index.size(); j++, i++)
+            index[i] = j;
+        return permute( index );
+    } else  {
+        // Shift to the right (padding with singletons)
+        N = -N;
+        ASSERT( d_size.ndim() + N < (int) ArraySize::maxDim() );
+        size_t dims[10] = { 1 };
+        for ( int i = 0; i < ndim(); i++ )
+            dims[N+i] = d_size[i];
+        auto y = *this;
+        y.reshape( ArraySize( ndim() + N, dims ) );
+        return y;
+    }
+}
+template<class TYPE, class FUN, class Allocator>
+Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::permute( const std::vector<uint8_t> &index ) const
+{
+    // Check the permutation
+    ASSERT( (int) index.size() == ndim() );
+    for ( int i=0; i < ndim(); i++) {
+        ASSERT( index[i] < ndim() );
+        for ( int j=0; j < i; j++)
+            ASSERT( index[i] != index[j] );
+    }
+    // Create the new Array
+    size_t dims[5] = { 1u, 1u, 1u, 1u, 1u };
+    for ( size_t i=0; i<index.size(); i++)
+        dims[i] = d_size[index[i]];
+    Array y( ArraySize( ndim(), dims ) );
+    y.fill( -1 );
+    ASSERT( y.length() == this->length() );
+    // Fill the data
+    size_t N[5] = { 1u, 1u, 1u, 1u, 1u };
+    for ( int i=0; i < ndim(); i++) {
+        std::array<size_t, 5> ijk = { 0, 0, 0, 0, 0 };
+        ijk[index[i]] = 1;
+        N[i] = d_size.index( ijk );
+    }
+    size_t tmp = ( dims[0] - 1 ) * N[0] + ( dims[1] - 1 ) * N[1] + ( dims[2] - 1 ) * N[2] + ( dims[3] - 1 ) * N[3] + ( dims[4] - 1 ) * N[4] + 1;
+    ASSERT( tmp == length() );
+    for ( size_t i4 = 0; i4 < dims[4]; i4++ ) {
+        for ( size_t i3 = 0; i3 < dims[3]; i3++ ) {
+            for ( size_t i2 = 0; i2 < dims[2]; i2++ ) {
+                for ( size_t i1 = 0; i1 < dims[1]; i1++ ) {
+                    for ( size_t i0 = 0; i0 < dims[0]; i0++ ) {
+                        size_t index2 = i0 * N[0] + i1 * N[1] + i2 * N[2] + i3 * N[3] + i4 * N[4];
+                        y( i0, i1, i2, i3, i4 ) = d_data[index2];
+                    }
+                }
+            }
+        }
+    }
+    return y;
+}
 
 
 /********************************************************
@@ -428,8 +498,8 @@ void Array<TYPE, FUN, Allocator>::reshape( const ArraySize &N )
  ********************************************************/
 // Helper function to check subset indices
 template<class TYPE, class FUN, class Allocator>
-inline void Array<TYPE, FUN, Allocator>::checkSubsetIndex(
-    const std::vector<Range<size_t>> &range ) const
+inline void
+Array<TYPE, FUN, Allocator>::checkSubsetIndex( const std::vector<Range<size_t>> &range ) const
 {
     bool test = (int) range.size() == d_size.ndim();
     for ( size_t d = 0; d < range.size(); d++ )
@@ -438,8 +508,8 @@ inline void Array<TYPE, FUN, Allocator>::checkSubsetIndex(
         throw std::logic_error( "indices for subset are invalid" );
 }
 template<class TYPE, class FUN, class Allocator>
-std::vector<Range<size_t>> Array<TYPE, FUN, Allocator>::convert(
-    const std::vector<size_t> &index ) const
+std::vector<Range<size_t>>
+Array<TYPE, FUN, Allocator>::convert( const std::vector<size_t> &index ) const
 {
     std::vector<Range<size_t>> range( d_size.ndim() );
     if ( index.size() % 2 != 0 || static_cast<int>( index.size() / 2 ) < d_size.ndim() )
@@ -451,8 +521,10 @@ std::vector<Range<size_t>> Array<TYPE, FUN, Allocator>::convert(
 // Helper function to return dimensions for the subset array
 template<class TYPE, class FUN, class Allocator>
 void Array<TYPE, FUN, Allocator>::getSubsetArrays( const std::vector<Range<size_t>> &index,
-    std::array<size_t, 5> &first, std::array<size_t, 5> &last, std::array<size_t, 5> &inc,
-    std::array<size_t, 5> &N )
+                                                   std::array<size_t, 5> &first,
+                                                   std::array<size_t, 5> &last,
+                                                   std::array<size_t, 5> &inc,
+                                                   std::array<size_t, 5> &N )
 {
     first.fill( 0 );
     last.fill( 0 );
@@ -467,8 +539,8 @@ void Array<TYPE, FUN, Allocator>::getSubsetArrays( const std::vector<Range<size_
     }
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::subset(
-    const std::vector<Range<size_t>> &index ) const
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::subset( const std::vector<Range<size_t>> &index ) const
 {
     // Get the subset indicies
     checkSubsetIndex( index );
@@ -476,9 +548,8 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::subset(
     getSubsetArrays( index, first, last, inc, N1 );
     ArraySize S1( d_size.ndim(), N1.data() );
     // Create the new array
-    Array<TYPE> subset_array( S1 );
+    Array<TYPE, FUN, Allocator> subset_array( S1 );
     // Fill the new array
-    static_assert( ArraySize::maxDim() == 5, "Not programmed for more than 5 dimensions" );
     TYPE *subset_data = subset_array.data();
     for ( size_t i4 = first[4], k1 = 0; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
@@ -495,22 +566,21 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::subset(
     return subset_array;
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::subset(
-    const std::vector<size_t> &index ) const
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::subset( const std::vector<size_t> &index ) const
 {
     auto range = convert( index );
     return subset( range );
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::copySubset(
-    const std::vector<Range<size_t>> &index, const Array<TYPE, FUN, Allocator> &subset )
+void Array<TYPE, FUN, Allocator>::copySubset( const std::vector<Range<size_t>> &index,
+                                              const Array<TYPE, FUN, Allocator> &subset )
 {
     // Get the subset indices
     checkSubsetIndex( index );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( index, first, last, inc, N1 );
     // Copy the sub-array
-    static_assert( ArraySize::maxDim() == 5, "Not programmed for more than 5 dimensions" );
     const TYPE *src_data = subset.data();
     for ( size_t i4 = first[4], k1 = 0; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
@@ -526,15 +596,14 @@ void Array<TYPE, FUN, Allocator>::copySubset(
     }
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::addSubset(
-    const std::vector<Range<size_t>> &index, const Array<TYPE, FUN, Allocator> &subset )
+void Array<TYPE, FUN, Allocator>::addSubset( const std::vector<Range<size_t>> &index,
+                                             const Array<TYPE, FUN, Allocator> &subset )
 {
     // Get the subset indices
     checkSubsetIndex( index );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( index, first, last, inc, N1 );
     // add the sub-array
-    static_assert( ArraySize::maxDim() == 5, "Not programmed for more than 5 dimensions" );
     for ( size_t i4 = first[4], k1 = 0; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
             for ( size_t i2 = first[2]; i2 <= last[2]; i2 += inc[2] ) {
@@ -549,16 +618,16 @@ void Array<TYPE, FUN, Allocator>::addSubset(
     }
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::copySubset(
-    const std::vector<size_t> &index, const Array<TYPE, FUN, Allocator> &subset )
+void Array<TYPE, FUN, Allocator>::copySubset( const std::vector<size_t> &index,
+                                              const Array<TYPE, FUN, Allocator> &subset )
 {
     auto range = convert( index );
     copySubset( range, subset );
 }
 
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::addSubset(
-    const std::vector<size_t> &index, const Array<TYPE, FUN, Allocator> &subset )
+void Array<TYPE, FUN, Allocator>::addSubset( const std::vector<size_t> &index,
+                                             const Array<TYPE, FUN, Allocator> &subset )
 {
     auto range = convert( index );
     addSubset( range, subset );
@@ -586,8 +655,8 @@ bool Array<TYPE, FUN, Allocator>::operator==( const Array &rhs ) const
  *  Get a view of an C array                             *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
-std::unique_ptr<Array<TYPE, FUN, Allocator>> Array<TYPE, FUN, Allocator>::view(
-    const ArraySize &N, std::shared_ptr<TYPE> &data )
+std::unique_ptr<Array<TYPE, FUN, Allocator>>
+Array<TYPE, FUN, Allocator>::view( const ArraySize &N, std::shared_ptr<TYPE> data )
 {
     auto array    = std::make_unique<Array<TYPE, FUN, Allocator>>();
     array->d_size = N;
@@ -596,8 +665,9 @@ std::unique_ptr<Array<TYPE, FUN, Allocator>> Array<TYPE, FUN, Allocator>::view(
     return array;
 }
 template<class TYPE, class FUN, class Allocator>
-std::unique_ptr<const Array<TYPE, FUN, Allocator>> Array<TYPE, FUN, Allocator>::constView(
-    const ArraySize &N, std::shared_ptr<const TYPE> const &data )
+std::unique_ptr<const Array<TYPE, FUN, Allocator>>
+Array<TYPE, FUN, Allocator>::constView( const ArraySize &N,
+                                        std::shared_ptr<const TYPE> const &data )
 {
     auto array    = std::make_unique<Array<TYPE, FUN, Allocator>>();
     array->d_size = N;
@@ -612,15 +682,17 @@ void Array<TYPE, FUN, Allocator>::view2( Array<TYPE, FUN, Allocator> &src )
     d_data = src.d_data;
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::view2( const ArraySize &N, std::shared_ptr<TYPE> const &data )
+void Array<TYPE, FUN, Allocator>::view2( const ArraySize &N, std::shared_ptr<TYPE> data )
 {
     d_size = N;
     d_ptr  = data;
     d_data = d_ptr.get();
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::viewRaw(
-    const ArraySize &N, TYPE *data, bool isCopyable, bool isFixedSize )
+void Array<TYPE, FUN, Allocator>::viewRaw( const ArraySize &N,
+                                           TYPE *data,
+                                           bool isCopyable,
+                                           bool isFixedSize )
 {
     d_isCopyable  = isCopyable;
     d_isFixedSize = isFixedSize;
@@ -644,20 +716,8 @@ void Array<TYPE, FUN, Allocator>::swap( Array &other )
     std::swap( d_ptr, other.d_ptr );
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::fill( const TYPE &value )
-{
-    for ( size_t i = 0; i < d_size.length(); i++ )
-        d_data[i] = value;
-}
-template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::scale( const TYPE &value )
-{
-    for ( size_t i = 0; i < d_size.length(); i++ )
-        d_data[i] *= value;
-}
-template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::pow(
-    const Array<TYPE, FUN, Allocator> &baseArray, const TYPE &exp )
+void Array<TYPE, FUN, Allocator>::pow( const Array<TYPE, FUN, Allocator> &baseArray,
+                                       const TYPE &exp )
 {
     // not insisting on the shapes being the same
     // but insisting on the total size being the same
@@ -674,8 +734,8 @@ void Array<TYPE, FUN, Allocator>::pow(
  *  Replicate the array                                  *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::repmat(
-    const std::vector<size_t> &N_rep ) const
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::repmat( const std::vector<size_t> &N_rep ) const
 {
     std::vector<size_t> N2( d_size.begin(), d_size.end() );
     if ( N2.size() < N_rep.size() )
@@ -689,7 +749,6 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::repmat(
         N2[d] *= N_rep[d];
     }
     Array<TYPE, FUN, Allocator> y( N2 );
-    static_assert( ArraySize::maxDim() <= 5, "Not programmed for dimensions > 5" );
     TYPE *y2 = y.data();
     for ( size_t i4 = 0, index = 0; i4 < N1[4]; i4++ ) {
         for ( size_t j4 = 0; j4 < Nr[4]; j4++ ) {
@@ -731,7 +790,7 @@ bool Array<TYPE, FUN, Allocator>::NaNs() const
 template<class TYPE, class FUN, class Allocator>
 TYPE Array<TYPE, FUN, Allocator>::mean( void ) const
 {
-    TYPE x = this->sum() / d_size.length();
+    TYPE x = sum() / d_size.length();
     return x;
 }
 template<class TYPE, class FUN, class Allocator>
@@ -813,7 +872,6 @@ TYPE Array<TYPE, FUN, Allocator>::min( const std::vector<Range<size_t>> &range )
     checkSubsetIndex( range );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( range, first, last, inc, N1 );
-    static_assert( ArraySize::maxDim() <= 5, "Function programmed for more than 5 dimensions" );
     TYPE x = std::numeric_limits<TYPE>::max();
     for ( size_t i4 = first[4]; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
@@ -836,7 +894,6 @@ TYPE Array<TYPE, FUN, Allocator>::max( const std::vector<Range<size_t>> &range )
     checkSubsetIndex( range );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( range, first, last, inc, N1 );
-    static_assert( ArraySize::maxDim() <= 5, "Function programmed for more than 5 dimensions" );
     TYPE x = std::numeric_limits<TYPE>::min();
     for ( size_t i4 = first[4]; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
@@ -859,7 +916,6 @@ TYPE Array<TYPE, FUN, Allocator>::sum( const std::vector<Range<size_t>> &range )
     checkSubsetIndex( range );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( range, first, last, inc, N1 );
-    static_assert( ArraySize::maxDim() <= 5, "Function programmed for more than 5 dimensions" );
     TYPE x = 0;
     for ( size_t i4 = first[4]; i4 <= last[4]; i4 += inc[4] ) {
         for ( size_t i3 = first[3]; i3 <= last[3]; i3 += inc[3] ) {
@@ -882,7 +938,6 @@ TYPE Array<TYPE, FUN, Allocator>::mean( const std::vector<Range<size_t>> &range
     checkSubsetIndex( range );
     std::array<size_t, 5> first, last, inc, N1;
     getSubsetArrays( range, first, last, inc, N1 );
-    static_assert( ArraySize::maxDim() <= 5, "Function programmed for more than 5 dimensions" );
     size_t n = 1;
     for ( auto &d : N1 )
         n *= d;
@@ -919,8 +974,9 @@ TYPE Array<TYPE, FUN, Allocator>::mean( const std::vector<size_t> &index ) const
  *  Find all elements that match the given operation     *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
-std::vector<size_t> Array<TYPE, FUN, Allocator>::find(
-    const TYPE &value, std::function<bool( const TYPE &, const TYPE & )> compare ) const
+std::vector<size_t>
+Array<TYPE, FUN, Allocator>::find( const TYPE &value,
+                                   std::function<bool( const TYPE &, const TYPE & )> compare ) const
 {
     std::vector<size_t> result;
     result.reserve( d_size.length() );
@@ -936,8 +992,9 @@ std::vector<size_t> Array<TYPE, FUN, Allocator>::find(
  *  Print an array to an output stream                   *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::print(
-    std::ostream &os, const std::string &name, const std::string &prefix ) const
+void Array<TYPE, FUN, Allocator>::print( std::ostream &os,
+                                         const std::string &name,
+                                         const std::string &prefix ) const
 {
     if ( d_size.ndim() == 1 ) {
         for ( size_t i = 0; i < d_size[0]; i++ )
@@ -961,12 +1018,11 @@ void Array<TYPE, FUN, Allocator>::print(
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::reverseDim() const
 {
-    size_t N2[ArraySize::maxDim()];
+    size_t N2[5];
     for ( int d = 0; d < ArraySize::maxDim(); d++ )
         N2[d] = d_size[ArraySize::maxDim() - d - 1];
     ArraySize S2( ArraySize::maxDim(), N2 );
     Array<TYPE, FUN, Allocator> y( S2 );
-    static_assert( ArraySize::maxDim() == 5, "Not programmed for dimensions other than 5" );
     TYPE *y2 = y.data();
     for ( size_t i0 = 0; i0 < d_size[0]; i0++ ) {
         for ( size_t i1 = 0; i1 < d_size[1]; i1++ ) {
@@ -991,8 +1047,8 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::reverseDim() const
  *  Coarsen the array                                    *
  ********************************************************/
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen(
-    const Array<TYPE, FUN, Allocator> &filter ) const
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::coarsen( const Array<TYPE, FUN, Allocator> &filter ) const
 {
     auto S2 = size();
     for ( size_t i = 0; i < S2.size(); i++ ) {
@@ -1012,8 +1068,9 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen(
                 for ( size_t k2 = 0; k2 < Nh[2]; k2++ ) {
                     for ( size_t j2 = 0; j2 < Nh[1]; j2++ ) {
                         for ( size_t i2 = 0; i2 < Nh[0]; i2++ ) {
-                            tmp += filter( i2, j2, k2 ) * this->operator()( i1 *Nh[0] + i2,
-                                                              j1 * Nh[1] + j2, k1 * Nh[2] + k2 );
+                            tmp += filter( i2, j2, k2 ) * operator()( i1 *Nh[0] + i2,
+                                                                      j1 * Nh[1] + j2,
+                                                                      k1 * Nh[2] + k2 );
                         }
                     }
                 }
@@ -1024,7 +1081,8 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen(
     return y;
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen( const std::vector<size_t> &ratio,
+Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen(
+    const std::vector<size_t> &ratio,
     std::function<TYPE( const Array<TYPE, FUN, Allocator> & )> filter ) const
 {
     if ( ratio.size() != d_size.ndim() )
@@ -1045,7 +1103,7 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::coarsen( const std::vec
                 for ( size_t k2 = 0; k2 < ratio[2]; k2++ ) {
                     for ( size_t j2 = 0; j2 < ratio[1]; j2++ ) {
                         for ( size_t i2 = 0; i2 < ratio[0]; i2++ ) {
-                            tmp( i2, j2, k2 ) = this->operator()(
+                            tmp( i2, j2, k2 ) = operator()(
                                 i1 *ratio[0] + i2, j1 * ratio[1] + j2, k1 * ratio[2] + k2 );
                         }
                     }
@@ -1070,13 +1128,25 @@ void Array<TYPE, FUN, Allocator>::cat( const Array<TYPE, FUN, Allocator> &x, int
     *this = cat( tmp, dim );
 }
 template<class TYPE, class FUN, class Allocator>
+Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::initializer_list<Array> &x,
+                                                              int dim )
+{
+    return cat( x.size(), x.begin(), dim );
+}
+template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::vector<Array> &x, int dim )
 {
-    if ( x.empty() )
+    return cat( x.size(), x.data(), dim );
+}
+template<class TYPE, class FUN, class Allocator>
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::cat( size_t N_array, const Array *x, int dim )
+{
+    if ( N_array == 0 )
         return Array<TYPE, FUN, Allocator>();
     // Check that the dimensions match
     bool check = true;
-    for ( size_t i = 1; i < x.size(); i++ ) {
+    for ( size_t i = 1; i < N_array; i++ ) {
         check = check && x[i].ndim() == x[0].ndim();
         for ( int d = 0; d < x[0].ndim(); d++ )
             if ( d != dim )
@@ -1086,7 +1156,7 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::vector<
         throw std::logic_error( "Array dimensions do not match for concatenation" );
     // Create the output array
     auto size = x[0].d_size;
-    for ( size_t i = 1; i < x.size(); i++ )
+    for ( size_t i = 1; i < N_array; i++ )
         size.resize( dim, size[dim] + x[i].size( dim ) );
     Array<TYPE, FUN, Allocator> out( size );
     size_t N1 = 1;
@@ -1097,7 +1167,7 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::vector<
     for ( size_t d = dim + 1; d < size.ndim(); d++ )
         N3 *= size[d];
     TYPE *data = out.data();
-    for ( size_t i = 0, i0 = 0; i < x.size(); i++ ) {
+    for ( size_t i = 0, i0 = 0; i < N_array; i++ ) {
         const TYPE *src = x[i].data();
         size_t N22      = x[i].size( dim );
         for ( size_t j2 = 0; j2 < N3; j2++ ) {
@@ -1117,87 +1187,82 @@ Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::cat( const std::vector<
  *  Interpolate                                          *
  ********************************************************/
 template<class T>
-struct is_compatible_double
-    : std::integral_constant<bool, std::is_floating_point<T>::value || std::is_integral<T>::value> {
-};
-template<class TYPE>
-inline typename std::enable_if<is_compatible_double<TYPE>::value, TYPE>::type Array_interp_1D(
-    double x, int N, const TYPE *data )
+constexpr bool is_compatible_double()
 {
-    int i = floor( x );
-    i     = std::max( i, 0 );
-    i     = std::min( i, N - 2 );
-    return ( i + 1 - x ) * data[i] + ( x - i ) * data[i + 1];
+    return std::is_floating_point<T>::value || std::is_integral<T>::value;
 }
 template<class TYPE>
-inline typename std::enable_if<is_compatible_double<TYPE>::value, TYPE>::type Array_interp_2D(
-    double x, double y, int Nx, int Ny, const TYPE *data )
+inline TYPE Array_interp_1D( double x, int N, const TYPE *data )
 {
-    int i       = floor( x );
-    i           = std::max( i, 0 );
-    i           = std::min( i, Nx - 2 );
-    double dx   = x - i;
-    double dx2  = 1.0 - dx;
-    int j       = floor( y );
-    j           = std::max( j, 0 );
-    j           = std::min( j, Ny - 2 );
-    double dy   = y - j;
-    double dy2  = 1.0 - dy;
-    double f[4] = { (double) data[i + j * Nx], (double) data[i + 1 + j * Nx],
-        (double) data[i + ( j + 1 ) * Nx], (double) data[i + 1 + ( j + 1 ) * Nx] };
-    return ( dx * f[1] + dx2 * f[0] ) * dy2 + ( dx * f[3] + dx2 * f[2] ) * dy;
+    if ( is_compatible_double<TYPE>() ) {
+        int i = floor( x );
+        i     = std::max( i, 0 );
+        i     = std::min( i, N - 2 );
+        return ( i + 1 - x ) * data[i] + ( x - i ) * data[i + 1];
+    } else {
+        throw std::logic_error( "Invalid conversion" );
+    }
 }
 template<class TYPE>
-inline typename std::enable_if<is_compatible_double<TYPE>::value, TYPE>::type Array_interp_3D(
-    double x, double y, double z, int Nx, int Ny, int Nz, const TYPE *data )
+inline TYPE Array_interp_2D( double x, double y, int Nx, int Ny, const TYPE *data )
 {
-    int i       = floor( x );
-    i           = std::max( i, 0 );
-    i           = std::min( i, Nx - 2 );
-    double dx   = x - i;
-    double dx2  = 1.0 - dx;
-    int j       = floor( y );
-    j           = std::max( j, 0 );
-    j           = std::min( j, Ny - 2 );
-    double dy   = y - j;
-    double dy2  = 1.0 - dy;
-    int k       = floor( z );
-    k           = std::max( k, 0 );
-    k           = std::min( k, Nz - 2 );
-    double dz   = z - k;
-    double dz2  = 1.0 - dz;
-    double f[8] = { (double) data[i + j * Nx + k * Nx * Ny],
-        (double) data[i + 1 + j * Nx + k * Nx * Ny],
-        (double) data[i + ( j + 1 ) * Nx + k * Nx * Ny],
-        (double) data[i + 1 + ( j + 1 ) * Nx + k * Nx * Ny],
-        (double) data[i + j * Nx + ( k + 1 ) * Nx * Ny],
-        (double) data[i + 1 + j * Nx + ( k + 1 ) * Nx * Ny],
-        (double) data[i + ( j + 1 ) * Nx + ( k + 1 ) * Nx * Ny],
-        (double) data[i + 1 + ( j + 1 ) * Nx + ( k + 1 ) * Nx * Ny] };
-    double h0   = ( dx * f[1] + dx2 * f[0] ) * dy2 + ( dx * f[3] + dx2 * f[2] ) * dy;
-    double h1   = ( dx * f[5] + dx2 * f[4] ) * dy2 + ( dx * f[7] + dx2 * f[6] ) * dy;
-    return h0 * dz2 + h1 * dz;
+    if ( is_compatible_double<TYPE>() ) {
+        int i       = floor( x );
+        i           = std::max( i, 0 );
+        i           = std::min( i, Nx - 2 );
+        double dx   = x - i;
+        double dx2  = 1.0 - dx;
+        int j       = floor( y );
+        j           = std::max( j, 0 );
+        j           = std::min( j, Ny - 2 );
+        double dy   = y - j;
+        double dy2  = 1.0 - dy;
+        double f[4] = { (double) data[i + j * Nx],
+                        (double) data[i + 1 + j * Nx],
+                        (double) data[i + ( j + 1 ) * Nx],
+                        (double) data[i + 1 + ( j + 1 ) * Nx] };
+        return ( dx * f[1] + dx2 * f[0] ) * dy2 + ( dx * f[3] + dx2 * f[2] ) * dy;
+    } else {
+        throw std::logic_error( "Invalid conversion" );
+    }
 }
 template<class TYPE>
-inline typename std::enable_if<!is_compatible_double<TYPE>::value, TYPE>::type Array_interp_1D(
-    double, int, const TYPE * )
+inline TYPE
+Array_interp_3D( double x, double y, double z, int Nx, int Ny, int Nz, const TYPE *data )
 {
-    throw std::logic_error( "Invalid conversion" );
-}
-template<class TYPE>
-inline typename std::enable_if<!is_compatible_double<TYPE>::value, TYPE>::type Array_interp_2D(
-    double, double, int, int, const TYPE * )
-{
-    throw std::logic_error( "Invalid conversion" );
-}
-template<class TYPE>
-inline typename std::enable_if<!is_compatible_double<TYPE>::value, TYPE>::type Array_interp_3D(
-    double, double, double, int, int, int, const TYPE * )
-{
-    throw std::logic_error( "Invalid conversion" );
+    if ( is_compatible_double<TYPE>() ) {
+        int i       = floor( x );
+        i           = std::max( i, 0 );
+        i           = std::min( i, Nx - 2 );
+        double dx   = x - i;
+        double dx2  = 1.0 - dx;
+        int j       = floor( y );
+        j           = std::max( j, 0 );
+        j           = std::min( j, Ny - 2 );
+        double dy   = y - j;
+        double dy2  = 1.0 - dy;
+        int k       = floor( z );
+        k           = std::max( k, 0 );
+        k           = std::min( k, Nz - 2 );
+        double dz   = z - k;
+        double dz2  = 1.0 - dz;
+        double f[8] = { (double) data[i + j * Nx + k * Nx * Ny],
+                        (double) data[i + 1 + j * Nx + k * Nx * Ny],
+                        (double) data[i + ( j + 1 ) * Nx + k * Nx * Ny],
+                        (double) data[i + 1 + ( j + 1 ) * Nx + k * Nx * Ny],
+                        (double) data[i + j * Nx + ( k + 1 ) * Nx * Ny],
+                        (double) data[i + 1 + j * Nx + ( k + 1 ) * Nx * Ny],
+                        (double) data[i + ( j + 1 ) * Nx + ( k + 1 ) * Nx * Ny],
+                        (double) data[i + 1 + ( j + 1 ) * Nx + ( k + 1 ) * Nx * Ny] };
+        double h0   = ( dx * f[1] + dx2 * f[0] ) * dy2 + ( dx * f[3] + dx2 * f[2] ) * dy;
+        double h1   = ( dx * f[5] + dx2 * f[4] ) * dy2 + ( dx * f[7] + dx2 * f[6] ) * dy;
+        return h0 * dz2 + h1 * dz;
+    } else {
+        throw std::logic_error( "Invalid conversion" );
+    }
 }
 template<class TYPE, class FUN, class Allocator>
-TYPE Array<TYPE, FUN, Allocator>::interp( const std::vector<double> &x ) const
+TYPE Array<TYPE, FUN, Allocator>::interp( const double *x ) const
 {
     int ndim = 0, dim[5];
     double x2[5];
@@ -1227,87 +1292,82 @@ TYPE Array<TYPE, FUN, Allocator>::interp( const std::vector<double> &x ) const
 /********************************************************
  *  Math operations (should call the Math class)         *
  ********************************************************/
-template<class TYPE, class FUN, class Allocator>
+/*template<class TYPE, class FUN, class Allocator>
 void Array<TYPE, FUN, Allocator>::rand()
 {
     FUN::rand( *this );
 }
+*/
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator+=(
-    const Array<TYPE, FUN, Allocator> &rhs )
+Array<TYPE, FUN, Allocator> &
+Array<TYPE, FUN, Allocator>::operator+=( const Array<TYPE, FUN, Allocator> &rhs )
 {
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a + b; };
-    FUN::transform( fun, *this, rhs, *this );
+    auto op = []( const TYPE &a, const TYPE &b ) { return a + b; };
+    FUN::transform( op, *this, rhs, *this );
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator-=(
-    const Array<TYPE, FUN, Allocator> &rhs )
+Array<TYPE, FUN, Allocator> &
+Array<TYPE, FUN, Allocator>::operator-=( const Array<TYPE, FUN, Allocator> &rhs )
 {
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a - b; };
-    FUN::transform( fun, *this, rhs, *this );
+    auto op = []( const TYPE &a, const TYPE &b ) { return a - b; };
+    FUN::transform( op, *this, rhs, *this );
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator+=( const TYPE &rhs )
 {
-    const auto &fun = [rhs]( const TYPE &x ) { return x + rhs; };
-    FUN::transform( fun, *this, *this );
+    auto op = [rhs]( const TYPE &x ) { return x + rhs; };
+    FUN::transform( op, *this, *this );
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
 Array<TYPE, FUN, Allocator> &Array<TYPE, FUN, Allocator>::operator-=( const TYPE &rhs )
 {
-    const auto &fun = [rhs]( const TYPE &x ) { return x - rhs; };
-    FUN::transform( fun, *this, *this );
+    auto op = [rhs]( const TYPE &x ) { return x - rhs; };
+    FUN::transform( op, *this, *this );
     return *this;
 }
 template<class TYPE, class FUN, class Allocator>
 TYPE Array<TYPE, FUN, Allocator>::min() const
 {
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a < b ? a : b; };
-    return FUN::reduce( fun, *this );
+    const auto &op = []( const TYPE &a, const TYPE &b ) { return a < b ? a : b; };
+    return FUN::reduce( op, *this, d_data[0] );
 }
 template<class TYPE, class FUN, class Allocator>
 TYPE Array<TYPE, FUN, Allocator>::max() const
 {
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a > b ? a : b; };
-    return FUN::reduce( fun, *this );
+    const auto &op = []( const TYPE &a, const TYPE &b ) { return a > b ? a : b; };
+    return FUN::reduce( op, *this, d_data[0] );
 }
 template<class TYPE, class FUN, class Allocator>
 TYPE Array<TYPE, FUN, Allocator>::sum() const
 {
-    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a + b; };
-    return FUN::reduce( fun, *this );
+    const auto &op = []( const TYPE &a, const TYPE &b ) { return a + b; };
+    return FUN::reduce( op, *this, static_cast<TYPE>( 0 ) );
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::multiply(
-    const Array<TYPE, FUN, Allocator> &a, const Array<TYPE, FUN, Allocator> &b )
+void Array<TYPE, FUN, Allocator>::axpby( const TYPE &alpha,
+                                         const Array<TYPE, FUN, Allocator> &x,
+                                         const TYPE &beta )
 {
-    Array<TYPE, FUN, Allocator> c;
-    FUN::multiply( a, b, c );
-    return c;
+    const auto &op = [alpha, beta]( const TYPE &x, const TYPE &y ) { return alpha * x + beta * y; };
+    return FUN::transform( op, x, *this, *this );
 }
 template<class TYPE, class FUN, class Allocator>
-void Array<TYPE, FUN, Allocator>::axpby(
-    const TYPE &alpha, const Array<TYPE, FUN, Allocator> &x, const TYPE &beta )
-{
-    const auto &fun = [alpha, beta](
-                          const TYPE &x, const TYPE &y ) { return alpha * x + beta * y; };
-    return FUN::transform( fun, x, *this, *this );
-}
-template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::transform(
-    std::function<TYPE( const TYPE & )> fun, const Array<TYPE, FUN, Allocator> &x )
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::transform( std::function<TYPE( const TYPE & )> fun,
+                                        const Array<TYPE, FUN, Allocator> &x )
 {
     Array<TYPE, FUN, Allocator> y;
     FUN::transform( fun, x, y );
     return y;
 }
 template<class TYPE, class FUN, class Allocator>
-Array<TYPE, FUN, Allocator> Array<TYPE, FUN, Allocator>::transform(
-    std::function<TYPE( const TYPE &, const TYPE & )> fun, const Array<TYPE, FUN, Allocator> &x,
-    const Array<TYPE, FUN, Allocator> &y )
+Array<TYPE, FUN, Allocator>
+Array<TYPE, FUN, Allocator>::transform( std::function<TYPE( const TYPE &, const TYPE & )> fun,
+                                        const Array<TYPE, FUN, Allocator> &x,
+                                        const Array<TYPE, FUN, Allocator> &y )
 {
     Array<TYPE, FUN, Allocator> z;
     FUN::transform( fun, x, y, z );
@@ -1319,4 +1379,5 @@ bool Array<TYPE, FUN, Allocator>::equals( const Array &rhs, TYPE tol ) const
     return FUN::equals( *this, rhs, tol );
 }
 
+
 #endif

common/ArraySize.h

@@ -1,9 +1,13 @@
 #ifndef included_ArraySizeClass
 #define included_ArraySizeClass
 
+#include "common/Utilities.h"
 
 #include <array>
 #include <cstdint>
+#include <cmath>
+#include <complex>
+#include <cstdlib>
 #include <cstring>
 #include <initializer_list>
 #include <stdexcept>
@@ -24,21 +28,22 @@
 
 
 #if ( defined( DEBUG ) || defined( _DEBUG ) ) && !defined( NDEBUG )
-#define CHECK_ARRAY_LENGTH( i )                                      \
+#define CHECK_ARRAY_LENGTH( i, length )                              \
     do {                                                             \
-        if ( i >= d_length )                                         \
+        if ( i >= length )                                           \
             throw std::out_of_range( "Index exceeds array bounds" ); \
     } while ( 0 )
 #else
-#define CHECK_ARRAY_LENGTH( i ) \
-    do {                        \
+#define CHECK_ARRAY_LENGTH( i, length ) \
+    do {                                \
     } while ( 0 )
 #endif
 
 
+
 // Forward declerations
 class FunctionTable;
-template<class TYPE, class FUN = FunctionTable, class Allocator = std::nullptr_t>
+template<class TYPE, class FUN = FunctionTable, class Allocator = std::allocator<TYPE>>
 class Array;
 
 
@@ -48,7 +53,7 @@ class Range final
 {
 public:
     //! Empty constructor
-    constexpr Range() : i( 0 ), j( -1 ), k( 1 ) {}
+    Range() : i( 0 ), j( -1 ), k( 1 ) {}
 
     /*!
      * Create a range i:k:j (or i:j)
@@ -56,8 +61,30 @@ public:
      * @param j_            Ending value
      * @param k_            Increment value
      */
-    constexpr Range( TYPE i_, TYPE j_, TYPE k_ = 1 ) : i( i_ ), j( j_ ), k( k_ ) {}
+    Range( const TYPE &i_, const TYPE &j_, const TYPE &k_ = 1 )
+        : i( i_ ), j( j_ ), k( k_ )
+    {
+    }
 
+    //! Get the number of values in the range
+    size_t size() const
+    {
+        if ( std::is_integral<TYPE>::value ) {
+            return ( static_cast<int64_t>( j ) - static_cast<int64_t>( i ) ) /
+                   static_cast<int64_t>( k );
+        } else if ( std::is_floating_point<TYPE>::value ) {
+            double tmp = static_cast<double>( ( j - i ) ) / static_cast<double>( k );
+            return static_cast<size_t>( floor( tmp + 1e-12 ) + 1 );
+        } else if ( std::is_same<TYPE, std::complex<float>>::value ||
+                                 std::is_same<TYPE, std::complex<double>>::value ) {
+            double tmp = std::real( ( j - i ) / ( k ) );
+            return static_cast<size_t>( floor( tmp + 1e-12 ) + 1 );
+        } else {
+            ERROR( "Unsupported type for range" );
+        }
+    }
+
+public:
     TYPE i, j, k;
 };
 
@@ -67,20 +94,20 @@ class ArraySize final
 {
 public:
     //! Empty constructor
-    constexpr ArraySize() : d_ndim( 1 ), d_length( 0 ), d_N{ 0, 1, 1, 1, 1 } {}
+    ArraySize() : d_ndim( 1 ), d_length( 0 ), d_N{ 0, 1, 1, 1, 1 } {}
 
     /*!
      * Create the vector size
      * @param N1            Number of elements in the first dimension
      */
-    constexpr ArraySize( size_t N1 ) : d_ndim( 1 ), d_length( N1 ), d_N{ N1, 1, 1, 1, 1 } {}
+    ArraySize( size_t N1 ) : d_ndim( 1 ), d_length( N1 ), d_N{ N1, 1, 1, 1, 1 } {}
 
     /*!
      * Create the vector size
      * @param N1            Number of elements in the first dimension
      * @param N2            Number of elements in the second dimension
      */
-    constexpr ArraySize( size_t N1, size_t N2 )
+    ArraySize( size_t N1, size_t N2 )
         : d_ndim( 2 ), d_length( N1 * N2 ), d_N{ N1, N2, 1, 1, 1 }
     {
     }
@@ -91,7 +118,7 @@ public:
      * @param N2            Number of elements in the second dimension
      * @param N3            Number of elements in the third dimension
      */
-    constexpr ArraySize( size_t N1, size_t N2, size_t N3 )
+    ArraySize( size_t N1, size_t N2, size_t N3 )
         : d_ndim( 3 ), d_length( N1 * N2 * N3 ), d_N{ N1, N2, N3, 1, 1 }
     {
     }
@@ -103,7 +130,7 @@ public:
      * @param N3            Number of elements in the third dimension
      * @param N4            Number of elements in the fourth dimension
      */
-    constexpr ArraySize( size_t N1, size_t N2, size_t N3, size_t N4 )
+    ArraySize( size_t N1, size_t N2, size_t N3, size_t N4 )
         : d_ndim( 4 ), d_length( N1 * N2 * N3 * N4 ), d_N{ N1, N2, N3, N4, 1 }
     {
     }
@@ -116,7 +143,7 @@ public:
      * @param N4            Number of elements in the fourth dimension
      * @param N5            Number of elements in the fifth dimension
      */
-    constexpr ArraySize( size_t N1, size_t N2, size_t N3, size_t N4, size_t N5 )
+    ArraySize( size_t N1, size_t N2, size_t N3, size_t N4, size_t N5 )
         : d_ndim( 5 ), d_length( N1 * N2 * N3 * N4 * N5 ), d_N{ N1, N2, N3, N4, N5 }
     {
     }
@@ -124,11 +151,14 @@ public:
     /*!
      * Create from initializer list
      * @param N             Size of the array
+     * @param ndim          Number of dimensions
      */
-    constexpr ArraySize( std::initializer_list<size_t> N )
+    ArraySize( std::initializer_list<size_t> N, int ndim = -1 )
         : d_ndim( N.size() ), d_length( 0 ), d_N{ 0, 1, 1, 1, 1 }
     {
-        if ( d_ndim > maxDim() )
+        if ( ndim >= 0 )
+            d_ndim = ndim;
+        if ( d_ndim > 5 )
             throw std::out_of_range( "Maximum number of dimensions exceeded" );
         auto it = N.begin();
         for ( size_t i = 0; i < d_ndim; i++, ++it )
@@ -146,10 +176,10 @@ public:
      * @param ndim          Number of dimensions
      * @param dims          Dimensions
      */
-    constexpr ArraySize( size_t ndim, const size_t *dims )
+    ArraySize( size_t ndim, const size_t *dims )
         : d_ndim( ndim ), d_length( 0 ), d_N{ 0, 1, 1, 1, 1 }
     {
-        if ( d_ndim > maxDim() )
+        if ( d_ndim > 5 )
             throw std::out_of_range( "Maximum number of dimensions exceeded" );
         for ( size_t i = 0; i < ndim; i++ )
             d_N[i] = dims[i];
@@ -160,35 +190,44 @@ public:
             d_length = 0;
     }
 
+    /*!
+     * Create from std::array
+     * @param N             Size of the array
+     */
+    template<std::size_t NDIM>
+    ArraySize( const std::array<size_t, NDIM> &N ) : ArraySize( NDIM, N.data() )
+    {
+    }
+
     /*!
      * Create from std::vector
      * @param N             Size of the array
      */
-    ArraySize( const std::vector<size_t> &N );
+    inline ArraySize( const std::vector<size_t> &N ) : ArraySize( N.size(), N.data() ) {}
 
     // Copy/assignment constructors
-    constexpr ArraySize( ArraySize &&rhs )      = default;
-    constexpr ArraySize( const ArraySize &rhs ) = default;
-    constexpr ArraySize &operator=( ArraySize &&rhs ) = default;
-    constexpr ArraySize &operator=( const ArraySize &rhs ) = default;
+    ArraySize( ArraySize &&rhs )      = default;
+    ArraySize( const ArraySize &rhs ) = default;
+    ArraySize &operator=( ArraySize &&rhs ) = default;
+    ArraySize &operator=( const ArraySize &rhs ) = default;
 
     /*!
      * Access the ith dimension
      * @param i             Index to access
      */
-    constexpr ARRAY_ATTRIBUTE size_t operator[]( size_t i ) const { return d_N[i]; }
+    ARRAY_ATTRIBUTE size_t operator[]( size_t i ) const { return d_N[i]; }
 
     //! Return the number of dimensions
-    constexpr ARRAY_ATTRIBUTE uint8_t ndim() const { return d_ndim; }
+    ARRAY_ATTRIBUTE uint8_t ndim() const { return d_ndim; }
 
     //! Return the number of dimensions
-    constexpr ARRAY_ATTRIBUTE size_t size() const { return d_ndim; }
+    ARRAY_ATTRIBUTE size_t size() const { return d_ndim; }
 
     //! Return the total number of elements in the array
-    constexpr ARRAY_ATTRIBUTE size_t length() const { return d_length; }
+    ARRAY_ATTRIBUTE size_t length() const { return d_length; }
 
     //! Resize the dimension
-    constexpr void resize( uint8_t dim, size_t N )
+    void resize( uint8_t dim, size_t N )
     {
         if ( dim >= d_ndim )
             throw std::out_of_range( "Invalid dimension" );
@@ -203,75 +242,141 @@ public:
      *    max of ndim and the largest dim>1.
      * @param ndim          Desired number of dimensions
      */
-    constexpr void setNdim( uint8_t ndim ) { d_ndim = std::max( ndim, d_ndim ); }
+    void setNdim( uint8_t ndim ) { d_ndim = std::max( ndim, d_ndim ); }
+
+    /*!
+     * Remove singleton dimensions
+     */
+    void squeeze()
+    {
+        d_ndim = 0;
+        for ( uint8_t i = 0; i < maxDim(); i++ ) {
+            if ( d_N[i] != 1 )
+                d_N[d_ndim++] = d_N[i];
+        }
+    }
 
     //! Returns an iterator to the beginning
-    constexpr const size_t *begin() const { return d_N; }
+    const size_t *begin() const { return d_N; }
 
     //! Returns an iterator to the end
-    constexpr const size_t *end() const { return d_N + d_ndim; }
+    const size_t *end() const { return d_N + d_ndim; }
 
     // Check if two array sizes are equal
-    constexpr ARRAY_ATTRIBUTE bool operator==( const ArraySize &rhs ) const
+    ARRAY_ATTRIBUTE bool operator==( const ArraySize &rhs ) const
     {
         return d_ndim == rhs.d_ndim && memcmp( d_N, rhs.d_N, sizeof( d_N ) ) == 0;
     }
 
     // Check if two array sizes are equal (ignoring the dimension)
-    constexpr ARRAY_ATTRIBUTE bool approxEqual( const ArraySize &rhs ) const
+    ARRAY_ATTRIBUTE bool approxEqual( const ArraySize &rhs ) const
     {
         return ( length() == 0 && rhs.length() == 0 ) || memcmp( d_N, rhs.d_N, sizeof( d_N ) ) == 0;
     }
 
     //! Check if two matrices are not equal
-    constexpr ARRAY_ATTRIBUTE bool operator!=( const ArraySize &rhs ) const
+    ARRAY_ATTRIBUTE bool operator!=( const ArraySize &rhs ) const
     {
         return d_ndim != rhs.d_ndim || memcmp( d_N, rhs.d_N, sizeof( d_N ) ) != 0;
     }
 
     //! Maximum supported dimension
-    constexpr ARRAY_ATTRIBUTE static uint8_t maxDim() { return 5u; }
+    ARRAY_ATTRIBUTE static uint8_t maxDim() { return 5; }
 
     //! Get the index
-    constexpr ARRAY_ATTRIBUTE size_t index( size_t i ) const
+    ARRAY_ATTRIBUTE size_t index( size_t i ) const
     {
-        CHECK_ARRAY_LENGTH( i );
+        CHECK_ARRAY_LENGTH( i, d_length );
         return i;
     }
 
     //! Get the index
-    constexpr ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2 ) const
+    ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2 ) const
     {
         size_t index = i1 + i2 * d_N[0];
-        CHECK_ARRAY_LENGTH( index );
+        CHECK_ARRAY_LENGTH( index, d_length );
         return index;
     }
 
     //! Get the index
-    constexpr ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2, size_t i3 ) const
+    ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2, size_t i3 ) const
     {
         size_t index = i1 + d_N[0] * ( i2 + d_N[1] * i3 );
-        CHECK_ARRAY_LENGTH( index );
+        CHECK_ARRAY_LENGTH( index, d_length );
         return index;
     }
 
     //! Get the index
-    constexpr ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2, size_t i3, size_t i4 ) const
+    ARRAY_ATTRIBUTE size_t index( size_t i1, size_t i2, size_t i3, size_t i4 ) const
     {
         size_t index = i1 + d_N[0] * ( i2 + d_N[1] * ( i3 + d_N[2] * i4 ) );
-        CHECK_ARRAY_LENGTH( index );
+        CHECK_ARRAY_LENGTH( index, d_length );
         return index;
     }
 
     //! Get the index
-    constexpr ARRAY_ATTRIBUTE size_t index(
-        size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const
+    ARRAY_ATTRIBUTE size_t
+    index( size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const
     {
         size_t index = i1 + d_N[0] * ( i2 + d_N[1] * ( i3 + d_N[2] * ( i4 + d_N[3] * i5 ) ) );
-        CHECK_ARRAY_LENGTH( index );
+        CHECK_ARRAY_LENGTH( index, d_length );
         return index;
     }
 
+    //! Get the index
+    size_t index( const std::array<size_t, 5> &i ) const
+    {
+        size_t j = 0;
+        for ( size_t m = 0, N = 1; m < 5; m++ ) {
+            j += i[m] * N;
+            N *= d_N[m];
+        }
+        return j;
+    }
+
+    //! Get the index
+    size_t index( std::initializer_list<size_t> i ) const
+    {
+        size_t N = 1;
+        size_t j = 0;
+        size_t m = 0;
+        for ( size_t k : i ) {
+            j += k * N;
+            N *= d_N[m++];
+        }
+        return j;
+    }
+
+    //! Convert the index to ijk values
+    std::array<size_t, 5> ijk( size_t index ) const
+    {
+        CHECK_ARRAY_LENGTH( index, d_length );
+        size_t i0 = index % d_N[0];
+        index     = index / d_N[0];
+        size_t i1 = index % d_N[1];
+        index     = index / d_N[1];
+        size_t i2 = index % d_N[2];
+        index     = index / d_N[2];
+        size_t i3 = index % d_N[3];
+        index     = index / d_N[3];
+        return { i0, i1, i2, i3, index };
+    }
+
+    //! Convert the index to ijk values
+    void ijk( size_t index, size_t *x ) const
+    {
+        CHECK_ARRAY_LENGTH( index, d_length );
+        x[0]  = index % d_N[0];
+        index = index / d_N[0];
+        x[1]  = index % d_N[1];
+        index = index / d_N[1];
+        x[2]  = index % d_N[2];
+        index = index / d_N[2];
+        x[3]  = index % d_N[3];
+        index = index / d_N[3];
+        x[4]  = index;
+    }
+
 private:
     uint8_t d_ndim;
     size_t d_length;
@@ -280,11 +385,11 @@ private:
 
 
 // Function to concatenate dimensions of two array sizes
-constexpr ArraySize cat( const ArraySize &x, const ArraySize &y )
+inline ArraySize cat( const ArraySize &x, const ArraySize &y )
 {
-    if ( x.ndim() + y.ndim() > ArraySize::maxDim() )
+    if ( x.ndim() + y.ndim() > 5 )
         throw std::out_of_range( "Maximum number of dimensions exceeded" );
-    size_t N[ArraySize::maxDim()] = { 0 };
+    size_t N[5] = { 0 };
     for ( int i = 0; i < x.ndim(); i++ )
         N[i] = x[i];
     for ( int i = 0; i < y.ndim(); i++ )
@@ -293,4 +398,36 @@ constexpr ArraySize cat( const ArraySize &x, const ArraySize &y )
 }
 
 
+// Operator overloads
+inline ArraySize operator*( size_t v, const ArraySize &x )
+{
+    size_t N[5] = { v * x[0], v * x[1], v * x[2], v * x[3], v * x[4] };
+    return ArraySize( x.ndim(), N );
+}
+inline ArraySize operator*( const ArraySize &x, size_t v )
+{
+    size_t N[5] = { v * x[0], v * x[1], v * x[2], v * x[3], v * x[4] };
+    return ArraySize( x.ndim(), N );
+}
+inline ArraySize operator-( const ArraySize &x, size_t v )
+{
+    size_t N[5] = { x[0] - v, x[1] - v, x[2] - v, x[3] - v, x[4] - v };
+    return ArraySize( x.ndim(), N );
+}
+inline ArraySize operator+( const ArraySize &x, size_t v )
+{
+    size_t N[5] = { x[0] + v, x[1] + v, x[2] + v, x[3] + v, x[4] + v };
+    return ArraySize( x.ndim(), N );
+}
+inline ArraySize operator+( size_t v, const ArraySize &x )
+{
+    size_t N[5] = { x[0] + v, x[1] + v, x[2] + v, x[3] + v, x[4] + v };
+    return ArraySize( x.ndim(), N );
+}
+
+#if defined( USING_ICC )
+ENABLE_WARNINGS
+#endif
+
+
 #endif

common/Communication.h

@@ -51,6 +51,7 @@ class fillHalo
 public:
     /*!
      * @brief  Default constructor
+     * @param[in] comm          Communicator to use
      * @param[in] info          Rank and neighbor rank info
      * @param[in] n             Number of local cells
      * @param[in] ng            Number of ghost cells

common/Domain.cpp

@@ -383,7 +383,7 @@ void Domain::Decomp( const std::string& Filename )
 				for (int i = 0; i<global_Nx; i++){
 					n = k*global_Nx*global_Ny+j*global_Nx+i;
 					//char locval = loc_id[n];
-					char locval = SegData[n];
+					signed char locval = SegData[n];
 					for (size_t idx=0; idx<ReadValues.size(); idx++){
 						signed char oldvalue=ReadValues[idx];
 						signed char newvalue=WriteValues[idx];
@@ -629,7 +629,8 @@ void Domain::ComputePorosity(){
 	double sum;
 	double sum_local=0.0;
 	double iVol_global = 1.0/(1.0*(Nx-2)*(Ny-2)*(Nz-2)*nprocx()*nprocy()*nprocz());
-	if (BoundaryCondition > 0 && BoundaryCondition !=5) iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-6));
+	if (BoundaryCondition > 0 && BoundaryCondition !=5) 
+		iVol_global = 1.0/(1.0*(Nx-2)*nprocx()*(Ny-2)*nprocy()*((Nz-2)*nprocz()-inlet_layers_z - outlet_layers_z));
 	//.........................................................
     for (int k=inlet_layers_z+1; k<Nz-outlet_layers_z-1;k++){
         for (int j=1;j<Ny-1;j++){

common/Domain.h

@@ -16,87 +16,98 @@
 #include "common/MPI.h"
 #include "common/Communication.h"
 #include "common/Database.h"
-
-class Domain;
-template<class TYPE> class PatchData;
+/**
+ * @file Domain.h
+ * \brief Parallel Domain data structures and helper functions
+ */
 
 
-//! Class to hold information about a box
+/**
+ * \class Box
+ *
+ * @details
+ * information about a box
+ */
 class Box {
 public:
     int ifirst[3];
     int ilast[3];
 };
 
+class Patch;
 
-enum class DataLocation { CPU, DEVICE };
+/**
+ * \class Domain
+ *
+ * @details
+ * the Domain class includes basic information to distribution 3D image data to multiple processes using MPI.
+ * A regular domain decomposision is performed, with each MPI process getting a [Nx,Ny,Nz] sub-domain.
+ * 8-bit image labels are retained internally.
+ * The domain class resides on the CPU and provides utilities to support CPU-based analysis.
+ * GPU-based data structures should be constructed separately but may utilize information that the Domain class provides.
+*/
 
-
-//! Class to hold information about a patch
-class Patch {
-public:
-    
-    //! Empty constructor
-    Patch() = delete;
-
-    //! Copy constructor
-    Patch( const Patch& ) = delete;
-
-    //! Assignment operator
-    Patch& operator=( const Patch& ) = delete;
-
-    //! Return the box for the patch
-    inline const Box& getBox() const { return d_box; } 
-
-    //! Create patch data
-    template<class TYPE>
-    std::shared_ptr<PatchData<TYPE>> createPatchData( DataLocation location ) const;
-
-private:
-    Box d_box;
-    int d_owner;
-    Domain *d_domain;
-
-};
-
-
-//! Class to hold domain info
 class Domain{
 public:
-    //! Default constructor
+    /**
+    * \brief Constructor
+    * @param db           input database
+    * @param Communicator MPI communicator  
+    */
     Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator);
 
-    //! Obsolete constructor
+    /**
+     * \brief Obsolete constructor
+     */
     Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz, 
                    double lx, double ly, double lz, int BC);
 
-    //! Empty constructor
+    /**
+    * \brief Empty constructor
+    */
     Domain() = delete;
 
-    //! Copy constructor
+    /**
+    * \brief Copy constructor
+    */
     Domain( const Domain& ) = delete;
 
-    //! Assignment operator
+    /**
+     * \brief Assignment operator
+     */ 
     Domain& operator=( const Domain& ) = delete;
 
-    //! Destructor
+    /**
+     * \brief Destructor
+     */ 
     ~Domain();
     
-    //! Get the database
+    /**
+     * \brief Get the database
+    */
     inline std::shared_ptr<const Database> getDatabase() const { return d_db; }
 
-    //! Get the domain box
+    /** 
+    * \brief Get the domain box
+    */
     inline const Box& getBox() const { return d_box; } 
 
-    //! Get local patch
+    /** 
+    * \brief Get local patch
+    */
     inline const Patch& getLocalPatch() const { return *d_localPatch; }
 
-    //! Get all patches
+    /** 
+     * \brief Get all patches
+    */ 
     inline const std::vector<Patch>& getAllPatch() const { return d_patches; }
 
 
 private:
 
+    /** 
+     * \brief initialize from database
+    */ 
     void initialize( std::shared_ptr<Database> db );
 
     std::shared_ptr<Database> d_db;
@@ -124,6 +135,9 @@ public: // Public variables (need to create accessors instead)
     //**********************************
     // MPI ranks for all 18 neighbors
     //**********************************
+    /** 
+     * \brief Compute the porosity based on the current domain id file
+    */ 
     inline double Porosity() const { return porosity; }
     inline int iproc() const { return rank_info.ix; }
     inline int jproc() const { return rank_info.jy; }
@@ -165,22 +179,78 @@ public: // Public variables (need to create accessors instead)
     // Solid indicator function
     std::vector<signed char> id;
 
+    /** 
+     * \brief Read domain IDs from file
+    */ 
     void ReadIDs();
+    
+    /** 
+     * \brief Compute the porosity
+    */ 
     void ComputePorosity();
+    
+    /** 
+     * \brief Read image and perform domain decomposition
+     * @param filename  - name of file to read IDs
+    */ 
     void Decomp( const std::string& filename );
+    
+    /** 
+     * \brief Perform a halo exchange using MPI
+     * @param Mesh - array data that holds scalar values
+    */ 
     void CommunicateMeshHalo(DoubleArray &Mesh);
+    
+    /** 
+     * \brief Initialize communication data structures within Domain object. 
+     * This routine needs to be called before the communication functionality will work
+    */ 
     void CommInit(); 
+    
+    /** 
+     * \brief Count number of pore nodes (labels > 1)
+    */ 
     int PoreCount();
     
+    /** 
+     * \brief Read array data from a file and distribute to local arrays for each MPI process
+     * @param Filename - name of the file to read the data 
+     * @param Datatype - data type to use 
+     * @param UserData - Array to store the values that are read
+    */ 
     void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
+    
+    /**
+     * \brief Aggregate labels from all MPI processes and write to a file
+     * @param filename - name of the file to write
+     */
     void AggregateLabels( const std::string& filename );
+    /**
+     * \brief Aggregate user provided array  from all MPI processes and write to a single file
+     * @param filename - name of the file to write
+     * @param UserData - array data to aggregate and write
+     */
     void AggregateLabels( const std::string& filename, DoubleArray &UserData );
 
 private:
 
+    /**
+     * \brief Pack halo data for 8-bit integer
+     * @param list - list of values in the halo
+     * @param count - count of values in the halo
+     * @param sendbuf - memory buffer to use to pack values for MPI
+     * @param ID - 8-bit values on mesh [Nx, Ny, Nz]
+     */
     void PackID(int *list, int count, signed char *sendbuf, signed char *ID);
+    
+    /**
+     * \brief Unpack halo data for 8-bit integer
+     * @param list - list of values in the halo
+     * @param count - count of values in the halo
+     * @param recvbuf - memory buffer containing values recieved by MPI
+     * @param ID - 8-bit values on mesh [Nx, Ny, Nz]
+     */
     void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID);
-    void CommHaloIDs();
     
 	//......................................................................................
 	MPI_Request req1[18], req2[18];
@@ -198,6 +268,44 @@ private:
 
 };
 
+template<class TYPE> class PatchData;
+
+
+enum class DataLocation { CPU, DEVICE };
+
+
+/**
+ * \class Patch
+ *
+ * @details
+ * store patch data
+ */
+class Patch {
+public:
+    
+    //! Empty constructor
+    Patch() = delete;
+
+    //! Copy constructor
+    Patch( const Patch& ) = delete;
+
+    //! Assignment operator
+    Patch& operator=( const Patch& ) = delete;
+
+    //! Return the box for the patch
+    inline const Box& getBox() const { return d_box; } 
+
+    //! Create patch data
+    template<class TYPE>
+    std::shared_ptr<PatchData<TYPE>> createPatchData( DataLocation location ) const;
+
+private:
+    Box d_box;
+    int d_owner;
+    Domain *d_domain;
+
+};
+
 
 // Class to hold data on a patch
 template<class TYPE>

common/FunctionTable.cpp

@@ -0,0 +1,147 @@
+#include "FunctionTable.hpp"
+
+
+/********************************************************
+ *  Random number generation                             *
+ ********************************************************/
+/*template<> char genRand<char>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<char> dis;
+    return dis( gen );
+}
+template<> int8_t genRand<int8_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<int8_t> dis;
+    return dis( gen );
+}
+template<> uint8_t genRand<uint8_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<uint8_t> dis;
+    return dis( gen );
+}
+template<> int16_t genRand<int16_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<int16_t> dis;
+    return dis( gen );
+}
+template<> uint16_t genRand<uint16_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<uint16_t> dis;
+    return dis( gen );
+}
+template<> int32_t genRand<int32_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<int32_t> dis;
+    return dis( gen );
+}
+template<> uint32_t genRand<uint32_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<uint32_t> dis;
+    return dis( gen );
+}
+template<> int64_t genRand<int64_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<int64_t> dis;
+    return dis( gen );
+}
+template<> uint64_t genRand<uint64_t>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_int_distribution<uint64_t> dis;
+    return dis( gen );
+}
+template<> float genRand<float>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_real_distribution<float> dis;
+    return dis( gen );
+}
+template<> double genRand<double>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_real_distribution<double> dis;
+    return dis( gen );
+}
+template<> long double genRand<long double>()
+{
+    static std::random_device rd;
+    static std::mt19937 gen( rd() );
+    static std::uniform_real_distribution<double> dis;
+    return dis( gen );
+}
+*/
+
+/********************************************************
+ *  axpy                                                 *
+ ********************************************************/
+template<>
+void call_axpy<float>( size_t N, const float alpha, const float *x, float *y )
+{
+    ERROR("Not finished");
+}
+template<>
+void call_axpy<double>( size_t N, const double alpha, const double *x, double *y )
+{
+    ERROR("Not finished");
+}
+
+
+/********************************************************
+ *  Multiply two arrays                                  *
+ ********************************************************/
+template<>
+void call_gemv<double>(
+    size_t M, size_t N, double alpha, double beta, const double *A, const double *x, double *y )
+{
+    ERROR("Not finished");
+}
+template<>
+void call_gemv<float>(
+    size_t M, size_t N, float alpha, float beta, const float *A, const float *x, float *y )
+{
+    ERROR("Not finished");
+}
+template<>
+void call_gemm<double>( size_t M,
+                        size_t N,
+                        size_t K,
+                        double alpha,
+                        double beta,
+                        const double *A,
+                        const double *B,
+                        double *C )
+{
+    ERROR("Not finished");
+}
+template<>
+void call_gemm<float>( size_t M,
+                       size_t N,
+                       size_t K,
+                       float alpha,
+                       float beta,
+                       const float *A,
+                       const float *B,
+                       float *C )
+{
+    ERROR("Not finished");
+}
+

common/FunctionTable.h

@@ -7,6 +7,7 @@
 #include <functional>
 
 
+
 /*!
  * Class FunctionTable is a serial function table class that defines
  *   a series of operations that can be performed on the Array class.
@@ -25,38 +26,55 @@ public:
 
     /*!
      * Perform a reduce operator y = f(x)
-     * @param[in] op        The function operation
-     *                      Note: the operator is a template parameter
-     *                      (compared to a std::function to improve performance)
-     * @param[in] A         The array to operate on
-     * @return              The reduction
+     * @param[in] op            The function operation
+     *                          Note: the operator is a template parameter to improve performance
+     * @param[in] A             The array to operate on
+     * @param[in] initialValue  The initial value for the reduction (0 for sum, +/- inf for min/max,
+     * ...)
+     * @return                  The reduction
      */
     template<class TYPE, class FUN, typename LAMBDA>
-    static inline TYPE reduce( LAMBDA &op, const Array<TYPE, FUN> &A );
+    static inline TYPE reduce( LAMBDA &op, const Array<TYPE, FUN> &A, const TYPE &initialValue );
+
+    /*!
+     * Perform a reduce operator z = f(x,y)
+     * @param[in] op            The function operation
+     *                          Note: the operator is a template parameter to improve performance
+     * @param[in] A             The first array to operate on
+     * @param[in] B             The second array to operate on
+     * @param[in] initialValue  The initial value for the reduction (0 for sum, +/- inf for min/max,
+     * ...)
+     * @return                  The reduction
+     */
+    template<class TYPE, class FUN, typename LAMBDA>
+    static inline TYPE reduce( LAMBDA &op,
+                               const Array<TYPE, FUN> &A,
+                               const Array<TYPE, FUN> &B,
+                               const TYPE &initialValue );
 
     /*!
      * Perform a element-wise operation y = f(x)
-     * @param[in] fun       The function operation
-     *                      Note: the operator is a template parameter
-     *                      (compared to a std::function to improve performance)
-     * @param[in] x         The input array to operate on
-     * @param[out] y        The output array
+     * @param[in] fun           The function operation
+     *                          Note: the function is a template parameter to improve performance
+     * @param[in,out] x         The array to operate on
+     * @param[out] y            The output array
      */
     template<class TYPE, class FUN, typename LAMBDA>
     static inline void transform( LAMBDA &fun, const Array<TYPE, FUN> &x, Array<TYPE, FUN> &y );
 
     /*!
      * Perform a element-wise operation z = f(x,y)
-     * @param[in] fun       The function operation
-     *                      Note: the operator is a template parameter
-     *                      (compared to a std::function to improve performance)
-     * @param[in] x         The first array
-     * @param[in] y         The second array
-     * @param[out] z        The result
+     * @param[in] fun           The function operation
+     *                          Note: the function is a template parameter to improve performance
+     * @param[in] x             The first array
+     * @param[in] y             The second array
+     * @param[out] z            The output array
      */
     template<class TYPE, class FUN, typename LAMBDA>
-    static inline void transform(
-        LAMBDA &fun, const Array<TYPE, FUN> &x, const Array<TYPE, FUN> &y, Array<TYPE, FUN> &z );
+    static inline void transform( LAMBDA &fun,
+                                  const Array<TYPE, FUN> &x,
+                                  const Array<TYPE, FUN> &y,
+                                  Array<TYPE, FUN> &z );
 
     /*!
      * Multiply two arrays
@@ -65,8 +83,8 @@ public:
      * @param[out] c            The output array
      */
     template<class TYPE, class FUN>
-    static inline void multiply(
-        const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, Array<TYPE, FUN> &c );
+    static void
+    multiply( const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, Array<TYPE, FUN> &c );
 
     /*!
      * Perform dgemv/dgemm equavalent operation ( C = alpha*A*B + beta*C )
@@ -74,11 +92,14 @@ public:
      * @param[in] A             The first array
      * @param[in] B             The second array
      * @param[in] beta          The scalar value alpha
-     * @param[in,out] c         The output array C
+     * @param[in,out] C         The output array C
      */
     template<class TYPE, class FUN>
-    static void gemm( const TYPE alpha, const Array<TYPE, FUN> &A, const Array<TYPE, FUN> &B,
-        const TYPE beta, Array<TYPE, FUN> &C );
+    static void gemm( const TYPE alpha,
+                      const Array<TYPE, FUN> &A,
+                      const Array<TYPE, FUN> &B,
+                      const TYPE beta,
+                      Array<TYPE, FUN> &C );
 
     /*!
      * Perform axpy equavalent operation ( y = alpha*x + y )
@@ -98,9 +119,84 @@ public:
     template<class TYPE, class FUN>
     static bool equals( const Array<TYPE, FUN> &A, const Array<TYPE, FUN> &B, TYPE tol );
 
+    template<class TYPE>
+    static inline void gemmWrapper( char TRANSA,
+                                    char TRANSB,
+                                    int M,
+                                    int N,
+                                    int K,
+                                    TYPE alpha,
+                                    const TYPE *A,
+                                    int LDA,
+                                    const TYPE *B,
+                                    int LDB,
+                                    TYPE beta,
+                                    TYPE *C,
+                                    int LDC );
+
+
+    /* Specialized Functions */
+
+    /*!
+     * Perform a element-wise operation y = max(x , 0)
+     * @param[in] A             The input array
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformReLU( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Perform a element-wise operation B = |A|
+     * @param[in] A             The array to operate on
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformAbs( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Perform a element-wise operation B = tanh(A)
+     * @param[in] A             The array to operate on
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformTanh( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Perform a element-wise operation B = max(-1 , min(1 , A) )
+     * @param[in] A             The array to operate on
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformHardTanh( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Perform a element-wise operation B = 1 / (1 + exp(-A))
+     * @param[in] A             The array to operate on
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformSigmoid( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Perform a element-wise operation B = log(exp(A) + 1)
+     * @param[in] A             The array to operate on
+     * @param[out] B            The output array
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static void transformSoftPlus( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B );
+
+    /*!
+     * Sum the elements of the Array
+     * @param[in] A             The array to sum
+     */
+    template<class TYPE, class FUN, class ALLOC>
+    static TYPE sum( const Array<TYPE, FUN, ALLOC> &A );
 
 private:
     FunctionTable();
+
+    template<class T>
+    static inline void rand( size_t N, T *x );
 };
 
 

common/FunctionTable.hpp

@@ -2,66 +2,63 @@
 #define included_FunctionTable_hpp
 
 #include "common/FunctionTable.h"
-#include "common/Utilities.h"
+#include "common/UtilityMacros.h"
 
 #include <algorithm>
 #include <cstring>
 #include <limits>
-#include <random>
+//#include <random>
 
 
 /********************************************************
  *  Random number initialization                         *
  ********************************************************/
-template<class TYPE>
-static inline typename std::enable_if<std::is_integral<TYPE>::value>::type genRand(
-    size_t N, TYPE* x )
-{
-    std::random_device rd;
-    std::mt19937 gen( rd() );
-    std::uniform_int_distribution<TYPE> dis;
-    for ( size_t i = 0; i < N; i++ )
-        x[i] = dis( gen );
-}
-template<class TYPE>
-static inline typename std::enable_if<std::is_floating_point<TYPE>::value>::type genRand(
-    size_t N, TYPE* x )
-{
-    std::random_device rd;
-    std::mt19937 gen( rd() );
-    std::uniform_real_distribution<TYPE> dis( 0, 1 );
-    for ( size_t i = 0; i < N; i++ )
-        x[i] = dis( gen );
-}
+/*template<class TYPE> TYPE genRand();
 template<class TYPE, class FUN>
-inline void FunctionTable::rand( Array<TYPE, FUN>& x )
+inline void FunctionTable::rand( Array<TYPE, FUN> &x )
 {
-    genRand<TYPE>( x.length(), x.data() );
+    for ( size_t i = 0; i < x.length(); i++ )
+        x( i ) = genRand<TYPE>();
 }
-
+*/
 
 /********************************************************
  *  Reduction                                            *
  ********************************************************/
 template<class TYPE, class FUN, typename LAMBDA>
-inline TYPE FunctionTable::reduce( LAMBDA& op, const Array<TYPE, FUN>& A )
+inline TYPE FunctionTable::reduce( LAMBDA &op, const Array<TYPE, FUN> &A, const TYPE &initialValue )
 {
     if ( A.length() == 0 )
         return TYPE();
-    const TYPE* x  = A.data();
-    TYPE y         = x[0];
-    const size_t N = A.length();
-    for ( size_t i = 1; i < N; i++ )
+    const TYPE *x = A.data();
+    TYPE y        = initialValue;
+    for ( size_t i = 0; i < A.length(); i++ )
         y = op( x[i], y );
     return y;
 }
+template<class TYPE, class FUN, typename LAMBDA>
+inline TYPE FunctionTable::reduce( LAMBDA &op,
+                                   const Array<TYPE, FUN> &A,
+                                   const Array<TYPE, FUN> &B,
+                                   const TYPE &initialValue )
+{
+    ARRAY_ASSERT( A.length() == B.length() );
+    if ( A.length() == 0 )
+        return TYPE();
+    const TYPE *x = A.data();
+    const TYPE *y = B.data();
+    TYPE z        = initialValue;
+    for ( size_t i = 0; i < A.length(); i++ )
+        z = op( x[i], y[i], z );
+    return z;
+}
 
 
 /********************************************************
  *  Unary transformation                                 *
  ********************************************************/
 template<class TYPE, class FUN, typename LAMBDA>
-inline void FunctionTable::transform( LAMBDA& fun, const Array<TYPE, FUN>& x, Array<TYPE, FUN>& y )
+inline void FunctionTable::transform( LAMBDA &fun, const Array<TYPE, FUN> &x, Array<TYPE, FUN> &y )
 {
     y.resize( x.size() );
     const size_t N = x.length();
@@ -69,8 +66,10 @@ inline void FunctionTable::transform( LAMBDA& fun, const Array<TYPE, FUN>& x, Ar
         y( i ) = fun( x( i ) );
 }
 template<class TYPE, class FUN, typename LAMBDA>
-inline void FunctionTable::transform(
-    LAMBDA& fun, const Array<TYPE, FUN>& x, const Array<TYPE, FUN>& y, Array<TYPE, FUN>& z )
+inline void FunctionTable::transform( LAMBDA &fun,
+                                      const Array<TYPE, FUN> &x,
+                                      const Array<TYPE, FUN> &y,
+                                      Array<TYPE, FUN> &z )
 {
     if ( x.size() != y.size() )
         throw std::logic_error( "Sizes of x and y do not match" );
@@ -85,25 +84,19 @@ inline void FunctionTable::transform(
  *  axpy                                                 *
  ********************************************************/
 template<class TYPE>
-inline void call_axpy( size_t N, const TYPE alpha, const TYPE* x, TYPE* y );
+void call_axpy( size_t N, const TYPE alpha, const TYPE *x, TYPE *y );
 template<>
-inline void call_axpy<float>( size_t, const float, const float*, float* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_axpy<float>( size_t N, const float alpha, const float *x, float *y );
 template<>
-inline void call_axpy<double>( size_t, const double, const double*, double* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_axpy<double>( size_t N, const double alpha, const double *x, double *y );
 template<class TYPE>
-inline void call_axpy( size_t N, const TYPE alpha, const TYPE* x, TYPE* y )
+void call_axpy( size_t N, const TYPE alpha, const TYPE *x, TYPE *y )
 {
     for ( size_t i = 0; i < N; i++ )
         y[i] += alpha * x[i];
 }
 template<class TYPE, class FUN>
-void FunctionTable::axpy( const TYPE alpha, const Array<TYPE, FUN>& x, Array<TYPE, FUN>& y )
+void FunctionTable::axpy( const TYPE alpha, const Array<TYPE, FUN> &x, Array<TYPE, FUN> &y )
 {
     if ( x.size() != y.size() )
         throw std::logic_error( "Array sizes do not match" );
@@ -115,21 +108,15 @@ void FunctionTable::axpy( const TYPE alpha, const Array<TYPE, FUN>& x, Array<TYP
  *  Multiply two arrays                                  *
  ********************************************************/
 template<class TYPE>
-inline void call_gemv( size_t M, size_t N, TYPE alpha, TYPE beta, const TYPE* A, const TYPE* x, TYPE* y );
+void call_gemv( size_t M, size_t N, TYPE alpha, TYPE beta, const TYPE *A, const TYPE *x, TYPE *y );
 template<>
-inline void call_gemv<double>(
-    size_t, size_t, double, double, const double*, const double*, double* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_gemv<double>(
+    size_t M, size_t N, double alpha, double beta, const double *A, const double *x, double *y );
 template<>
-inline void call_gemv<float>( size_t, size_t, float, float, const float*, const float*, float* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_gemv<float>(
+    size_t M, size_t N, float alpha, float beta, const float *A, const float *x, float *y );
 template<class TYPE>
-inline void call_gemv(
-    size_t M, size_t N, TYPE alpha, TYPE beta, const TYPE* A, const TYPE* x, TYPE* y )
+void call_gemv( size_t M, size_t N, TYPE alpha, TYPE beta, const TYPE *A, const TYPE *x, TYPE *y )
 {
     for ( size_t i = 0; i < M; i++ )
         y[i] = beta * y[i];
@@ -139,21 +126,29 @@ inline void call_gemv(
     }
 }
 template<class TYPE>
-inline void call_gemm(
-    size_t M, size_t N, size_t K, TYPE alpha, TYPE beta, const TYPE* A, const TYPE* B, TYPE* C );
+void call_gemm(
+    size_t M, size_t N, size_t K, TYPE alpha, TYPE beta, const TYPE *A, const TYPE *B, TYPE *C );
 template<>
-inline void call_gemm<double>( size_t, size_t, size_t, double, double, const double*, const double*, double* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_gemm<double>( size_t M,
+                        size_t N,
+                        size_t K,
+                        double alpha,
+                        double beta,
+                        const double *A,
+                        const double *B,
+                        double *C );
 template<>
-inline void call_gemm<float>( size_t, size_t, size_t, float, float, const float*, const float*, float* )
-{
-    throw std::logic_error( "LapackWrappers not configured" );
-}
+void call_gemm<float>( size_t M,
+                       size_t N,
+                       size_t K,
+                       float alpha,
+                       float beta,
+                       const float *A,
+                       const float *B,
+                       float *C );
 template<class TYPE>
-inline void call_gemm(
-    size_t M, size_t N, size_t K, TYPE alpha, TYPE beta, const TYPE* A, const TYPE* B, TYPE* C )
+void call_gemm(
+    size_t M, size_t N, size_t K, TYPE alpha, TYPE beta, const TYPE *A, const TYPE *B, TYPE *C )
 {
     for ( size_t i = 0; i < K * M; i++ )
         C[i] = beta * C[i];
@@ -165,16 +160,17 @@ inline void call_gemm(
     }
 }
 template<class TYPE, class FUN>
-void FunctionTable::gemm( const TYPE alpha, const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b,
-    const TYPE beta, Array<TYPE, FUN>& c )
+void FunctionTable::gemm( const TYPE alpha,
+                          const Array<TYPE, FUN> &a,
+                          const Array<TYPE, FUN> &b,
+                          const TYPE beta,
+                          Array<TYPE, FUN> &c )
 {
+    if ( a.size( 1 ) != b.size( 0 ) )
+        throw std::logic_error( "Inner dimensions must match" );
     if ( a.ndim() == 2 && b.ndim() == 1 ) {
-        if ( a.size( 1 ) != b.size( 0 ) )
-            throw std::logic_error( "Inner dimensions must match" );
         call_gemv<TYPE>( a.size( 0 ), a.size( 1 ), alpha, beta, a.data(), b.data(), c.data() );
     } else if ( a.ndim() <= 2 && b.ndim() <= 2 ) {
-        if ( a.size( 1 ) != b.size( 0 ) )
-            throw std::logic_error( "Inner dimensions must match" );
         call_gemm<TYPE>(
             a.size( 0 ), a.size( 1 ), b.size( 1 ), alpha, beta, a.data(), b.data(), c.data() );
     } else {
@@ -182,17 +178,16 @@ void FunctionTable::gemm( const TYPE alpha, const Array<TYPE, FUN>& a, const Arr
     }
 }
 template<class TYPE, class FUN>
-void FunctionTable::multiply(
-    const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, Array<TYPE, FUN>& c )
+void FunctionTable::multiply( const Array<TYPE, FUN> &a,
+                              const Array<TYPE, FUN> &b,
+                              Array<TYPE, FUN> &c )
 {
+    if ( a.size( 1 ) != b.size( 0 ) )
+        throw std::logic_error( "Inner dimensions must match" );
     if ( a.ndim() == 2 && b.ndim() == 1 ) {
-        if ( a.size( 1 ) != b.size( 0 ) )
-            throw std::logic_error( "Inner dimensions must match" );
         c.resize( a.size( 0 ) );
         call_gemv<TYPE>( a.size( 0 ), a.size( 1 ), 1, 0, a.data(), b.data(), c.data() );
     } else if ( a.ndim() <= 2 && b.ndim() <= 2 ) {
-        if ( a.size( 1 ) != b.size( 0 ) )
-            throw std::logic_error( "Inner dimensions must match" );
         c.resize( a.size( 0 ), b.size( 1 ) );
         call_gemm<TYPE>(
             a.size( 0 ), a.size( 1 ), b.size( 1 ), 1, 0, a.data(), b.data(), c.data() );
@@ -206,8 +201,8 @@ void FunctionTable::multiply(
  *  Check if two arrays are equal                        *
  ********************************************************/
 template<class TYPE, class FUN>
-inline typename std::enable_if<!std::is_floating_point<TYPE>::value, bool>::type
-FunctionTableCompare( const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, TYPE )
+inline typename std::enable_if<std::is_integral<TYPE>::value, bool>::type
+FunctionTableCompare( const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, TYPE )
 {
     bool pass = true;
     if ( a.size() != b.size() )
@@ -218,7 +213,7 @@ FunctionTableCompare( const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, TYPE
 }
 template<class TYPE, class FUN>
 inline typename std::enable_if<std::is_floating_point<TYPE>::value, bool>::type
-FunctionTableCompare( const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, TYPE tol )
+FunctionTableCompare( const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, TYPE tol )
 {
     bool pass = true;
     if ( a.size() != b.size() )
@@ -228,10 +223,89 @@ FunctionTableCompare( const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, TYPE
     return pass;
 }
 template<class TYPE, class FUN>
-bool FunctionTable::equals( const Array<TYPE, FUN>& a, const Array<TYPE, FUN>& b, TYPE tol )
+bool FunctionTable::equals( const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, TYPE tol )
 {
     return FunctionTableCompare( a, b, tol );
 }
 
 
+/********************************************************
+ *  Specialized Functions                                *
+ ********************************************************/
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformReLU( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B )
+{
+    const auto &fun = []( const TYPE &a ) { return std::max( a, static_cast<TYPE>( 0 ) ); };
+    transform( fun, A, B );
+}
+
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformAbs( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B )
+{
+    B.resize( A.size() );
+    const auto &fun = []( const TYPE &a ) { return std::abs( a ); };
+    transform( fun, A, B );
+}
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformTanh( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B )
+{
+    B.resize( A.size() );
+    const auto &fun = []( const TYPE &a ) { return tanh( a ); };
+    transform( fun, A, B );
+}
+
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformHardTanh( const Array<TYPE, FUN, ALLOC> &A,
+                                       Array<TYPE, FUN, ALLOC> &B )
+{
+    B.resize( A.size() );
+    const auto &fun = []( const TYPE &a ) {
+        return std::max( -static_cast<TYPE>( 1.0 ), std::min( static_cast<TYPE>( 1.0 ), a ) );
+    };
+    transform( fun, A, B );
+}
+
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformSigmoid( const Array<TYPE, FUN, ALLOC> &A, Array<TYPE, FUN, ALLOC> &B )
+{
+    B.resize( A.size() );
+    const auto &fun = []( const TYPE &a ) { return 1.0 / ( 1.0 + exp( -a ) ); };
+    transform( fun, A, B );
+}
+
+template<class TYPE, class FUN, class ALLOC>
+void FunctionTable::transformSoftPlus( const Array<TYPE, FUN, ALLOC> &A,
+                                       Array<TYPE, FUN, ALLOC> &B )
+{
+    B.resize( A.size() );
+    const auto &fun = []( const TYPE &a ) { return log1p( exp( a ) ); };
+    transform( fun, A, B );
+}
+
+template<class TYPE, class FUN, class ALLOC>
+TYPE FunctionTable::sum( const Array<TYPE, FUN, ALLOC> &A )
+{
+    const auto &fun = []( const TYPE &a, const TYPE &b ) { return a + b; };
+    return reduce( fun, A, (TYPE) 0 );
+}
+
+template<class TYPE>
+inline void FunctionTable::gemmWrapper( char TRANSA,
+                                        char TRANSB,
+                                        int M,
+                                        int N,
+                                        int K,
+                                        TYPE alpha,
+                                        const TYPE *A,
+                                        int LDA,
+                                        const TYPE *B,
+                                        int LDB,
+                                        TYPE beta,
+                                        TYPE *C,
+                                        int LDC )
+{
+    ERROR("Not finished");
+}
+
+
 #endif

common/MPI.h

@@ -821,7 +821,7 @@ public: // Member functions
      * @return           Output array for allGather
      */
     template<class type>
-    std::vector<type> allGather( const std::vector<type> &x_in ) const;
+    std::vector<type> allGather( const std::vector<type> &x ) const;
 
 
     /*!

common/ScaLBL.cpp

@@ -2297,24 +2297,70 @@ void ScaLBL_Communicator::D3Q7_Ion_Concentration_BC_Z(int *neighborList, double
 	}
 }
 
-void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){
+void ScaLBL_Communicator::D3Q7_Ion_Flux_Diff_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){
 	if (kproc == 0) {
 		if (time%2==0){
-			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+			ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
 		}
 		else{
-			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+			ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
 		}
 	}
 }
 
-void ScaLBL_Communicator::D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){
+void ScaLBL_Communicator::D3Q7_Ion_Flux_Diff_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){
 	if (kproc == nprocz-1){
 		if (time%2==0){
-			ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+			ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
 		}
 		else{
-			ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+			ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_DiffAdvc_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_DiffAdvc_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, sendCount_Z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_DiffAdvcElec_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                          double Di, double zi, double Vt, int time){
+	if (kproc == 0) {
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z(dvcSendList_z, fq, Cin, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, sendCount_z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z(neighborList, dvcSendList_z, fq, Cin, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, sendCount_z, N);
+		}
+	}
+}
+
+void ScaLBL_Communicator::D3Q7_Ion_Flux_DiffAdvcElec_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                          double Di, double zi, double Vt, int time){
+	if (kproc == nprocz-1){
+		if (time%2==0){
+			ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z(dvcSendList_Z, fq, Cout, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, sendCount_Z, N);
+		}
+		else{
+			ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z(neighborList, dvcSendList_Z, fq, Cout, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, sendCount_Z, N);
 		}
 	}
 }

common/ScaLBL.h

@@ -1,5 +1,5 @@
-/* ScaLBL.h 
- *  Header file for Scalable Lattice Boltzmann Library
+/** @file  ScaLBL.h */
+/*  \details Header file for Scalable Lattice Boltzmann Library
  *  Separate implementations for GPU and CPU must both follow the conventions defined in this header
  *  This libarry contains the essential components of the LBM
  *     - streaming implementations
@@ -11,48 +11,194 @@
 #define ScalLBL_H
 #include "common/Domain.h"
 
+/**
+* \brief Set compute device
+* @param rank      rank of MPI process 
+*/
 extern "C" int ScaLBL_SetDevice(int rank);
 
+/**
+* \brief Allocate memory 
+* @param address      memory address
+* @param size         size in bytes
+*/
 extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size);
 
+/**
+* \brief Free memory
+* @param pointer         pointer to memory to free
+*/
 extern "C" void ScaLBL_FreeDeviceMemory(void* pointer);
 
+/**
+* \brief Copy memory from host to device 
+* \details  Device memory should be close to simulation (based on NUMA cost)
+*          Host memory may be a shared memory region (with possibly higher NUMA cost for simulation)
+*          Analysis routine should minimize NUMA for host memory (based on process placement)
+* @param dest    memory location to copy to
+* @param source  memory region to copy from
+* @param size    size of the region to copy in bytes        
+*/
 extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size);
 
+
+/**
+* \brief Copy memory from device to host
+* \details  Device memory should be close to simulation (based on NUMA cost)
+*          Host memory may be a shared memory region (with possibly higher NUMA cost for simulation)
+*          Analysis routine should minimize NUMA for host memory (based on process placement)
+* @param dest    memory location to copy to
+* @param source  memory region to copy from
+* @param size    size of the region to copy in bytes        
+*/
 extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size);
 
+/**
+=* \brief Allocate zero copy memory buffer (i.e. shared memory)
+* @param address      memory address
+* @param size         size in bytes
+*/
 extern "C" void ScaLBL_AllocateZeroCopy(void** address, size_t size);
 
+/**
+* \brief Copy memory from host to zero copy buffer
+* \details  Device memory should be close to simulation (based on NUMA cost)
+*          Host memory may be a shared memory region (with possibly higher NUMA cost for simulation)
+*          Analysis routine should minimize NUMA for host memory (based on process placement)
+* @param dest    memory location to copy to
+* @param source  memory region to copy from
+* @param size    size of the region to copy in bytes        
+*/
 extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t size);
 
+/**
+* \brief  Device barrier routine     
+*/
 extern "C" void ScaLBL_DeviceBarrier();
 
+/**
+* \brief  Pack D3Q19 distributions for communication
+* @param q  - index for distribution based on D3Q19 discrete velocity structure
+* @param list - list of distributions to communicate
+* @param start -  index to start parsing the list 
+* @param count -  number of values to pack 
+* @param sendbuf - memory buffer to hold values that will be sent
+* @param dist - memory buffer to hold the distributions
+* @param N - size of the distributions (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N);
 
+/**
+* \brief  Unpack D3Q19 distributions after communication
+* @param q  - index for distribution based on D3Q19 discrete velocity structure
+* @param list - list of distributions to communicate
+* @param start -  index to start parsing the list 
+* @param count -  number of values to unppack 
+* @param recvbuf - memory buffer where recieved values have been stored
+* @param dist - memory buffer to hold the distributions
+* @param N - size of the distributions (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N);
 
+/**
+* \brief Unpack D3Q7 distributions after communication
+* @param q  - index for distribution based on D3Q19 discrete velocity structure
+* @param list - list of distributions to communicate
+* @param start -  index to start parsing the list 
+* @param count -  number of values to unppack 
+* @param recvbuf - memory buffer where recieved values have been stored
+* @param dist - memory buffer to hold the distributions
+* @param N - size of the distributions (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list,  int start, int count, double *recvbuf, double *dist, int N);
 
+/**
+* \brief  Pack halo for scalar field to be prepare for communication
+* @param list - list of distributions to communicate
+* @param count -  number of values to ppack 
+* @param sendbuf - memory buffer to pack values into
+* @param Data - scalar field 
+* @param N - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N);
 
+/**
+* \brief  Pack halo for scalar field to be prepare for communication
+* @param list - list of distributions to communicate
+* @param count -  number of values to unppack 
+* @param recvbuf - memory buffer where recieved values have been stored
+* @param Data - scalar field 
+* @param N - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N);
 
+/**
+* \brief  Unpack values and compute Shan-Chen type of gradient
+* @param weight - weight value for gradient sum
+* @param Cqx - contribution to x-part of gradient
+* @param Cqy - contribution to y-part of gradient
+* @param Cqz - contribution to z-part of gradient
+* @param list - list of distributions to communicate
+* @param start - location to start reading the list
+* @param count -  number of values to unppack 
+* @param recvbuf - memory buffer where recieved values have been stored
+* @param phi - scalar field 
+* @param grad - gradient 
+* @param N - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, double Cqz, 
 		int *list, int start, int count, double *recvbuf, double *phi, double *grad, int N);
 
-extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N);
-
-extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N);
 
+/**
+* \brief Initialize D3Q19 distributions
+* @param Dist - D3Q19 distributions
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np);
 
+/**
+* \brief Compute momentum from D3Q19 distribution
+* @param dist - D3Q19 distributions
+* @param vel - memory buffer to store the momentum that is computed
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np);
 
+/**
+* \brief compute pressure from D3Q19 distribution
+* @param dist - D3Q19 distributions
+* @param press - memory buffer to store the pressure field that is computed
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np);
 
 // BGK MODEL
+/**
+* \brief BGK collision based on AA even access pattern for D3Q19 
+* @param dist - D3Q19 distributions
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+* @param rlx - relaxation parameter
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+*/
 extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
 
+/**
+* \brief BGK collision based on AA odd access pattern for D3Q19 
+* @param neighborList - neighbors based on D3Q19 lattice structure
+* @param dist - D3Q19 distributions
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+* @param rlx - relaxation parameter
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+*/
 extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
 
 // GREYSCALE MODEL (Single-component)
@@ -88,12 +234,12 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
-        double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *Poros,double *Perm,double *Vel, double *Pressure,
+        double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *GreyKn, double *GreyKw, double *Poros,double *Perm,double *Vel, double *MobilityRatio, double *Pressure,
         double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
 		double Fx, double Fy, double Fz, bool RecoloringOff, int strideY, int strideZ, int start, int finish, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
-		double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *Poros,double *Perm,double *Vel,double *Pressure, 
+		double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *GreyKn, double *GreyKw, double *Poros, double *Perm,double *Vel, double *MobilityRatio, double *Pressure, 
         double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
 		double Fx, double Fy, double Fz, bool RecoloringOff, int strideY, int strideZ, int start, int finish, int Np);
 
@@ -107,10 +253,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *di
 extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
 
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                        double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
 
 extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
@@ -120,24 +266,73 @@ extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity
 
 // LBM Poisson solver
 
+/**
+* \brief Poisson-Boltzmann collision based on AA odd access pattern for D3Q19 
+* @param neighborList - neighbors based on D3Q19 lattice structure
+* @param Map - mapping between sparse and dense representations
+* @param dist - D3Q7 distributions
+* @param Den_charge - charge density
+* @param Psi - 
+* @param ElectricField - electric field
+* @param tau - relaxation time
+* @param epsilon_LB - 
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
         int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
-        int start, int finish, int Np);
-
+/**
+* \brief Poisson-Boltzmann collision based on AA even access pattern for D3Q7 
+* @param Map - mapping between sparse and dense representations
+* @param dist - D3Q7 distributions
+* @param Den_charge - charge density
+* @param Psi - 
+* @param ElectricField - electric field
+* @param tau - relaxation time
+* @param epsilon_LB - 
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
+extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau,
+					   double epsilon_LB, int start, int finish, int Np);
+/**
+* \brief Poisson-Boltzmann solver / solve electric potential based on AA odd access pattern for D3Q7 
+* @param neighborList - neighbors based on D3Q19 lattice structure
+* @param Map - mapping between sparse and dense representations
+* @param dist - D3Q7 distributions
+* @param Psi - 
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
 
+/**
+* \brief Poisson-Boltzmann solver / solve electric potential based on AA odd access pattern for D3Q7
+* @param Map - mapping between sparse and dense representations
+* @param dist - D3Q7 distributions
+* @param Psi - 
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
 
+/**
+* \brief Initialize Poisson-Boltzmann solver 
+* @param Map - mapping between sparse and dense representations
+* @param dist - D3Q7 distributions
+* @param Psi - 
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
 
-//maybe deprecated
-//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
-//        int strideY, int strideZ,int start, int finish, int Np);
-
 // LBM Stokes Model (adapted from MRT model)
-
 extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, 
                 double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
 
@@ -147,27 +342,138 @@ extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, do
 extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np);
 
 // MRT MODEL
+/**
+* \brief MRT collision based on AA even access pattern for D3Q19 
+* @param dist - D3Q19 distributions
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+* @param rlx_setA - relaxation parameter for viscous modes
+* @param rlx_setB - relaxation parameter for non-viscous modes
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+*/
 extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
 		double Fy, double Fz);
 
-extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int start, int finish, int Np,
+/**
+* \brief MRT collision based on AA even access pattern for D3Q19 
+* @param neighborList - index of neighbors based on D3Q19 lattice structure
+* @param dist - D3Q19 distributions
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+* @param rlx_setA - relaxation parameter for viscous modes
+* @param rlx_setB - relaxation parameter for non-viscous modes
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+*/
+extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *neighborList, double *dist, int start, int finish, int Np,
 		double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz);
 
 // COLOR MODEL
+/**
+* \brief Color model collision based on AA even access pattern for D3Q19 
+* @param Map - mapping between sparse and dense data structures
+* @param dist - D3Q19 distributions
+* @param Aq - D3Q7 distribution for component A
+* @param Bq - D3Q7 distribution for component B
+* @param Den - density field
+* @param Phi - phase indicator field
+* @param Vel - velocity field
+* @param rhoA - density of component A
+* @param rhoB - density of component B
+* @param tauA - relaxation time for component A
+* @param tauB - relaxation time for component B
+* @param alpha - parameter to control interfacial tension
+* @param beta - parameter to control interface width
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+* @param strideY - stride in y-direction for gradient computation
+* @param strideZ - stride in z-direction for gradient computation
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
 		double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q19_AAodd_Color(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+/**
+* \brief Color model collision based on AA even access pattern for D3Q19 
+* @param NeighborList - neighbors based on D3Q19 lattice structure
+* @param Map - mapping between sparse and dense data structures
+* @param dist - D3Q19 distributions
+* @param Aq - D3Q7 distribution for component A
+* @param Bq - D3Q7 distribution for component B
+* @param Den - density field
+* @param Phi - phase indicator field
+* @param Vel - velocity field
+* @param rhoA - density of component A
+* @param rhoB - density of component B
+* @param tauA - relaxation time for component A
+* @param tauB - relaxation time for component B
+* @param alpha - parameter to control interfacial tension
+* @param beta - parameter to control interface width
+* @param Fx - force in x direction
+* @param Fy - force in y direction
+* @param Fz - force in z direction
+* @param strideY - stride in y-direction for gradient computation
+* @param strideZ - stride in z-direction for gradient computation
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
+extern "C" void ScaLBL_D3Q19_AAodd_Color(int *NeighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
 		double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
 		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
 
+/**
+* \brief Compute phase field based on AA odd access pattern for D3Q19 
+* @param NeighborList - neighbors based on D3Q19 lattice structure
+* @param Map - mapping between sparse and dense data structures
+* @param Aq - D3Q7 distribution for component A
+* @param Bq - D3Q7 distribution for component B
+* @param Den - density field
+* @param Phi - phase indicator field
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double *Aq, double *Bq, 
 			double *Den, double *Phi, int start, int finish, int Np);
 
+/**
+* \brief Compute phase field based on AA even access pattern for D3Q19 
+* @param Map - mapping between sparse and dense data structures
+* @param Aq - D3Q7 distribution for component A
+* @param Bq - D3Q7 distribution for component B
+* @param Den - density field
+* @param Phi - phase indicator field
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
 extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi, 
 			int start, int finish, int Np);
 
+/**
+* \brief Initialize phase field for color model
+* @param Map - mapping between sparse and dense data structures
+* @param Phi - phase indicator field
+* @param Den - density field
+* @param Aq - D3Q7 distribution for component A
+* @param Bq - D3Q7 distribution for component B
+* @param start - lattice node to start loop
+* @param finish - lattice node to finish loop
+* @param Np - size of local sub-domain (derived from Domain structure)
+*/
+extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
+
+
 extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, 
 		double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
 
@@ -178,7 +484,6 @@ extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad,
 
 extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz);
 
-extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
 
 // Density functional hydrodynamics LBM
 extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
@@ -314,20 +619,39 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, in
 
 extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
-
-extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ,double *ElectricField,double Di,double zi,double Vt,int count,int Np);
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ,double *ElectricField,double Di,double zi,double Vt,int count,int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ,double *ElectricField,double Di,double zi,double Vt, int count, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ,double *ElectricField,double Di,double zi,double Vt, int count, int Np);
 
+/**
+ * \class ScaLBL_Communicator
+ *
+ * @brief Generalized communication routines for lattice Boltzmann methods.
+ *
+*/
 class ScaLBL_Communicator{
 public:
 	//......................................................................................
-	ScaLBL_Communicator(std::shared_ptr <Domain> Dm);
+        /**
+        *\brief Constructor
+	* @param Dm - Domain information
+        */
+        ScaLBL_Communicator(std::shared_ptr <Domain> Dm);
 
-	//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
+        /**
+        *\brief Destructor
+        */
 	~ScaLBL_Communicator();
 	//......................................................................................
 	unsigned long int CommunicationCount,SendCount,RecvCount;
@@ -393,8 +717,12 @@ public:
     void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout);
     void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time);
     void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time);
-    void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
-    void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
+    void D3Q7_Ion_Flux_Diff_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
+    void D3Q7_Ion_Flux_Diff_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
+    void D3Q7_Ion_Flux_DiffAdvc_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
+    void D3Q7_Ion_Flux_DiffAdvc_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
+    void D3Q7_Ion_Flux_DiffAdvcElec_BC_z(int *neighborList,double *fq,double Cin,double tau,double *VelocityZ,double *ElectricField_Z,double Di,double zi,double Vt, int time);
+    void D3Q7_Ion_Flux_DiffAdvcElec_BC_Z(int *neighborList,double *fq,double Cout,double tau,double *VelocityZ,double *ElectricField_Z,double Di,double zi,double Vt, int time);
     void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
     void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
     void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);

common/Utilities.h

@@ -30,6 +30,7 @@ using StackTrace::Utilities::sleep_s;
  * \details This routine will peform the default startup sequence
  * \param argc              argc from main
  * \param argv              argv from main
+ * \param multiple          Intialize mpi with MPI_THREAD_MULTIPLE support?
  */
 void startup( int argc, char **argv, bool multiple=true );
 

cpu/D3Q7BC.cpp

@@ -396,3 +396,406 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list,
 		dist[nr6] = f6;
 	}
 }
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*(f6+uz*fsum_partial))/(1.0-0.5/tau)/(1.0-uz); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*(f5-uz*fsum_partial))/(1.0-0.5/tau)/(1.0+uz); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+
+	for (int idx=0; idx<count; idx++){
+
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*(f6+uz*fsum_partial))/(1.0-0.5/tau)/(1.0-uz); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (int idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*(f6+uz*fsum_partial))/(1.0-0.5/tau)/(1.0-uz); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                              double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau-0.5*uz/tau-uEPz); 
+		dist[5*Np+n] = f6;
+	}
+}
+ 
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                             double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau+0.5*uz/tau+uEPz); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                              double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau-0.5*uz/tau-uEPz); 
+		dist[5*Np+n] = f6;
+	}
+}
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                             double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau-0.5*uz/tau-uEPz); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
+
+
+
+
+
+
+
+
+
+
+

cpu/GreyscaleColor.cpp

@@ -1271,7 +1271,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
 				-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17);
 		dist[16*Np+n] = fq;
 
-
 		// q = 17
 		fq = mrt_V1*rho+mrt_V9*m1
 				+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
@@ -1341,7 +1340,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
 //CP: capillary penalty
 // also turn off recoloring for grey nodes
 extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
-		 double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *Poros,double *Perm, double *Velocity, double *Pressure,
+		 double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *GreyKn, double *GreyKw, double *Poros,double *Perm, 
+		 double *Velocity, double *MobilityRatio, double *Pressure,
         double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff,double alpha, double beta,
 		double Gx, double Gy, double Gz, bool RecoloringOff, int strideY, int strideZ, int start, int finish, int Np){
 
@@ -1375,6 +1375,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
     double W;//greyscale wetting strength
     double Sn_grey,Sw_grey;
 
+    /* Corey model parameters */
+    double Kn_grey,Kw_grey;    
+    double Swn,Krn_grey,Krw_grey,mobility_ratio,jA,jB;
+    double GreyDiff=0.0e-4; // grey diffusion
+    
 	const double mrt_V1=0.05263157894736842;
 	const double mrt_V2=0.012531328320802;
 	const double mrt_V3=0.04761904761904762;
@@ -1394,16 +1399,19 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 		nB = Den[Np + n];
 
         porosity = Poros[n];
-        perm = Perm[n];
+        //GreyDiff = Perm[n];
+        perm = 1.0;
         W = GreySolidW[n];
         Sn_grey = GreySn[n];
         Sw_grey = GreySw[n];
-
+        Kn_grey = GreyKn[n];
+        Kw_grey = GreyKw[n];
+        
 		// compute phase indicator field
 		phi=(nA-nB)/(nA+nB);
-
 		// local density
 		rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA);
+		//rho0 *= porosity;
 		// local relaxation time
 		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
 		tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff);
@@ -1411,6 +1419,39 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 		rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
         mu_eff = (tau_eff-0.5)/3.0;//kinematic viscosity
 		
+        mobility_ratio = 1.0;
+        Krn_grey = 0.0;
+        Krw_grey = 0.0;
+        if (nA/(nA+nB)<Sn_grey && porosity !=1.0){
+        	perm = Kw_grey;
+        	Krw_grey = Kw_grey;
+        	Swn = 0.0;
+        }
+        else if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){ 
+        	Swn = (nA/(nA+nB) - Sn_grey) /(Sw_grey - Sn_grey);
+        	Krn_grey = Kn_grey*Swn*Swn; // Corey model with exponent = 2, make sure that W cannot shift to zero
+        	Krw_grey = Kw_grey*(1.0-Swn)*(1.0-Swn); // Corey model with exponent = 4, make sure that W cannot shift to zero
+        	// recompute the effective permeability
+        	perm = mu_eff*(Krn_grey*3.0/(tauA-0.5) + Krw_grey*3.0/(tauB-0.5));
+        	//mobility_ratio =(nA*Krn_grey*3.0/(tauA-0.5) - nB*Krw_grey*3.0/(tauB-0.5))/(nA*Krn_grey*3.0/(tauA-0.5) + nB*Krw_grey*3.0/(tauB-0.5));
+        }
+        else if (nA/(nA+nB)>Sw_grey && porosity !=1.0){
+        	perm = Kn_grey;
+        	Krn_grey = Kn_grey;
+        	Swn = 1.0;
+        }
+        /* compute the mobility ratio */
+        if (porosity != 1.0){
+        	mobility_ratio =(Krn_grey/(tauA-0.5) - Krw_grey/(tauB-0.5))/(Krn_grey/(tauA-0.5) + Krw_grey/(tauB-0.5));
+        }
+        else if (phi > 0.0){
+        	mobility_ratio = 1.0;
+        }
+        else {
+        	mobility_ratio = -1.0;
+        }
+        MobilityRatio[n] = mobility_ratio;
+
 		// Get the 1D index based on regular data layout
 		ijk = Map[n];
 		//					COMPUTE THE COLOR GRADIENT
@@ -2053,21 +2094,28 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 		nAB = 1.0/(nA+nB);
 		Aq[n] = 0.3333333333333333*nA;
 		Bq[n] = 0.3333333333333333*nB;
-
+		
 		//...............................................
 		// q = 0,2,4
 		// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+		jA = nA*ux;
+		jB = nB*ux;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*nx;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*nx;
+        	jA = 0.5*ux*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*ux*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		// q = 1
 		//nread = neighborList[n+Np];
@@ -2080,17 +2128,24 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 
 		//...............................................
 		// Cq = {0,1,0}
+		jA = nA*uy;
+		jB = nB*uy;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*ny;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*ny;
+    		jA = 0.5*uy*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*uy*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		// q = 3
 		//nread = neighborList[n+3*Np];
@@ -2104,17 +2159,25 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 		//...............................................
 		// q = 4
 		// Cq = {0,0,1}
+		jA = nA*uz;
+		jB = nB*uz;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*nz;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*nz;
+    		jA = 0.5*uz*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*uz*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		// q = 5
 		//nread = neighborList[n+5*Np];
@@ -2131,7 +2194,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor_CP(int *neighborList, int *Map
 //CP: capillary penalty
 // also turn off recoloring for grey nodes
 extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, double *Aq, double *Bq, double *Den, 
-        double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *Poros,double *Perm, double *Velocity, double *Pressure, 
+        double *Phi, double *GreySolidW, double *GreySn, double *GreySw, double *GreyKn, double *GreyKw, double *Poros,double *Perm, 
+        double *Velocity, double *MobilityRatio, double *Pressure, 
         double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
 		double Gx, double Gy, double Gz, bool RecoloringOff, int strideY, int strideZ, int start, int finish, int Np){
 
@@ -2152,6 +2216,11 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, do
     double W;//greyscale wetting strength
     double Sn_grey,Sw_grey;
     
+    /* Corey model parameters */
+    double Kn_grey,Kw_grey;
+    double Swn,Krn_grey,Krw_grey,mobility_ratio,jA,jB;
+    double GreyDiff=0.0e-4; // grey diffusion
+
     //double GeoFun=0.0;//geometric function from Guo's PRE 66, 036304 (2002)
     double porosity;
     double perm;//voxel permeability
@@ -2180,16 +2249,21 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, do
 		nB = Den[Np + n];
 
         porosity = Poros[n];
-        perm = Perm[n];
+        //GreyDiff = Perm[n];
+        perm = 1.0;
         W = GreySolidW[n];
         Sn_grey = GreySn[n];
         Sw_grey = GreySw[n];
-
+        Kn_grey = GreyKn[n];
+        Kw_grey = GreyKw[n];
+        
 		// compute phase indicator field
 		phi=(nA-nB)/(nA+nB);
 
 		// local density
 		rho0=rhoA + 0.5*(1.0-phi)*(rhoB-rhoA);
+		//rho0 *= porosity;
+
 		// local relaxation time
 		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
 		tau_eff=tauA_eff + 0.5*(1.0-phi)*(tauB_eff-tauA_eff);
@@ -2197,6 +2271,39 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, do
 		rlx_setB = 8.f*(2.f-rlx_setA)/(8.f-rlx_setA);
         mu_eff = (tau_eff-0.5)/3.0;//kinematic viscosity
 
+        mobility_ratio = 1.0;
+        Krn_grey = 0.0;
+        Krw_grey = 0.0;
+        if (nA/(nA+nB)<Sn_grey && porosity !=1.0){
+        	perm = Kw_grey;
+        	Krw_grey = Kw_grey;
+        	Swn = 0.0;
+        }
+        else if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){ 
+        	Swn = (nA/(nA+nB) - Sn_grey) /(Sw_grey - Sn_grey);
+        	Krn_grey = Kn_grey*Swn*Swn; // Corey model with exponent = 2, make sure that W cannot shift to zero
+        	Krw_grey = Kw_grey*(1.0-Swn)*(1.0-Swn); // Corey model with exponent = 4, make sure that W cannot shift to zero
+        	// recompute the effective permeability
+        	perm = mu_eff*(Krn_grey*3.0/(tauA-0.5) + Krw_grey*3.0/(tauB-0.5));
+        	//mobility_ratio =(nA*Krn_grey*3.0/(tauA-0.5) - nB*Krw_grey*3.0/(tauB-0.5))/(nA*Krn_grey*3.0/(tauA-0.5) + nB*Krw_grey*3.0/(tauB-0.5));
+        }
+        else if (nA/(nA+nB)>Sw_grey && porosity !=1.0){
+        	perm = Kn_grey;
+        	Krn_grey = Kn_grey;
+        	Swn = 1.0;
+        }
+        /* compute the mobility ratio */
+        if (porosity != 1.0){
+        	mobility_ratio =(Krn_grey/(tauA-0.5) - Krw_grey/(tauB-0.5))/(Krn_grey/(tauA-0.5) + Krw_grey/(tauB-0.5));
+        }
+        else if (phi > 0.0){
+        	mobility_ratio = 1.0;
+        }
+        else {
+        	mobility_ratio = -1.0;
+        }
+        MobilityRatio[n] = mobility_ratio;
+        
 		// Get the 1D index based on regular data layout
 		ijk = Map[n];
 		//					COMPUTE THE COLOR GRADIENT
@@ -2772,21 +2879,28 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, do
 		nAB = 1.0/(nA+nB);
 		Aq[n] = 0.3333333333333333*nA;
 		Bq[n] = 0.3333333333333333*nB;
-
+		
 		//...............................................
 		// q = 0,2,4
 		// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+		jA = nA*ux;
+		jB = nB*ux;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*nx;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*nx;
+    		jA = 0.5*ux*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*ux*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		Aq[1*Np+n] = a1;
 		Bq[1*Np+n] = b1;
@@ -2794,38 +2908,53 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor_CP(int *Map, double *dist, do
 		Bq[2*Np+n] = b2;
 
 		//...............................................
-		// q = 2
 		// Cq = {0,1,0}
+		jA = nA*uy;
+		jB = nB*uy;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*ny;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*ny;
+    		jA = 0.5*uy*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*uy*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		Aq[3*Np+n] = a1;
 		Bq[3*Np+n] = b1;
 		Aq[4*Np+n] = a2;
 		Bq[4*Np+n] = b2;
+
 		//...............................................
 		// q = 4
 		// Cq = {0,0,1}
+		jA = nA*uz;
+		jB = nB*uz;		
 		delta = beta*nA*nB*nAB*0.1111111111111111*nz;
 		if (!(nA*nB*nAB>0)) delta=0;
         //----------------newly added for better control of recoloring---------------//
-        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0) delta = 0.0; 
+        if (nA/(nA+nB)>=Sn_grey && nA/(nA+nB) <= Sw_grey && porosity !=1.0){
+        	//delta = 0.0; 
+        	delta = -0.111111111111111*C*W*GreyDiff*nA*nB*nAB*nz;
+    		jA = 0.5*uz*(nA+nB)*(1.0+mobility_ratio);
+    		jB = 0.5*uz*(nA+nB)*(1.0-mobility_ratio);
+        }
         if (nA/(nA+nB)>Sw_grey && porosity !=1.0) delta = -1.0*delta; 
         //---------------------------------------------------------------------------//
         if (RecoloringOff==true && porosity !=1.0) delta=0;
-		a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
-		b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
-		a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
-		b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+
+		a1 = (0.1111111111111111*(nA+4.5*jA))+delta;
+		b1 = (0.1111111111111111*(nB+4.5*jB))-delta;
+		a2 = (0.1111111111111111*(nA-4.5*jA))-delta;
+		b2 = (0.1111111111111111*(nB-4.5*jB))+delta;
 
 		Aq[5*Np+n] = a1;
 		Bq[5*Np+n] = b1;
@@ -2864,32 +2993,4 @@ extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, doubl
 	}
 }
 
-//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np){
-//	int n;
-//    double porosity;
-//	for (n=0; n<Np; n++){
-//        porosity = Porosity[n];
-//        if (porosity==0.0) porosity=1.f;
-//		dist[n] = 0.3333333333333333/porosity;
-//		dist[Np+n] = 0.055555555555555555/porosity;		//double(100*n)+1.f;
-//		dist[2*Np+n] = 0.055555555555555555/porosity;	//double(100*n)+2.f;
-//		dist[3*Np+n] = 0.055555555555555555/porosity;	//double(100*n)+3.f;
-//		dist[4*Np+n] = 0.055555555555555555/porosity;	//double(100*n)+4.f;
-//		dist[5*Np+n] = 0.055555555555555555/porosity;	//double(100*n)+5.f;
-//		dist[6*Np+n] = 0.055555555555555555/porosity;	//double(100*n)+6.f;
-//		dist[7*Np+n] = 0.0277777777777778/porosity;   //double(100*n)+7.f;
-//		dist[8*Np+n] = 0.0277777777777778/porosity;   //double(100*n)+8.f;
-//		dist[9*Np+n] = 0.0277777777777778/porosity;   //double(100*n)+9.f;
-//		dist[10*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+10.f;
-//		dist[11*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+11.f;
-//		dist[12*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+12.f;
-//		dist[13*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+13.f;
-//		dist[14*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+14.f;
-//		dist[15*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+15.f;
-//		dist[16*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+16.f;
-//		dist[17*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+17.f;
-//		dist[18*Np+n] = 0.0277777777777778/porosity;  //double(100*n)+18.f;
-//	}
-//}
-
 

cpu/Ion.cpp

@@ -80,13 +80,14 @@ extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, i
     }
 }
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	int n;
 	double Ci;
     double ux,uy,uz;
     double uEPx,uEPy,uEPz;//electrochemical induced velocity
     double Ex,Ey,Ez;//electrical field
+    double flux_diffusive_x,flux_diffusive_y,flux_diffusive_z;
 	double f0,f1,f2,f3,f4,f5,f6;
 	int nr1,nr2,nr3,nr4,nr5,nr6;
 
@@ -124,6 +125,20 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *D
 		// q=6
 		nr6 = neighborList[n+5*Np];
 		f6 = dist[nr6];
+
+        // compute diffusive flux
+		flux_diffusive_x = (1.0-0.5*rlx)*((f1-f2)-ux*Ci);
+		flux_diffusive_y = (1.0-0.5*rlx)*((f3-f4)-uy*Ci);
+		flux_diffusive_z = (1.0-0.5*rlx)*((f5-f6)-uz*Ci);
+        FluxDiffusive[n+0*Np] = flux_diffusive_x;
+        FluxDiffusive[n+1*Np] = flux_diffusive_y;
+        FluxDiffusive[n+2*Np] = flux_diffusive_z;
+        FluxAdvective[n+0*Np] = ux*Ci;
+        FluxAdvective[n+1*Np] = uy*Ci;
+        FluxAdvective[n+2*Np] = uz*Ci;
+        FluxElectrical[n+0*Np] = uEPx*Ci;
+        FluxElectrical[n+1*Np] = uEPy*Ci;
+        FluxElectrical[n+2*Np] = uEPz*Ci;
 		
 		// q=0
 		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
@@ -149,13 +164,14 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *D
 	}
 }
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                        double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	int n;
 	double Ci;
     double ux,uy,uz;
     double uEPx,uEPy,uEPz;//electrochemical induced velocity
     double Ex,Ey,Ez;//electrical field
+    double flux_diffusive_x,flux_diffusive_y,flux_diffusive_z;
 	double f0,f1,f2,f3,f4,f5,f6;
 
 	for (n=start; n<finish; n++){
@@ -180,6 +196,20 @@ extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Veloci
 		f5 = dist[6*Np+n];
 		f6 = dist[5*Np+n];
 		
+        // compute diffusive flux
+		flux_diffusive_x = (1.0-0.5*rlx)*((f1-f2)-ux*Ci);
+		flux_diffusive_y = (1.0-0.5*rlx)*((f3-f4)-uy*Ci);
+		flux_diffusive_z = (1.0-0.5*rlx)*((f5-f6)-uz*Ci);
+        FluxDiffusive[n+0*Np] = flux_diffusive_x;
+        FluxDiffusive[n+1*Np] = flux_diffusive_y;
+        FluxDiffusive[n+2*Np] = flux_diffusive_z;
+        FluxAdvective[n+0*Np] = ux*Ci;
+        FluxAdvective[n+1*Np] = uy*Ci;
+        FluxAdvective[n+2*Np] = uz*Ci;
+        FluxElectrical[n+0*Np] = uEPx*Ci;
+        FluxElectrical[n+1*Np] = uEPy*Ci;
+        FluxElectrical[n+2*Np] = uEPz*Ci;
+
 		// q=0
 		dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
 

cpu/Poisson.cpp

@@ -132,7 +132,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *d
 		f6 = dist[nr6];
 		
 		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
-		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice squared speed of sound
 		Ez = (f5-f6)*rlx*4.0;
         ElectricField[n+0*Np] = Ex;
         ElectricField[n+1*Np] = Ey;
@@ -189,7 +189,7 @@ extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_c
 
 
 		Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
-		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
+		Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice squared speed of sound
 		Ez = (f5-f6)*rlx*4.0;
         ElectricField[n+0*Np] = Ex;
         ElectricField[n+1*Np] = Ey;
@@ -235,6 +235,87 @@ extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, in
 	}
 }
 
+extern "C" void ScaLBL_D3Q7_PoissonResidualError(int *neighborList, int *Map, double *ResidualError, double *Psi, double *Den_charge, double epsilon_LB,int strideY, int strideZ,int start, int finish){
+
+	int n,nn,ijk;
+	double psi;//electric potential
+    double rho_e;//local charge density
+	// neighbors of electric potential psi
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+	double m3,m5,m7;
+    double psi_Laplacian;
+    double residual_error;
+
+	for (n=start; n<finish; n++){
+
+        //Load data
+        rho_e = Den_charge[n];
+        ijk=Map[n];
+        psi = Psi[ijk];
+
+		//					COMPUTE THE COLOR GRADIENT
+		//........................................................................
+		//.................Read Phase Indicator Values............................
+		//........................................................................
+		nn = ijk-1;							// neighbor index (get convention)
+		m1 = Psi[nn];						// get neighbor for phi - 1
+		//........................................................................
+		nn = ijk+1;							// neighbor index (get convention)
+		m2 = Psi[nn];						// get neighbor for phi - 2
+		//........................................................................
+		nn = ijk-strideY;							// neighbor index (get convention)
+		m3 = Psi[nn];					// get neighbor for phi - 3
+		//........................................................................
+		nn = ijk+strideY;							// neighbor index (get convention)
+		m4 = Psi[nn];					// get neighbor for phi - 4
+		//........................................................................
+		nn = ijk-strideZ;						// neighbor index (get convention)
+		m5 = Psi[nn];					// get neighbor for phi - 5
+		//........................................................................
+		nn = ijk+strideZ;						// neighbor index (get convention)
+		m6 = Psi[nn];					// get neighbor for phi - 6
+		//........................................................................
+		nn = ijk-strideY-1;						// neighbor index (get convention)
+		m7 = Psi[nn];					// get neighbor for phi - 7
+		//........................................................................
+		nn = ijk+strideY+1;						// neighbor index (get convention)
+		m8 = Psi[nn];					// get neighbor for phi - 8
+		//........................................................................
+		nn = ijk+strideY-1;						// neighbor index (get convention)
+		m9 = Psi[nn];					// get neighbor for phi - 9
+		//........................................................................
+		nn = ijk-strideY+1;						// neighbor index (get convention)
+		m10 = Psi[nn];					// get neighbor for phi - 10
+		//........................................................................
+		nn = ijk-strideZ-1;						// neighbor index (get convention)
+		m11 = Psi[nn];					// get neighbor for phi - 11
+		//........................................................................
+		nn = ijk+strideZ+1;						// neighbor index (get convention)
+		m12 = Psi[nn];					// get neighbor for phi - 12
+		//........................................................................
+		nn = ijk+strideZ-1;						// neighbor index (get convention)
+		m13 = Psi[nn];					// get neighbor for phi - 13
+		//........................................................................
+		nn = ijk-strideZ+1;						// neighbor index (get convention)
+		m14 = Psi[nn];					// get neighbor for phi - 14
+		//........................................................................
+		nn = ijk-strideZ-strideY;					// neighbor index (get convention)
+		m15 = Psi[nn];					// get neighbor for phi - 15
+		//........................................................................
+		nn = ijk+strideZ+strideY;					// neighbor index (get convention)
+		m16 = Psi[nn];					// get neighbor for phi - 16
+		//........................................................................
+		nn = ijk+strideZ-strideY;					// neighbor index (get convention)
+		m17 = Psi[nn];					// get neighbor for phi - 17
+		//........................................................................
+		nn = ijk-strideZ+strideY;					// neighbor index (get convention)
+		m18 = Psi[nn];					// get neighbor for phi - 18
+
+		psi_Laplacian = 2.0*3.0/18.0*(m1+m2+m3+m4+m5+m6-6*psi+0.5*(m7+m8+m9+m10+m11+m12+m13+m14+m15+m16+m17+m18-12*psi));//Laplacian of electric potential
+        residual_error = psi_Laplacian+rho_e/epsilon_LB;
+		ResidualError[n] = residual_error;
+	}
+}
 //extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
 //        int strideY, int strideZ,int start, int finish, int Np){
 //

cuda/D3Q7BC.cu

@@ -316,7 +316,133 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList
 	}
 }
 
-__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*(f6+uz*fsum_partial))/(1.0-0.5/tau)/(1.0-uz); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*(f5-uz*fsum_partial))/(1.0-0.5/tau)/(1.0+uz); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*(f6+uz*fsum_partial))/(1.0-0.5/tau)/(1.0-uz); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*(f5-uz*fsum_partial))/(1.0-0.5/tau)/(1.0+uz); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
 {
     //NOTE: FluxIn is the inward flux
     int idx,n;
@@ -340,7 +466,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, do
 	}
 }
 
-__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
 {
     //NOTE: FluxIn is the inward flux
     int idx,n;
@@ -364,7 +490,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, do
 	}
 }
 
-__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
 {
     //NOTE: FluxIn is the inward flux
 	int idx, n;
@@ -403,7 +529,7 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *li
 	}
 }
 
-__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
 {
     //NOTE: FluxIn is the inward flux
 	int idx, n;
@@ -441,6 +567,152 @@ __global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *li
 		dist[nr6] = f6;
 	}
 }
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                                  double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f6 = dist[5*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau+0.5*uz/tau+uEPz); 
+		dist[6*Np+n] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z, 
+                                                                  double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+    int idx,n;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau-0.5*uz/tau-uEPz); 
+		dist[5*Np+n] = f6;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                                 double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr5;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+		nread = d_neighborList[n+5*Np];
+		f6 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f6;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f5 =(FluxIn+(1.0-0.5/tau)*f6-(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau+0.5*uz/tau+uEPz); 
+
+		// Unknown distributions
+		nr5 = d_neighborList[n+4*Np];
+		dist[nr5] = f5;
+	}
+}
+
+__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                                 double Di, double zi, double Vt, int count, int Np)
+{
+    //NOTE: FluxIn is the inward flux
+	int idx, n;
+    int nread,nr6;
+	double f0,f1,f2,f3,f4,f5,f6;
+    double fsum_partial;
+    double uz;
+    double uEPz;//electrochemical induced velocity
+    double Ez;//electrical field
+	idx = blockIdx.x*blockDim.x + threadIdx.x;
+	if (idx < count){
+		n = list[idx];
+		f0 = dist[n];
+
+		nread = d_neighborList[n];
+		f1 = dist[nread];
+
+		nread = d_neighborList[n+2*Np];
+		f3 = dist[nread];
+
+		nread = d_neighborList[n+4*Np];
+		f5 = dist[nread];
+
+		nread = d_neighborList[n+Np];
+		f2 = dist[nread];
+
+		nread = d_neighborList[n+3*Np];
+		f4 = dist[nread];
+
+        fsum_partial = f0+f1+f2+f3+f4+f5;
+        uz = VelocityZ[n];
+        Ez = ElectricField_Z[n];
+        uEPz=zi*Di/Vt*Ez;
+		//...................................................
+        f6 =(FluxIn+(1.0-0.5/tau)*f5+(0.5*uz/tau+uEPz)*fsum_partial)/(1.0-0.5/tau-0.5*uz/tau-uEPz); 
+
+		// unknown distributions
+		nr6 = d_neighborList[n+5*Np];
+		dist[nr6] = f6;
+	}
+}
 //*************************************************************************
 
 extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
@@ -567,39 +839,116 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, in
 		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
 	}
 }
-
-extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+//------------Diff-----------------
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
 	int GRID = count / 512 + 1;
-	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_z (kernel): %s \n",cudaGetErrorString(err));
 	}
 }
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
 	int GRID = count / 512 + 1;
-	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_Diff_BC_Z (kernel): %s \n",cudaGetErrorString(err));
 	}
 }
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
 	int GRID = count / 512 + 1;
-	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_z (kernel): %s \n",cudaGetErrorString(err));
 	}
 }
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
 	int GRID = count / 512 + 1;
-	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
-		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_Diff_BC_Z (kernel): %s \n",cudaGetErrorString(err));
 	}
 }
+//----------DiffAdvc-------------
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvc_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvc_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+//----------DiffAdvcElec-------------
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                              double Di, double zi, double Vt, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                              double Di, double zi, double Vt, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_DiffAdvcElec_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                             double Di, double zi, double Vt, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, double *ElectricField_Z,
+                                                             double Di, double zi, double Vt, int count, int Np){
+	int GRID = count / 512 + 1;
+	dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, ElectricField_Z, Di, zi, Vt, count, Np);
+	cudaError_t err = cudaGetLastError();
+	if (cudaSuccess != err){
+		printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_DiffAdvcElec_BC_Z (kernel): %s \n",cudaGetErrorString(err));
+	}
+}
+//-------------------------------

cuda/Ion.cu

@@ -97,13 +97,14 @@ __global__  void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *D
 	}
 }
 
-__global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField, 
+__global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                            double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	int n;
 	double Ci;
     double ux,uy,uz;
     double uEPx,uEPy,uEPz;//electrochemical induced velocity
     double Ex,Ey,Ez;//electrical field
+    double flux_diffusive_x,flux_diffusive_y,flux_diffusive_z;
 	double f0,f1,f2,f3,f4,f5,f6;
 	int nr1,nr2,nr3,nr4,nr5,nr6;
 
@@ -146,6 +147,20 @@ __global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, doub
             nr6 = neighborList[n+5*Np];
             f6 = dist[nr6];
             
+            // compute diffusive flux
+            flux_diffusive_x = (1.0-0.5*rlx)*((f1-f2)-ux*Ci);
+            flux_diffusive_y = (1.0-0.5*rlx)*((f3-f4)-uy*Ci);
+            flux_diffusive_z = (1.0-0.5*rlx)*((f5-f6)-uz*Ci);
+            FluxDiffusive[n+0*Np] = flux_diffusive_x;
+            FluxDiffusive[n+1*Np] = flux_diffusive_y;
+            FluxDiffusive[n+2*Np] = flux_diffusive_z;
+            FluxAdvective[n+0*Np] = ux*Ci;
+            FluxAdvective[n+1*Np] = uy*Ci;
+            FluxAdvective[n+2*Np] = uz*Ci;
+            FluxElectrical[n+0*Np] = uEPx*Ci;
+            FluxElectrical[n+1*Np] = uEPy*Ci;
+            FluxElectrical[n+2*Np] = uEPz*Ci;
+
             // q=0
             dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
             //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
@@ -177,13 +192,14 @@ __global__  void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, doub
 	}
 }
 
-__global__  void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+__global__  void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                             double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	int n;
 	double Ci;
     double ux,uy,uz;
     double uEPx,uEPy,uEPz;//electrochemical induced velocity
     double Ex,Ey,Ez;//electrical field
+    double flux_diffusive_x,flux_diffusive_y,flux_diffusive_z;
 	double f0,f1,f2,f3,f4,f5,f6;
 
 	int S = Np/NBLOCKS/NTHREADS + 1;
@@ -212,6 +228,20 @@ __global__  void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *V
             f5 = dist[6*Np+n];
             f6 = dist[5*Np+n];
             
+            // compute diffusive flux
+            flux_diffusive_x = (1.0-0.5*rlx)*((f1-f2)-ux*Ci);
+            flux_diffusive_y = (1.0-0.5*rlx)*((f3-f4)-uy*Ci);
+            flux_diffusive_z = (1.0-0.5*rlx)*((f5-f6)-uz*Ci);
+            FluxDiffusive[n+0*Np] = flux_diffusive_x;
+            FluxDiffusive[n+1*Np] = flux_diffusive_y;
+            FluxDiffusive[n+2*Np] = flux_diffusive_z;
+            FluxAdvective[n+0*Np] = ux*Ci;
+            FluxAdvective[n+1*Np] = uy*Ci;
+            FluxAdvective[n+2*Np] = uz*Ci;
+            FluxElectrical[n+0*Np] = uEPx*Ci;
+            FluxElectrical[n+1*Np] = uEPy*Ci;
+            FluxElectrical[n+2*Np] = uEPz*Ci;
+
             // q=0
             dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
             //dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
@@ -330,10 +360,10 @@ extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, i
 	//cudaProfilerStop();
 }
 
-extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,  
+extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField,  
                                       double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	//cudaProfilerStart();
-	dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+	dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,FluxDiffusive,FluxAdvective,FluxElectrical,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
 
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){
@@ -342,10 +372,10 @@ extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *D
 	//cudaProfilerStop();
 }
 
-extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField, 
+extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *FluxDiffusive, double *FluxAdvective, double *FluxElectrical, double *Velocity, double *ElectricField, 
                                        double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
 	//cudaProfilerStart();
-	dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
+	dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,FluxDiffusive,FluxAdvective,FluxElectrical,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
 
 	cudaError_t err = cudaGetLastError();
 	if (cudaSuccess != err){

docs/README.md

@@ -6,14 +6,20 @@ pip install Sphinx
 # foamatting requires sphinx read-the-docs-theme
 pip install sphinx-rtd-theme
 
+# support for doxygen requires breathe
+pip install breathe
+
 # equation rendering requires latex and dvipng command
 sudo apt-get install dvipng
 sudo apt-get install texlive texstudio
 sudo apt-get install texlive-latex-recommended texlive-pictures texlive-latex-extra
 
-
 # To build the docs
 Step 1) install dependencies listed above
-Step 2) type 'make html' from the docs/ directory
-Step 3) point your browser at ~/local/doc/build/html/index.html
+Step 2) build LBPM with doxygen support enabled in CMake
+Step 3) From the build directory run the command 'make doc' to build html and xml from doxygen
+Step 4) modify source/conf.py so that the breathe directory specifies the full path to the xml
+        documentation built in Step 3
+Step 5) type 'make html' from the docs/ directory (the same directory that contains this README file)
+Step 6) point your browser at ~/local/doc/build/html/index.html
 # 

docs/source/conf.py

@@ -18,7 +18,7 @@ import sys
 # -- Project information -----------------------------------------------------
 
 project = 'LBPM'
-copyright = '2021, James E McClure'
+copyright = '2021'
 author = 'James E McClure'
 
 # The full version, including alpha/beta/rc tags
@@ -31,7 +31,8 @@ release = '1.0'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.imgmath'
+    'sphinx.ext.imgmath',
+    'breathe'
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -55,7 +56,6 @@ html_theme = 'alabaster'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
-
 ## Read the docs style:
 if os.environ.get('READTHEDOCS') != 'True':
     try:
@@ -68,3 +68,12 @@ if os.environ.get('READTHEDOCS') != 'True':
     
 #def setup(app):
 #    app.add_stylesheet("fix_rtd.css")
+
+
+# -- Breathe configuration -------------------------------------------------
+
+breathe_projects = {
+	"LBPM Doxygen": "/home/mcclurej/local/dev/LBPM/doc/xml/"
+}
+breathe_default_project = "LBPM Doxygen"
+breathe_default_members = ('members', 'undoc-members')

docs/source/developerGuide/designOverview/dataStructures.rst

@@ -3,4 +3,121 @@ Data Structures
 ===============
 
 LBPM includes a variety of generalized data structures to facilitate the implementation
-of different lattice Boltzmann models. 
+of different lattice Boltzmann models. These core data structures are designed so that
+they are agnostic to the particular physics of a model. Many different lattice Boltzmann
+schemes can be constructed using the same basic data structures. The core data structures
+are primarily designed to
+
+1) prepare 3D image data for distributed memory simulation based on MPI
+2) develop common data structures to facilitate the local streaming step
+3) generate and store information needed to exchange data between MPI processes
+4) develop common routines for parallel communication for lattice Boltzmann methods.
+
+By using the core data structures, it is possible to develop new physical models
+without having to revisit the data structures or parallel algorithms on which a model is
+built. Understanding and using the core data structures is very useful if you intend
+to build new physical models within LBPM. In most cases, only the collision step
+will need to be developed to implement a particular physical model. 
+
+
+------------------
+Domain
+------------------
+
+The ``Domain`` data structure includes information needed to carry out generic
+parallel computations on 3D image data. LBPM is designed to support parallel
+simulation where the image data is distributed over a potentially large number of
+processors. Each processor recieves an equal size chunk of the input image,
+which is specified based on the input database file in the form ``n = nx, ny, nz``.
+The total image size is specified as ``N = Nx, Ny, Nz``, which is needed to read
+the input image. In the typical case, the input image data will be read only by MPI
+rank 0, and sub-domains will be assigned and distributed to each MPI process.
+The regular process grid is specified as ``nproc = Px, Py, Pz``. If the entire image
+is to be distributed across the processors, the values must be chosen such that
+``Nx = nx*Px``, ``Ny = ny*Py`` and ``Nz = nz*Pz``. The basic purpose for the
+data structures contained in the ``Domain`` class is to retain basic information
+about the domain structure to support parallel computations in MPI. Code within the
+``analysis/`` directory will generally rely on these data structures to determine
+how to perform computations in parallel. For GPU-based application,  information
+contained within the ``Domain`` structure is usually retained only the CPU.
+Lattice Boltzmann algorithms rely on dedicated data structures within the ``ScaLBL``
+class to perform MPI computations, which are designed for these applications
+specifically.
+
+.. figure:: ../../_static/images/domain-decomp.png
+    :width: 600
+    :alt: domain decomposition
+
+    Domain decomposition used by LBPM to distribute 3D image data across processors.
+    Each processor recieves an equal size block of the input image. The Domain class
+    organizes MPI communications with adjacent blocks based on the processor layout.
+
+
+------------------
+ScaLBL
+------------------   
+
+
+While the ``Domain`` data structures retain essential information needed to
+carry out generic types of computations based on regular 3D image files,
+``ScaLBL`` data structures are designed specifically for lattice Boltzmann methods.
+The main purposes for these data structures are to:
+
+1) allow for overlap between computations that do not depend on MPI communication and the
+   MPI communications required by LBMs (e.g. to carry out the streaming step, perform
+   halo exchanges, etc.)
+2) reduce the computational and memory requirements by excluding image sites where flow
+   does not occur; and
+3) to facilitate efficient data access patterns based on the memory layout used by
+   data structures.
+
+These are critical steps to provide good parallel performance and make efficient use
+of GPUs. 
+
+
+In many cases flow will occur only in a fraction of the voxels from the original 3D input
+image. Since LBM schemes store a lot of data for each active voxels (e.g. 19 double precision
+values for a single D3Q19 model), significant memory savings can be realized by excluding
+immobile voxels from the data structure. Furthermore, it is useful to distiguish between
+the interior lattice sites (which do not require MPI communications to update) and the exterior
+lattice sites (which must wait for MPI communications to complete before the local update can
+be completed). Within LBPM it is almost always advantageous to overlap interior computations with
+MPI communications, so that the latter costs can be hidden behind the computational density
+of the main lattice Boltzmann kernels. As the local sub-domain size increases, the number of
+interior sites will tend to be significantly larger than the number of exterior sites. 
+The data layout is summarized in the image below.
+
+
+.. figure:: ../../_static/images/data-structures-v0.png
+	   :width: 600
+	   :alt: ScaLBL data structure layout
+
+	   Data structure used by ScaLBL to support lattice Boltzmann methods.
+	   Regions of the input image that are occupied by solid are excluded.
+	   Interior and exterior lattice sites are stored separately so that
+	   computations can efficiently overlap with MPI communication.
+
+Another factor in the data structure design are the underlying algorithms used by the LBM
+to carry out the streaming step. There are at least a half dozen distinct algorithms that
+can be used to perform streaming for lattice Boltzmann methods, each with their own advantages
+and disadvantages. Generally, the choice of streaming algorithm should reduce the overall memory
+footprint while also avoiding unnecessary memory accesses. LBPM uses the AA algorithm so
+that a single array is needed to perform the streaming step. The AA algorithm relies on the symmetry
+of the discrete velocity set used by LBMs to implement an efficient algorithm. Conceptually,
+we can easily see that for each distribution that needs to be accessed to perform streaming,
+there is an opposite distribution. The AA algorithm proceeds by identifying these distributions
+and exchanging their storage locations in memory, as depicted in the image below. The data
+access pattern will therefore alternate between even and odd timesteps. Within LBPM, a ``NeighborList``
+is constructed to store the memory location for the accompanying pair for each distribution.
+If the neighboring site is in the solid, the neighbor is specified such that the halfway
+bounceback rule will applied automatically. In this way, the streaming step can be performed
+very efficiently on either GPU or CPU.
+
+	   
+.. figure:: ../../_static/images/AA-stream.png
+	   :width: 600
+	   :alt: AA streaming pattern
+
+           Data access pattern for AA streaming algorithm used by LBPM. The location to read
+	   and write distributions alternates between even and odd timesteps. A solid bounceback
+	   rule is built into the data structure.

docs/source/developerGuide/doxygen/Analysis.rst

@@ -0,0 +1,9 @@
+###############################################################################
+Analysis methods
+###############################################################################
+
+.. toctree::
+   :glob:
+   :maxdepth: 2
+   
+   analysis/*

docs/source/developerGuide/doxygen/Domain.rst

@@ -0,0 +1,7 @@
+============================================
+Domain
+============================================
+.. doxygenfile:: Domain.h
+   :project: LBPM Doxygen
+
+

docs/source/developerGuide/doxygen/Models.rst

@@ -0,0 +1,10 @@
+###############################################################################
+Models
+###############################################################################
+
+.. toctree::
+   :glob:
+   :maxdepth: 2
+
+   models/*
+   

docs/source/developerGuide/doxygen/ScaLBL.rst

@@ -0,0 +1,5 @@
+============================================
+Scalable Lattice Boltzmann Library (ScalBL)
+============================================
+.. doxygenfile:: ScaLBL.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/analysis/DCEL.rst

@@ -0,0 +1,5 @@
+============================================
+DCEL
+============================================
+.. doxygenfile:: dcel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/analysis/FlowAdaptor.rst

@@ -0,0 +1,5 @@
+============================================
+FlowAdaptor
+============================================
+.. doxygenfile:: FlowAdaptor.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/analysis/Minkowski.rst

@@ -0,0 +1,5 @@
+============================================
+Minkowski
+============================================
+.. doxygenfile:: Minkowski.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/analysis/Morphology.rst

@@ -0,0 +1,5 @@
+============================================
+Morphology
+============================================
+.. doxygenfile:: morphology.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/analysis/SubPhase.rst

@@ -0,0 +1,5 @@
+============================================
+SubPhase analysis 
+============================================
+.. doxygenfile:: SubPhase.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/ColorModel.rst

@@ -0,0 +1,5 @@
+============================================
+Color Model 
+============================================
+.. doxygenfile:: ColorModel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/FreeEnergyModel.rst

@@ -0,0 +1,5 @@
+============================================
+Free Energy Model
+============================================
+.. doxygenfile:: FreeLeeModel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/GreyscaleModel.rst

@@ -0,0 +1,5 @@
+============================================
+Greyscale Model 
+============================================
+.. doxygenfile:: GreyscaleModel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/MRTModel.rst

@@ -0,0 +1,5 @@
+============================================
+Multi-relaxation time (MRT) Model 
+============================================
+.. doxygenfile:: MRTModel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/NernstPlanckModel.rst

@@ -0,0 +1,5 @@
+============================================
+Ion Model 
+============================================
+.. doxygenfile:: IonModel.h
+   :project: LBPM Doxygen

docs/source/developerGuide/doxygen/models/PoissonModel.rst

@@ -0,0 +1,5 @@
+============================================
+Poisson Model 
+============================================
+.. doxygenfile:: PoissonSolver.h
+   :project: LBPM Doxygen

docs/source/developerGuide/index.rst

@@ -15,4 +15,10 @@ into the framework.
 
    testingModels/*
 
- 
+   doxygen/ScaLBL.rst
+
+   doxygen/Domain.rst
+
+   doxygen/Models.rst
+
+   doxygen/Analysis.rst

docs/source/examples/color/steadyState.rst

@@ -0,0 +1,5 @@
+*******************
+Steady-state flow
+*******************
+
+In this example we simulate a steady-state flow with a constant driving force.

docs/source/examples/domain/domain.rst

@@ -0,0 +1,52 @@
+*****************
+Input Domain
+*****************
+
+LBPM provides a flexible framework to ingest 3D image data.
+To illustrate the basic capabilities, this tutorial considers a quasi-2D
+flow cell. Source files for the example are included in the LBPM repository
+in the directory ``examples/DiscPack``. A simple python code is included
+to set up the flow domain.
+
+Based on LBPM convention, external boundary conditions are applied in the
+z-direction. This means that the domain should be set up so that the direction
+you want to set boundary conditions is aligned with the z-axis. For the quasi-2D
+example, a depth of ``3`` voxels is used for the x-direction. *Based on LBPM
+internal data structures at least three voxels must be provided in each direction*
+The specified domain decomposition must also observe this rule.
+
+Image data is stored internally within LBPM as signed 8-bit binary data. This means that
+up to 256 labels can be provided in the input image. LBPM convention takes all
+non-positive labels to be immobile (treated as solid). In this example, the solid regions
+are assigned a value of ``0``.  It is possible to provide up to ``128`` different labels
+for the solid. Also, note that python supports only the unsigned 8-bit datatype. For the unsigned data
+type, labels assigned values ``128,...255`` in python will correspond to labels
+``-127,...-1`` when read in as type ``signed char`` within LBPM. 
+
+.. code:: python
+
+	  import numpy as np
+	  import matplotlib.pylab as plt
+	  import pandas as pd
+	  # Set the size of the domain
+	  Nx=3
+	  Ny=128
+	  Nz=128
+	  D=pd.read_csv("discs.csv",sep=" ")
+	  ID = np.ones(Nx*Ny*Nz,dtype='uint8')
+	  ID.shape = (Nz,Ny,Nx)
+	  # Set the solid labels
+	  for idx in range(len(D)):
+             cx=D['cx'][idx] / dx
+             cy=D['cy'][idx] /dx
+             r=D['r'][idx] /dx
+             for i in range(0,Nz):
+                for j in range(0,Ny):
+                   if ( (cx-i)*(cx-i) + (cy-j)*(cy-j) < r*r ):
+                      ID[i,j,0] = 0
+                      ID[i,j,1] = 0
+                      ID[i,j,2] = 0
+        # write input file to disc
+	ID.tofile("discs_3x128x128.raw")
+
+

docs/source/examples/morphology/morphOpen.rst

@@ -0,0 +1,110 @@
+*****************************
+Morphological pre-processors
+*****************************
+
+LBPM includes morphological pre-processing tools as utility functions.
+It is often useful to generate initial conditions for a 2-phase flow simulation based on a morphological approach. In particular, morphological tools can be used to provide a physical reasonable initial condition in cases where direct experimental observations are not available. These initial configurations are compatible with any of the 2-phase simulation protocols used by lbpm_color_simulator. These initialization approaches alter the fluid labels within the input files, writing a new file with the new morphologically assigned labels.
+
+There are two main morphological pre-processors in LBPM
+
+* ``lbpm_morphdrain_pp`` -- initialize fluid configuration based on morphological drainage
+* ``lbpm_morphopen_pp`` -- initialize fluid configuration based on morphological opening
+
+Here we demonstrate ``lbpm_morphdrain_pp`` because it offers the most information. Although it is not perfect, the morphological drainage operation does a good job of approximating configurations observed along primary drainage processes performed under water-wet conditions. A limitation is that fluid trapped in the corners will not stop the morphological operation from eroding it. This should not discourage you too much -- morphological tools are very practical and can save you a lot of time! It is also a good thing to be skeptical.
+
+Since the morphological operation works on the input domain, associated parameters are added to the ``Domain`` section of the input file. Here we will set a target saturation Sw = 0.20, which will run the morphological drainage operation until the fluid labeled as 2 occupies 20% of the pore space or less. For the case considered in
+``example/DiscPack`` we specify the following information in the input file
+
+.. code:: c
+
+	  Domain {
+	     Filename = "discs_3x128x128.raw"
+	     ReadType = "8bit"    // data type
+	     N = 3, 128, 128       // size of original image
+	     nproc = 1, 2, 2       // process grid
+	     n = 3, 64, 64         // sub-domain size
+	     voxel_length = 1.0    // voxel length (in microns)
+	     ReadValues = 0, 1, 2  // labels within the original image
+	     WriteValues = 0, 2, 2 // associated labels to be used by LBPM
+	     BC = 0                // fully periodic BC
+	     Sw = 0.35             // target saturation for morphological tools
+	  }
+
+Once this has been set, we launch lbpm_morphdrain_pp in the same way as other parallel tools
+
+.. code:: bash
+
+	  mpirun -np 4 $LBPM_BIN/lbpm_morphdrain_pp input.db
+
+Successful output looks like the following
+
+
+.. code:: bash
+
+
+	  Performing morphological opening with target saturation 0.350000 
+	  voxel length = 1.000000 micron 
+	  voxel length = 1.000000 micron 
+	  Input media: discs_3x128x128.raw
+	  Relabeling 3 values
+	  oldvalue=0, newvalue =0 
+	  oldvalue=1, newvalue =2 
+	  oldvalue=2, newvalue =2 
+	  Dimensions of segmented image: 3 x 128 x 128 
+	  Reading 8-bit input data 
+	  Read segmented data from discs_3x128x128.raw 
+	  Label=0, Count=11862 
+	  Label=1, Count=37290 
+	  Label=2, Count=0 
+	  Distributing subdomains across 4 processors 
+	  Process grid: 1 x 2 x 2 
+	  Subdomain size: 3 x 64 x 64 
+	  Size of transition region: 0 
+	  Media porosity = 0.758667 
+	  Initialized solid phase -- Converting to Signed Distance function 
+	  Volume fraction for morphological opening: 0.758667 
+	  Maximum pore size: 116.773801 
+	     1.000000      110.935111
+	     1.000000      105.388355
+	     1.000000      100.118937
+	     1.000000      95.112990
+	     1.000000      90.357341
+	     1.000000      85.839474
+	     1.000000      81.547500
+	     1.000000      77.470125
+	     1.000000      73.596619
+	     1.000000      69.916788
+	     1.000000      66.420949
+	     1.000000      63.099901
+	     1.000000      59.944906
+	     1.000000      56.947661
+	     1.000000      54.100278
+	     1.000000      51.395264
+	     1.000000      48.825501
+	     1.000000      46.384226
+	     1.000000      44.065014
+	     1.000000      41.861764
+	     1.000000      39.768675
+	     1.000000      37.780242
+	     1.000000      35.891230
+	     1.000000      34.096668
+	     1.000000      32.391835
+	     0.575114      30.772243
+	     0.433119      29.233631
+	     0.291231      27.771949
+	  Final void fraction =0.291231
+	  Final critical radius=27.771949
+	  Writing ID file 
+	  Writing file to: discs_3x128x128.raw.morphdrain.raw
+
+
+The final configuration can be visualized in python by loading the output file
+``discs_3x128x128.raw.morphdrain.raw``.
+
+
+.. figure:: ../../_static/images/DiscPack-morphdrain.png
+    :width: 600
+    :alt: morphdrain
+
+    Fluid configuration resulting from orphological drainage algorithm applied to a 2D disc pack.
+

docs/source/examples/running.rst

@@ -1,9 +1,21 @@
-============
-Running LBPM
-============
+==============
+Examples 
+==============
 
 There are two main components to running LBPM simulators.
 First is understanding how to launch MPI tasks on your system,
 which depends on the particular implementation of MPI that you are using,
 as well as other details of the local configuration. The second component is
-understanding the LBPM input file structure. 
+understanding the LBPM input file structure. The examples included provide
+a basic introduction to working with LBPM. 
+
+.. toctree::
+   :glob:
+   :maxdepth: 2
+
+   domain/*
+
+   morphology/*	      
+   
+   color/*
+   

docs/source/index.rst

@@ -12,9 +12,9 @@ LBPM -- Documentation
    :caption: Contents:
 
    install
-   examples/*
    userGuide/*
    developerGuide/*
+   examples/*
    publications/*
 
 Indices and tables

docs/source/install.rst

@@ -2,7 +2,161 @@
 Installation
 ============
 
-LBPM requires CMake, MPI and HDF5 as required dependencies. 
+Sample scripts to configure and build LBPM are included in the `sample_scripts` directory. To build this package, MPI and cmake are required, along with a compiler that supports the C++-14 standard. Building in source is not supported.
+
+The essential dependencies needed to build LBPM are:
+
+1.  cmake (version 3.9 or higher)
+2.  MPI
+3.  HDF5
+4.  silo
+
+*************************
+Building Dependencies 
+*************************
+(skip if they are already installed on your system)
+
+1. Download third-party library source files
+
+.. code-block:: bash
+
+   zlib-1.2.11.tar.gz
+   hdf5-1.8.12.tar.gz 
+   silo-4.10.2.tar.gz 
+
+
+2. Set up path where you want to install 
+
+.. code-block:: bash
+
+   export MPI_DIR=/path/to/mpi/
+   export LBPM_ZLIB_DIR=/path/to/zlib
+   export LBPM_HDF5_DIR=/path/to/hdf5
+   export LBPM_SILO_DIR=/path/to/silo
+
+3. Build third-party library dependencies 
+
+.. code-block:: bash
+
+   tar -xzvf zlib-1.2.11.tar.gz 
+   tar -xzvf hdf5-1.8.12.tar.gz 
+   tar -xzvf silo-4.10.2.tar.gz 
+   cd zlib-1.2.11
+
+   ./configure --prefix=$LBPM_ZLIB_DIR && make && make install
+
+   cd ../hdf5-1.8.12
+
+   CC=$MPI_DIR/bin/mpicc  CXX=$MPI_DIR/bin/mpicxx CXXFLAGS="-fPIC -O3 -std=c++14" \
+   ./configure --prefix=$LBPM_HDF5_DIR --enable-parallel --enable-shared --with-zlib=$LBPM_ZLIB_DIR 
+    make && make install
+
+
+   cd ../silo-4.10.2
+
+   CC=$MPI_DIR/bin/mpicc  CXX=$MPI_DIR/bin/mpicxx CXXFLAGS="-fPIC -O3 -std=c++14" \
+   ./configure --prefix=$LBPM_SILO_DIR -with-hdf5=$LBPM_HDF5_DIR/include,$LBPM_HDF5_DIR/lib --enable-static 
+    make && make install
+ 
+*************************
+Building LBPM 
+*************************
+
+Many HPC systems will include all of these dependencies, and LBPM can be built simply by setting the paths to the associated libraries in the cmake configure line. 
+
+The steps typically used to build LBPM are as follows:
+
+1. Set environment variables for the source directory `$LBPM_WIA_SOURCE` and build directory `$LBPM_WIA_DIR`  
+
+.. code-block:: bash
+
+   export LBPM_SOURCE=/path/to/source/LBPM  
+   export LBPM_DIR=/path/to/build/LBPM
+
+2. Set environment variables for the path to HDF5 and SILO (required),  and optionally for TimerUtility and NetCDF (optional)  
+
+.. code-block:: bash
+
+   export LBPM_HDF5_DIR=/path/to/hdf5  
+   export LBPM_SILO_DIR=/path/to/silo 
+   export LBPM_TIMER_DIR=/path/to/timer 
+   export LBPM_NETCDF_DIR=/path/to/netcdf
+
+3. Create the build directory and navigate to it  
+
+.. code-block:: bash
+
+   mkdir $LBPM_WIA_DIR  
+   cd $LBPM_WIA_DIR
+
+4. Configure the project. Numerous scripts exist to build LBPM on different HPC clusters, which are available in the  `$LBPM_SOURCE/sample_scripts/` directory. It is often possible to run these scripts directly if one exists for a system similar to the one you are building on. For a standard CPU build:  
+
+.. code-block:: bash
+
+    cmake                                           \  
+        -D CMAKE_BUILD_TYPE:STRING=Release          \   
+        -D CMAKE_C_COMPILER:PATH=mpicc              \   
+        -D CMAKE_CXX_COMPILER:PATH=mpicxx           \  
+        -D CMAKE_C_FLAGS="-fPIC"                    \  
+        -D CMAKE_CXX_FLAGS="-fPIC"                  \  
+        -D CMAKE_CXX_STD=14                         \  
+        -D USE_TIMER=0                              \  
+            -D TIMER_DIRECTORY=$LBPM_TIMER_DIR     \  
+        -D USE_NETCDF=0                             \  
+            -D NETCDF_DIRECTORY=$LBPM_NETCDF_DIR   \  
+        -D USE_SILO=1                               \  
+           -D HDF5_DIRECTORY=$LBPM_HDF5_DIR         \  
+           -D SILO_DIRECTORY=$LBPM_SILO_DIR         \  
+        -D USE_CUDA=0                               \
+        $LBPM_SOURCE  
+
+For GPU support, it is necessary to have CUDA along with a GPU-aware MPI implementation. Otherwise, the LBPM routines should behave identically irrespective of the underlying hardware.  
+
+.. code-block:: bash
+
+    cmake                                           \  
+        -D CMAKE_BUILD_TYPE:STRING=Release          \   
+        -D CMAKE_C_COMPILER:PATH=mpicc              \   
+        -D CMAKE_CXX_COMPILER:PATH=mpicxx           \  
+        -D CMAKE_C_FLAGS="-fPIC"                    \  
+        -D CMAKE_CXX_FLAGS="-fPIC"                  \  
+        -D CMAKE_CXX_STD=14                         \  
+        -D USE_TIMER=0                              \  
+            -D TIMER_DIRECTORY=$LBPM_TIMER_DIR     \  
+        -D USE_NETCDF=0                             \  
+            -D NETCDF_DIRECTORY=$LBPM_NETCDF_DIR   \  
+        -D USE_SILO=1                               \  
+           -D HDF5_DIRECTORY=$LBPM_HDF5_DIR         \  
+           -D SILO_DIRECTORY=$LBPM_SILO_DIR         \  
+        -D USE_CUDA=1                               \  
+        -D CMAKE_CUDA_FLAGS="-arch sm_70"           \  
+        $LBPM_SOURCE   
+
+5. Build the project (using four cores to build)   
+
+.. code-block:: bash
+  
+   make -j4
+
+6. Install the project  
+
+.. code-block:: bash
+
+   make install
+
+7. Run the tests to make sure they execute correctly (on a cluster, it is recommended to run these using the batch system rather than on the head node)  
+
+.. code-block:: bash
+
+   ctest
+
+
+*************************
+Sample Scripts
+*************************
+
+The LBPM repository contains sample scripts showing successful CMake configuration, build and
+install steps for a range of systems. Refer to the project sub-directory below for these examples.
 
 .. code-block:: bash
 

docs/source/userGuide/IO/fileformat.rst

@@ -5,9 +5,9 @@ I/O conventions for LBPM
 There are three main kinds of output file that are supported by LBPM.
 
 
-* CSV files --
+* CSV files -- space-delimited CSV files are used by the internal analysis framework
 
-* formatted binary files --
+* formatted binary files -- SILO and HDF5 formats are supported for visualization data
   
-* unformatted binary files --
+* unformatted binary files -- ``.raw`` extension
 

docs/source/userGuide/analysis/analysis.rst

@@ -1,5 +0,0 @@
-===========================
-Internal Analysis Framework
-===========================
-
-placeholder for analysis 

docs/source/userGuide/index.rst

@@ -10,8 +10,6 @@ Welcome to the LBPM user guide.
 
    models/*
 
-   analysis/*
-
    visualization/*
 
    IO/*

docs/source/userGuide/models/color/analysis/basic.rst

@@ -0,0 +1,38 @@
+======================================
+Color model -- Basic Analysis
+======================================
+
+The basic analysis routine for the LBPM color model logs a time series
+of averaged information to the space-delimited CSV file ``timelog.csv``.
+The ``analysis_interval`` key should be specified to control the interval at
+to perform basic analysis, which corresponds to the number of simulation timesteps
+to perform between analyses. The basic analysis routine is designed to
+be lightweight so that it can be performed frequently, and it is almost always
+useful to enable it. Results  will be immediately logged
+to ``timelog.csv`` which can be used to assess whether or not the simulation is
+behaving as expected. Furthermore, the analysis framework will check
+simulation measures to verfiy that no ``NaN`` are detected for the fluid
+pressure and flow rate.
+
+The list of measures logged to ``timelog.csv`` are defined as follows.
+The region of space occupied by the wetting fluid is determined from the
+phase indicator field, :math:`\Omega_w:\phi<0` 
+
+
+* ``sw`` -- water saturation (fluid component 2)
+* ``krw`` -- water effective permeability 
+* ``krw`` -- effective permeability for fluid w
+* ``krn`` -- effective permeability for fluid n
+* ``krwf`` -- effective permeability for fluid w (with film uncertainty estimate)
+* ``krnf`` -- effective permeability for fluid n (with film uncertainty estimate)
+* ``vw`` -- speed for the non-wetting fluid
+* ``vn`` -- speed for the wetting fluid
+* ``force`` -- magnitude for effective driving force
+* ``pw`` -- average pressure for fluid w
+* ``pn`` -- average pressure for fluid n
+* ``wet`` -- total solid wetting energy
+
+  
+More comprehensive analysis is performed in the ``subphase`` analysis module. 
+
+