attached gpl license

.gitignore

@@ -0,0 +1,9 @@
+*~
+*.o
+*.o*
+*.e*
+*.dat
+*.swp
+hybrid
+bin
+lib

.project

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>LBPM-WIA-Source@LBPM-WIA</name>
+	<comment></comment>
+	<projects></projects>
+	<buildSpec></buildSpec>
+	<natures></natures>
+	<linkedResources>
+		<link>
+			<name>[Subprojects]</name>
+			<type>2</type>
+			<locationURI>virtual:/virtual</locationURI>
+		</link>
+	</linkedResources>
+</projectDescription>

CMakeLists.txt

@@ -0,0 +1,181 @@
+# Set some CMake properties    
+CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
+
+
+MESSAGE("====================")
+MESSAGE("Configuring LBPM-WIA")
+MESSAGE("====================")
+
+
+# Set the project name
+SET( PROJ LBPM )          # Set the project name for CMake
+SET( LBPM_LIB lbpm-wia )  # Set the final library name
+SET( LBPM_INC  )          # Set an optional subfolder for includes (e.g. include/name/...)
+SET( TEST_MAX_PROCS 16 )
+
+
+# Initialize the project
+PROJECT( ${PROJ} LANGUAGES CXX )
+
+
+# Prevent users from building in place
+IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )
+    MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )
+ENDIF()
+
+
+# Set the default C++ standard
+SET( CMAKE_CXX_EXTENSIONS OFF )
+IF ( NOT CMAKE_CXX_STANDARD )
+    IF ( CXX_STD )
+        MESSAGE( WARNING "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" )
+        SET( CMAKE_CXX_STANDARD ${CXX_STD} )
+    ENDIF()
+    SET( CMAKE_CXX_STANDARD 11 )
+ENDIF()
+
+
+# Disable Link Time Optimization until we have a chance to test
+SET( DISABLE_LTO )
+
+
+# Set source/install paths
+SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
+SET( ${PROJ}_BUILD_DIR  "${CMAKE_CURRENT_BINARY_DIR}" )
+IF( ${PROJ}_INSTALL_DIR )
+    SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" )
+ELSEIF( PREFIX )
+    SET( ${PROJ}_INSTALL_DIR "${PREFIX}" )
+ELSEIF( NOT ${PROJ}_INSTALL_DIR )
+    SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
+ENDIF()
+INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
+SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake )
+
+
+# Include macros
+INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" )
+INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" )
+INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" )
+
+
+# Check if we are only compiling docs
+CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
+
+
+# Set testing paramaters
+SET( DROP_METHOD "http" )
+SET( DROP_SITE "" )
+SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
+SET( TRIGGER_SITE "" )
+SET( DROP_SITE_CDASH TRUE )
+ENABLE_TESTING()
+INCLUDE( CTest )
+
+
+# Check the compile mode and compile flags
+IF ( NOT ONLY_BUILD_DOCS )
+    CONFIGURE_SYSTEM()
+ENDIF()
+
+
+# Add some directories to include
+INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
+
+
+# Create the target for documentation
+ADD_CUSTOM_TARGET( doc )
+ADD_CUSTOM_TARGET( latex_docs )
+CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
+CHECK_ENABLE_FLAG( USE_LATEX 1 )
+FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
+IF ( USE_DOXYGEN )
+    SET( DOXYFILE_LATEX YES )
+    SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
+    SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
+    SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )
+    SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" )
+    SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" )
+    SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" )
+    SET( REL_PACKAGE_HTML "" )
+    SET( DOXYGEN_MACROS "" )
+    MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
+    INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" )
+    IF ( DOXYGEN_FOUND )
+        ADD_DEPENDENCIES( doxygen latex_docs )
+        ADD_DEPENDENCIES( doc latex_docs doxygen )
+    ELSE()
+        SET( USE_DOXYGEN 0 )
+    ENDIF()
+ENDIF()
+
+
+# Create custom targets for build-test, check, and distclean
+ADD_CUSTOM_TARGET( build-test )
+ADD_CUSTOM_TARGET( build-examples )
+ADD_CUSTOM_TARGET( check COMMAND  make test  )
+ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common visit IO threadpool )
+
+
+# Check for CUDA
+CHECK_ENABLE_FLAG( USE_CUDA 0 )
+NULL_USE( CMAKE_CUDA_FLAGS )
+IF ( USE_CUDA )
+    ADD_DEFINITIONS( -DUSE_CUDA )
+    ENABLE_LANGUAGE( CUDA )
+ENDIF()
+
+
+# Configure external packages
+IF ( NOT ONLY_BUILD_DOCS )
+    CONFIGURE_MPI()     # MPI must be before other libraries
+    CONFIGURE_MIC()
+    CONFIGURE_NETCDF()
+    CONFIGURE_SILO()
+    CONFIGURE_LBPM()
+    CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
+    CONFIGURE_LINE_COVERAGE()
+    INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/SharedPtr.cmake" )
+    CONFIGURE_SHARED_PTR( "${${PROJ}_INSTALL_DIR}/include" "std" )
+    # Set the external library link list
+    SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
+ENDIF()
+
+
+
+# Macro to create 1,2,4 processor tests
+MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
+    ADD_LBPM_TEST( ${EXENAME} ${ARGN} )
+    ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} )
+    ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} )
+ENDMACRO()
+
+
+# Add the src directories
+IF ( NOT ONLY_BUILD_DOCS )
+    BEGIN_PACKAGE_CONFIG( lbpm-wia-library )
+    ADD_PACKAGE_SUBDIRECTORY( common )
+    ADD_PACKAGE_SUBDIRECTORY( analysis )
+    ADD_PACKAGE_SUBDIRECTORY( IO )
+    ADD_PACKAGE_SUBDIRECTORY( threadpool )
+    IF ( USE_CUDA )
+        ADD_PACKAGE_SUBDIRECTORY( gpu )
+    ELSE()
+        ADD_PACKAGE_SUBDIRECTORY( cpu )
+    ENDIF()
+    INSTALL_LBPM_TARGET( lbpm-wia-library  )
+    ADD_SUBDIRECTORY( tests )
+    ADD_SUBDIRECTORY( threadpool/test )
+    ADD_SUBDIRECTORY( example )
+    #ADD_SUBDIRECTORY( workflows )
+    INSTALL_PROJ_LIB()
+ENDIF()
+
+
+# Create the visit plugin
+IF ( USE_VISIT )
+    INCLUDE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindVisIt.cmake )
+    VISIT_PLUGIN( visit LBM )
+ENDIF()
+
+

IO/IOHelpers.h

@@ -0,0 +1,64 @@
+#ifndef IO_HELPERS_INC
+#define IO_HELPERS_INC
+
+#include <string.h>
+#include <vector>
+
+namespace IO {
+
+
+// Find a character in a line
+inline size_t find( const char *line, char token )
+{
+    size_t i=0;
+    while ( 1 ) {
+        if ( line[i]==token || line[i]<32 || line[i]==0 )
+            break;
+        ++i;
+    }
+    return i;
+}
+
+
+// Remove preceeding/trailing whitespace
+inline std::string deblank( const std::string& str )
+{
+    size_t i1 = str.size();
+    size_t i2 = 0;
+    for (size_t i=0; i<str.size(); i++) {
+        if ( str[i]!=' ' && str[i]>=32 ) {
+            i1 = std::min(i1,i);
+            i2 = std::max(i2,i);
+        }
+    }
+    return str.substr(i1,i2-i1+1);
+}
+
+
+// Split a list by the given token
+inline std::vector<std::string> splitList( const char *line, const char token )
+{
+    std::vector<std::string> list;
+    size_t i1 = 0;
+    size_t i2 = 0;
+    while ( 1 ) {
+        if ( line[i2]==token || line[i2]<32 ) {
+            std::string tmp(&line[i1],i2-i1);
+            tmp = deblank(tmp);
+            if ( !tmp.empty() )
+                list.push_back(tmp);
+            i1 = i2+1;
+        }
+        if ( line[i2]==0 )
+            break;
+        i2++;
+    }
+    return list;
+}
+
+
+
+};
+
+#endif
+

IO/Mesh.cpp

@@ -0,0 +1,481 @@
+#include "Mesh.h"
+#include "common/Utilities.h"
+#include "shared_ptr.h"
+
+#include <limits>
+#include <stdint.h>
+
+namespace IO {
+
+
+inline Point nullPoint()
+{
+    Point tmp;
+    tmp.x = std::numeric_limits<double>::quiet_NaN();
+    tmp.y = std::numeric_limits<double>::quiet_NaN();
+    tmp.z = std::numeric_limits<double>::quiet_NaN();
+    return tmp;
+}
+
+
+/****************************************************
+* Mesh                                              *
+****************************************************/
+Mesh::Mesh( )
+{
+}
+Mesh::~Mesh( )
+{
+}
+
+
+/****************************************************
+* MeshDataStruct                                    *
+****************************************************/
+bool MeshDataStruct::check() const
+{
+    enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
+    bool pass = mesh != nullptr;
+    for ( const auto& var : vars ) {
+        pass = pass && static_cast<int>(var->type)>=1 && static_cast<int>(var->type)<=3;
+        pass = pass && !var->data.empty();
+    }
+    if ( !pass )
+        return false;
+    const std::string& meshClass = mesh->className();
+    if ( meshClass == "PointList" ) {
+        const auto mesh2 = dynamic_cast<IO::PointList*>( mesh.get() );
+        if ( mesh2 == nullptr )
+            return false;
+        for ( const auto& var : vars ) {
+            if ( var->type == IO::VariableType::NodeVariable ) {
+                pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim;
+            } else if ( var->type == IO::VariableType::EdgeVariable ) {
+                ERROR("Invalid type for PointList");
+            } else if ( var->type == IO::VariableType::SurfaceVariable ) {
+                ERROR("Invalid type for PointList");
+            } else if ( var->type == IO::VariableType::VolumeVariable ) {
+                ERROR("Invalid type for PointList");
+            } else {
+                ERROR("Invalid variable type");
+            }
+        }
+    } else if ( meshClass == "TriMesh" || meshClass == "TriList" ) {
+        const auto mesh2 = getTriMesh( mesh );
+        if ( mesh2 == nullptr )
+            return false;
+        for ( const auto& var : vars ) {
+            if ( var->type == IO::VariableType::NodeVariable ) {
+                pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim;
+            } else if ( var->type == IO::VariableType::EdgeVariable ) {
+                ERROR("Not finished");
+            } else if ( var->type == IO::VariableType::SurfaceVariable ) {
+                ERROR("Not finished");
+            } else if ( var->type == IO::VariableType::VolumeVariable ) {
+                pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim;
+            } else {
+                ERROR("Invalid variable type");
+            }
+        }
+    } else if ( meshClass == "DomainMesh" ) {
+        const auto mesh2 = dynamic_cast<IO::DomainMesh*>( mesh.get() );
+        if ( mesh2 == nullptr )
+            return false;
+        for ( const auto& var : vars ) {
+            if ( var->type == IO::VariableType::NodeVariable ) {
+                pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1)
+                            && (int) var->data.size(2)==(mesh2->nz+1) &&       var->data.size(3)==var->dim;
+            } else if ( var->type == IO::VariableType::EdgeVariable ) {
+                ERROR("Not finished");
+            } else if ( var->type == IO::VariableType::SurfaceVariable ) {
+                ERROR("Not finished");
+            } else if ( var->type == IO::VariableType::VolumeVariable ) {
+                pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny
+                            && (int) var->data.size(2)==mesh2->nz &&       var->data.size(3)==var->dim;
+            } else {
+                ERROR("Invalid variable type");
+            }
+        }
+    } else {
+        ERROR("Unknown mesh class: "+mesh->className());
+    }
+    return pass;
+}
+
+
+/****************************************************
+* PointList                                         *
+****************************************************/
+PointList::PointList( )
+{
+}
+PointList::PointList( size_t N )
+{
+    Point tmp = nullPoint();
+    points.resize(N,tmp);
+}
+PointList::~PointList( )
+{
+}
+size_t PointList::numberPointsVar( VariableType type ) const
+{
+    size_t N = 0;
+    if ( type == VariableType::NodeVariable )
+        N = points.size();
+    return N;
+}
+std::pair<size_t,void*> PointList::pack( int level ) const
+{
+    std::pair<size_t,void*> data_out(0,NULL);
+    if ( level==0 ) {
+        data_out.first = (2+3*points.size())*sizeof(double);
+        double *data_ptr = new double[2+3*points.size()];
+        data_out.second = data_ptr;
+        uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
+        data_int[0] = level;
+        data_int[1] = points.size();
+        double *data = &data_ptr[2];
+        for (size_t i=0; i<points.size(); i++) {
+            data[3*i+0] = points[i].x;
+            data[3*i+1] = points[i].y;
+            data[3*i+2] = points[i].z;
+        }
+    }
+    return data_out;
+}
+void PointList::unpack( const std::pair<size_t,void*>& data_in )
+{
+    uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
+    const double *data = reinterpret_cast<const double*>(data_in.second);
+    int level = data_int[0];
+    uint64_t N = data_int[1];
+    data = &data[2];
+    if ( level==0 ) {
+        ASSERT((2+3*N)*sizeof(double)==data_in.first);
+        points.resize(N);
+        for (size_t i=0; i<points.size(); i++) {
+            points[i].x = data[3*i+0];
+            points[i].y = data[3*i+1];
+            points[i].z = data[3*i+2];
+        }
+    }
+}
+
+
+/****************************************************
+* TriList                                           *
+****************************************************/
+TriList::TriList( )
+{
+}
+TriList::TriList( size_t N_tri )
+{
+    Point tmp = nullPoint();
+    A.resize(N_tri,tmp);
+    B.resize(N_tri,tmp);
+    C.resize(N_tri,tmp);
+}
+TriList::TriList( const TriMesh& mesh )
+{
+    Point tmp = nullPoint();
+    A.resize(mesh.A.size(),tmp);
+    B.resize(mesh.B.size(),tmp);
+    C.resize(mesh.C.size(),tmp);
+    ASSERT(mesh.vertices.get()!=NULL);
+    const std::vector<Point>& P = mesh.vertices->points;
+    for (size_t i=0; i<A.size(); i++)
+        A[i] = P[mesh.A[i]];
+    for (size_t i=0; i<B.size(); i++)
+        B[i] = P[mesh.B[i]];
+    for (size_t i=0; i<C.size(); i++)
+        C[i] = P[mesh.C[i]];
+}
+TriList::~TriList( )
+{
+}
+size_t TriList::numberPointsVar( VariableType type ) const
+{
+    size_t N = 0;
+    if ( type==VariableType::NodeVariable )
+        N = 3*A.size();
+    else if ( type==VariableType::SurfaceVariable ||  type==VariableType::VolumeVariable )
+        N = A.size();
+    return N;
+}
+std::pair<size_t,void*> TriList::pack( int level ) const
+{
+    std::pair<size_t,void*> data_out(0,NULL);
+    if ( level==0 ) {
+        data_out.first = (2+9*A.size())*sizeof(double);
+        double *data_ptr = new double[2+9*A.size()];
+        data_out.second = data_ptr;
+        uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
+        data_int[0] = level;
+        data_int[1] = A.size();
+        double *data = &data_ptr[2];
+        for (size_t i=0; i<A.size(); i++) {
+            data[9*i+0] = A[i].x;
+            data[9*i+1] = A[i].y;
+            data[9*i+2] = A[i].z;
+            data[9*i+3] = B[i].x;
+            data[9*i+4] = B[i].y;
+            data[9*i+5] = B[i].z;
+            data[9*i+6] = C[i].x;
+            data[9*i+7] = C[i].y;
+            data[9*i+8] = C[i].z;
+        }
+    }
+    return data_out;
+}
+void TriList::unpack( const std::pair<size_t,void*>& data_in )
+{
+    uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
+    const double *data = reinterpret_cast<const double*>(data_in.second);
+    int level = data_int[0];
+    uint64_t N = data_int[1];
+    data = &data[2];
+    if ( level==0 ) {
+        ASSERT((2+9*N)*sizeof(double)==data_in.first);
+        A.resize(N);
+        B.resize(N);
+        C.resize(N);
+        for (size_t i=0; i<A.size(); i++) {
+            A[i].x = data[9*i+0];
+            A[i].y = data[9*i+1];
+            A[i].z = data[9*i+2];
+            B[i].x = data[9*i+3];
+            B[i].y = data[9*i+4];
+            B[i].z = data[9*i+5];
+            C[i].x = data[9*i+6];
+            C[i].y = data[9*i+7];
+            C[i].z = data[9*i+8];
+        }
+    }
+}
+
+
+/****************************************************
+* TriMesh                                           *
+****************************************************/
+TriMesh::TriMesh(  )
+{
+}
+TriMesh::TriMesh( size_t N_tri, size_t N_point )
+{
+    vertices.reset( new PointList(N_point) );
+    A.resize(N_tri,-1);
+    B.resize(N_tri,-1);
+    C.resize(N_tri,-1);
+}
+TriMesh::TriMesh( size_t N_tri, std::shared_ptr<PointList> points )
+{
+    vertices = points;
+    A.resize(N_tri,-1);
+    B.resize(N_tri,-1);
+    C.resize(N_tri,-1);
+}
+TriMesh::TriMesh( const TriList& mesh )
+{
+    // For simlicity we will just create a mesh with ~3x the verticies for now
+    ASSERT(mesh.A.size()==mesh.B.size()&&mesh.A.size()==mesh.C.size());
+    A.resize(mesh.A.size());
+    B.resize(mesh.B.size());
+    C.resize(mesh.C.size());
+    vertices.reset( new PointList(3*mesh.A.size()) );
+    for (size_t i=0; i<A.size(); i++) {
+        A[i] = 3*i+0;
+        B[i] = 3*i+1;
+        C[i] = 3*i+2;
+        vertices->points[A[i]] = mesh.A[i];
+        vertices->points[B[i]] = mesh.B[i];
+        vertices->points[C[i]] = mesh.C[i];
+    }
+}
+TriMesh::~TriMesh( )
+{
+    vertices.reset();
+    A.clear();
+    B.clear();
+    C.clear();
+}
+size_t TriMesh::numberPointsVar( VariableType type ) const
+{
+    size_t N = 0;
+    if ( type==VariableType::NodeVariable )
+        N = vertices->points.size();
+    else if ( type==VariableType::SurfaceVariable ||  type==VariableType::VolumeVariable )
+        N = A.size();
+    return N;
+}
+std::pair<size_t,void*> TriMesh::pack( int level ) const
+{
+    std::pair<size_t,void*> data_out(0,NULL);
+    if ( level==0 ) {
+        const std::vector<Point>& points = vertices->points;
+        data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int);
+        double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)];
+        data_out.second = data_ptr;
+        uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_ptr);
+        data_int64[0] = level;
+        data_int64[1] = points.size();
+        data_int64[2] = A.size();
+        double *data = &data_ptr[3];
+        for (size_t i=0; i<points.size(); i++) {
+            data[3*i+0] = points[i].x;
+            data[3*i+1] = points[i].y;
+            data[3*i+2] = points[i].z;
+        }
+        int *data_int = reinterpret_cast<int*>(&data[3*points.size()]);
+        for (size_t i=0; i<A.size(); i++) {
+            data_int[3*i+0] = A[i];
+            data_int[3*i+1] = B[i];
+            data_int[3*i+2] = C[i];
+        }
+    }
+    return data_out;
+}
+void TriMesh::unpack( const std::pair<size_t,void*>& data_in )
+{
+    uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_in.second);
+    const double *data = reinterpret_cast<const double*>(data_in.second);
+    int level = data_int64[0];
+    uint64_t N_P = data_int64[1];
+    uint64_t N_A = data_int64[2];
+    data = &data[3];
+    if ( level==0 ) {
+        size_t size = (3+3*N_P)*sizeof(double)+3*N_A*sizeof(int);
+        ASSERT(size==data_in.first);
+        vertices.reset( new PointList(N_P) );
+        std::vector<Point>& points = vertices->points;
+        for (size_t i=0; i<points.size(); i++) {
+            points[i].x = data[3*i+0];
+            points[i].y = data[3*i+1];
+            points[i].z = data[3*i+2];
+        }
+        const int *data_int = reinterpret_cast<const int*>(&data[3*N_P]);
+        A.resize(N_A);
+        B.resize(N_A);
+        C.resize(N_A);
+        for (size_t i=0; i<A.size(); i++) {
+            A[i] = data_int[3*i+0];
+            B[i] = data_int[3*i+1];
+            C[i] = data_int[3*i+2];
+        }
+    }
+}
+
+
+/****************************************************
+* Domain mesh                                       *
+****************************************************/
+DomainMesh::DomainMesh():
+    nprocx(0), nprocy(0), nprocz(0), rank(0),
+    nx(0), ny(0), nz(0),
+    Lx(0), Ly(0), Lz(0)
+{
+}
+DomainMesh::DomainMesh( RankInfoStruct data, 
+    int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ):
+    nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]),
+    nx(nx2), ny(ny2), nz(nz2),
+    Lx(Lx2), Ly(Ly2), Lz(Lz2)
+{
+}
+DomainMesh::~DomainMesh()
+{
+}
+size_t DomainMesh::numberPointsVar( VariableType type ) const
+{
+    size_t N = 0;
+    if ( type==VariableType::NodeVariable )
+        N = (nx+1)*(ny+1)*(nz+1);
+    else if ( type==VariableType::SurfaceVariable )
+        N = (nx+1)*ny*nz + nx*(ny+1)*nz + nx*ny*(nz+1);
+    else if ( type==VariableType::VolumeVariable )
+        N = nx*ny*nz;
+    return N;
+}
+std::pair<size_t,void*> DomainMesh::pack( int level ) const
+{
+    std::pair<size_t,void*> data(0,NULL);
+    data.first = 7*sizeof(double);
+    data.second = new double[7];
+    memset(data.second,0,7*sizeof(double));
+    int *data_int = reinterpret_cast<int*>(data.second);
+    double *data_double = &reinterpret_cast<double*>(data.second)[4];
+    data_int[0] = nprocx;
+    data_int[1] = nprocy;
+    data_int[2] = nprocz;
+    data_int[3] = rank;
+    data_int[4] = nx;
+    data_int[5] = ny;
+    data_int[6] = nz;
+    data_double[0] = Lx;
+    data_double[1] = Ly;
+    data_double[2] = Lz;
+    return data;
+}
+void DomainMesh::unpack( const std::pair<size_t,void*>& data )
+{
+    const int *data_int = reinterpret_cast<const int*>(data.second);
+    const double *data_double = &reinterpret_cast<const double*>(data.second)[4];
+    nprocx = data_int[0];
+    nprocy = data_int[1];
+    nprocz = data_int[2];
+    rank = data_int[3];
+    nx = data_int[4];
+    ny = data_int[5];
+    nz = data_int[6];
+    Lx = data_double[0];
+    Ly = data_double[1];
+    Lz = data_double[2];
+}
+
+
+/****************************************************
+* Converters                                        *
+****************************************************/
+std::shared_ptr<PointList> getPointList( std::shared_ptr<Mesh> mesh )
+{
+    return std::dynamic_pointer_cast<PointList>(mesh);
+}
+std::shared_ptr<TriMesh> getTriMesh( std::shared_ptr<Mesh> mesh )
+{
+    std::shared_ptr<TriMesh> mesh2;
+    if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
+        mesh2 = std::dynamic_pointer_cast<TriMesh>(mesh);
+    } else if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
+        std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>(mesh);
+        ASSERT(trilist.get()!=NULL);
+        mesh2.reset( new TriMesh(*trilist) );
+    }
+    return mesh2;
+}
+std::shared_ptr<TriList> getTriList( std::shared_ptr<Mesh> mesh )
+{
+    std::shared_ptr<TriList> mesh2;
+    if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
+        mesh2 = std::dynamic_pointer_cast<TriList>(mesh);
+    } else if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
+        std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>(mesh);
+        ASSERT(trimesh.get()!=NULL);
+        mesh2.reset( new TriList(*trimesh) );
+    }
+    return mesh2;
+}
+std::shared_ptr<const PointList> getPointList( std::shared_ptr<const Mesh> mesh )
+{
+    return getPointList( std::const_pointer_cast<Mesh>(mesh) );
+}
+std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh )
+{
+    return getTriMesh( std::const_pointer_cast<Mesh>(mesh) );
+}
+std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh )
+{
+    return getTriList( std::const_pointer_cast<Mesh>(mesh) );
+}
+
+
+} // IO namespace
+

IO/Mesh.h

@@ -0,0 +1,221 @@
+#ifndef MESH_INC
+#define MESH_INC
+
+#include <iostream>
+#include <string.h>
+#include <vector>
+
+#include "common/Array.h"
+#include "common/Communication.h"
+#include "analysis/PointList.h"
+#include "shared_ptr.h"
+
+
+
+namespace IO {
+
+
+//! Possible variable types
+enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
+enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 };
+
+
+/*! \class Mesh
+    \brief A base class for meshes
+*/
+class Mesh
+{
+public:
+    //! Destructor
+    virtual ~Mesh();
+    //! Mesh class name (eg. PointList)
+    virtual std::string className() const = 0;
+    //! Number of points for the given variable type
+    virtual size_t numberPointsVar( VariableType type ) const = 0;
+    //! Pack the data
+    virtual std::pair<size_t,void*> pack( int level ) const = 0;
+    //! Unpack the data
+    virtual void unpack( const std::pair<size_t,void*>& data ) = 0;
+protected:
+    //! Empty constructor
+    Mesh();
+    Mesh(const Mesh&);
+    Mesh& operator=(const Mesh&);
+};
+
+
+/*! \class PointList
+    \brief A class used to hold a list of verticies
+*/
+class PointList: public Mesh
+{
+public:
+    //! Empty constructor
+    PointList();
+    //! Constructor for N points
+    PointList( size_t N );
+    //! Destructor
+    virtual ~PointList();
+    //! Mesh class name
+    virtual std::string className() const { return "PointList"; }
+    //! Number of points for the given variable type
+    virtual size_t numberPointsVar( VariableType type ) const;
+    //! Pack the data
+    virtual std::pair<size_t,void*> pack( int level ) const;
+    //! Unpack the data
+    virtual void unpack( const std::pair<size_t,void*>& data );
+    //! Access the points
+    const std::vector<Point>& getPoints() const { return points; }
+public:
+    std::vector<Point>  points;  //!< List of points vertex
+};
+
+
+/*! \class TriList
+    \brief A class used to hold a list of triangles specified by their vertex coordinates
+*/
+class TriMesh;
+class TriList: public Mesh
+{
+public:
+    //! Empty constructor
+    TriList();
+    //! Constructor for N triangles
+    TriList( size_t N_tri );
+    //! Constructor from TriMesh
+    TriList( const TriMesh& );
+    //! Destructor
+    virtual ~TriList();
+    //! Mesh class name
+    virtual std::string className() const { return "TriList"; }
+    //! Number of points for the given variable type
+    virtual size_t numberPointsVar( VariableType type ) const;
+    //! Pack the data
+    virtual std::pair<size_t,void*> pack( int level ) const;
+    //! Unpack the data
+    virtual void unpack( const std::pair<size_t,void*>& data );
+public:
+    std::vector<Point>  A;      //!< First vertex
+    std::vector<Point>  B;      //!< Second vertex
+    std::vector<Point>  C;      //!< Third vertex
+};
+
+
+/*! \class TriMesh
+    \brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes
+*/
+class TriMesh: public Mesh
+{
+public:
+    //! TriMesh constructor
+    TriMesh();
+    //! Constructor for Nt triangles and Np points
+    TriMesh( size_t N_tri, size_t N_point );
+    //! Constructor for Nt triangles and the given points
+    TriMesh( size_t N_tri, std::shared_ptr<PointList> points );
+    //! Constructor from TriList
+    TriMesh( const TriList& );
+    //! Destructor
+    virtual ~TriMesh();
+    //! Mesh class name
+    virtual std::string className() const { return "TriMesh"; }
+    //! Number of points for the given variable type
+    virtual size_t numberPointsVar( VariableType type ) const;
+    //! Pack the data
+    virtual std::pair<size_t,void*> pack( int level ) const;
+    //! Unpack the data
+    virtual void unpack( const std::pair<size_t,void*>& data );
+public:
+    std::shared_ptr<PointList> vertices;    //!< List of verticies
+    std::vector<int>    A;                  //!< First vertex
+    std::vector<int>    B;                  //!< Second vertex
+    std::vector<int>    C;                  //!< Third vertex
+};
+
+
+/*! \class Domain
+    \brief A class used to hold the domain
+*/
+class DomainMesh: public Mesh
+{
+public:
+    //! Empty constructor
+    DomainMesh();
+    //! Default constructor
+    DomainMesh( RankInfoStruct rank_data, int nx, int ny, int nz, double Lx, double Ly, double Lz );
+    //! Destructor
+    virtual ~DomainMesh();
+    //! Mesh class name
+    virtual std::string className() const { return "DomainMesh"; }
+    //! Number of points for the given variable type
+    virtual size_t numberPointsVar( VariableType type ) const;
+    //! Pack the data
+    virtual std::pair<size_t,void*> pack( int level ) const;
+    //! Unpack the data
+    virtual void unpack( const std::pair<size_t,void*>& data );
+public:
+    int nprocx, nprocy, nprocz, rank;
+    int nx, ny, nz;
+    double Lx, Ly, Lz;
+};
+
+
+
+/*! \class Variable
+    \brief A base class for variables
+*/
+struct Variable
+{
+public:
+    // Internal variables
+    unsigned char dim;          //!< Number of points per grid point (1: scalar, 3: vector, ...)
+    VariableType type;          //!< Variable type
+    DataType precision;         //!< Variable precision to use for IO
+    std::string name;           //!< Variable name
+    Array<double> data;         //!< Variable data
+    //! Empty constructor
+    Variable(): dim(0), type(VariableType::NullVariable), precision(DataType::Double) {}
+    //! Constructor
+    Variable( int dim_, IO::VariableType type_, const std::string& name_ ):
+        dim(dim_), type(type_), precision(DataType::Double), name(name_) {}
+    //! Constructor
+    Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array<double>& data_ ):
+        dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {}
+    //! Destructor
+    virtual ~Variable() {}
+protected:
+    //! Empty constructor
+    Variable(const Variable&);
+    Variable& operator=(const Variable&);
+};
+
+
+
+/*! \class MeshDataStruct
+    \brief A class used to hold database info for saving a mesh
+*/
+struct MeshDataStruct {
+    DataType precision;         //!< Precision to use for IO (mesh)
+    std::string meshName;       //!< Mesh name
+    std::shared_ptr<Mesh> mesh; //!< Mesh data
+    std::vector<std::shared_ptr<Variable> >  vars;
+    //! Empty constructor
+    MeshDataStruct(): precision(DataType::Double) {}
+    //! Check the data
+    bool check() const;
+};
+
+
+//! Convert the mesh to a TriMesh (will return NULL if this is invalid)
+std::shared_ptr<PointList> getPointList( std::shared_ptr<Mesh> mesh );
+std::shared_ptr<TriMesh> getTriMesh( std::shared_ptr<Mesh> mesh );
+std::shared_ptr<TriList> getTriList( std::shared_ptr<Mesh> mesh );
+std::shared_ptr<const PointList> getPointList( std::shared_ptr<const Mesh> mesh );
+std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh );
+std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh );
+
+
+} // IO namespace
+
+#endif
+

IO/MeshDatabase.cpp

@@ -0,0 +1,434 @@
+#include "IO/MeshDatabase.h"
+#include "IO/Mesh.h"
+#include "IO/IOHelpers.h"
+#include "common/MPI_Helpers.h"
+#include "common/Utilities.h"
+
+#include <vector>
+#include <map>
+#include <set>
+#include <cstdio>
+
+#include <ProfilerApp.h>
+
+
+
+/****************************************************
+****************************************************/
+// MeshType
+template<>
+size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
+{
+    return sizeof(IO::MeshType);
+}
+template<>
+void pack<IO::MeshType>( const IO::MeshType& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(IO::MeshType));
+}
+template<>
+void unpack<IO::MeshType>( IO::MeshType& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(IO::MeshType));
+}
+// Variable::VariableType
+template<>
+size_t packsize<IO::VariableType>( const IO::VariableType& rhs )
+{
+    return sizeof(IO::VariableType);
+}
+template<>
+void pack<IO::VariableType>( const IO::VariableType& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(IO::VariableType));
+}
+template<>
+void unpack<IO::VariableType>( IO::VariableType& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(IO::VariableType));
+}
+// DatabaseEntry
+template<>
+size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs )
+{
+    return packsize(rhs.name)+packsize(rhs.file)+packsize(rhs.offset);
+}
+template<>
+void pack<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs, char *buffer )
+{
+    size_t i=0;
+    pack(rhs.name,&buffer[i]);      i+=packsize(rhs.name);
+    pack(rhs.file,&buffer[i]);      i+=packsize(rhs.file);
+    pack(rhs.offset,&buffer[i]);    i+=packsize(rhs.offset);
+}
+template<>
+void unpack<IO::DatabaseEntry>( IO::DatabaseEntry& data, const char *buffer )
+{
+    size_t i=0;
+    unpack(data.name,&buffer[i]);   i+=packsize(data.name);
+    unpack(data.file,&buffer[i]);   i+=packsize(data.file);
+    unpack(data.offset,&buffer[i]); i+=packsize(data.offset);
+}
+// VariableDatabase
+template<>
+size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase& rhs )
+{
+    return packsize(rhs.name)+packsize(rhs.type)+packsize(rhs.dim);
+}
+template<>
+void pack<IO::VariableDatabase>( const IO::VariableDatabase& rhs, char *buffer )
+{
+    size_t i=0;
+    pack(rhs.name,&buffer[i]);      i+=packsize(rhs.name);
+    pack(rhs.type,&buffer[i]);      i+=packsize(rhs.type);
+    pack(rhs.dim,&buffer[i]);       i+=packsize(rhs.dim);
+}
+template<>
+void unpack<IO::VariableDatabase>( IO::VariableDatabase& data, const char *buffer )
+{
+    size_t i=0;
+    unpack(data.name,&buffer[i]);   i+=packsize(data.name);
+    unpack(data.type,&buffer[i]);   i+=packsize(data.type);
+    unpack(data.dim,&buffer[i]);    i+=packsize(data.dim);
+}
+// MeshDatabase
+template<>
+size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase& data )
+{
+    return packsize(data.name) 
+         + packsize(data.type)
+         + packsize(data.meshClass)
+         + packsize(data.format)
+         + packsize(data.domains)
+         + packsize(data.variables)
+         + packsize(data.variable_data);
+}
+template<>
+void pack<IO::MeshDatabase>( const IO::MeshDatabase& rhs, char *buffer )
+{
+    size_t i = 0;
+    pack(rhs.name,&buffer[i]);          i+=packsize(rhs.name);
+    pack(rhs.type,&buffer[i]);          i+=packsize(rhs.type);
+    pack(rhs.meshClass,&buffer[i]);     i+=packsize(rhs.meshClass);
+    pack(rhs.format,&buffer[i]);        i+=packsize(rhs.format);
+    pack(rhs.domains,&buffer[i]);       i+=packsize(rhs.domains);
+    pack(rhs.variables,&buffer[i]);     i+=packsize(rhs.variables);
+    pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data);
+}
+template<>
+void unpack<IO::MeshDatabase>( IO::MeshDatabase& data, const char *buffer )
+{
+    size_t i=0;
+    unpack(data.name,&buffer[i]);       i+=packsize(data.name);
+    unpack(data.type,&buffer[i]);       i+=packsize(data.type);
+    unpack(data.meshClass,&buffer[i]);  i+=packsize(data.meshClass);
+    unpack(data.format,&buffer[i]);     i+=packsize(data.format);
+    unpack(data.domains,&buffer[i]);    i+=packsize(data.domains);
+    unpack(data.variables,&buffer[i]);  i+=packsize(data.variables);
+    unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data);
+}
+
+
+namespace IO {
+
+
+/****************************************************
+* VariableDatabase                                  *
+****************************************************/
+bool VariableDatabase::operator==(const VariableDatabase& rhs ) const
+{
+    return type==rhs.type && dim==rhs.dim && name==rhs.name;
+}
+bool VariableDatabase::operator!=(const VariableDatabase& rhs ) const
+{
+    return type!=rhs.type || dim!=rhs.dim || name!=rhs.name;
+}
+bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const
+{
+    return operator>(rhs) || operator==(rhs);
+}
+bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const
+{
+    return !operator>(rhs);
+}
+bool VariableDatabase::operator>(const VariableDatabase& rhs ) const
+{
+    if ( name>rhs.name )
+        return true;
+    else if ( name<rhs.name )
+        return false;
+    if ( type>rhs.type )
+        return true;
+    else if ( type<rhs.type )
+        return false;
+    if ( dim>rhs.dim )
+        return true;
+    else if ( dim<rhs.dim )
+        return false;
+    return false;
+}
+bool VariableDatabase::operator<(const VariableDatabase& rhs ) const
+{
+    return !operator>(rhs) && operator!=(rhs);
+}
+
+
+/****************************************************
+* MeshDatabase                                      *
+****************************************************/
+MeshDatabase::MeshDatabase()
+{
+}
+MeshDatabase::~MeshDatabase()
+{
+}
+MeshDatabase::MeshDatabase(const MeshDatabase& rhs)
+{
+    name = rhs.name;
+    type = rhs.type;
+    meshClass = rhs.meshClass;
+    format = rhs.format;
+    domains = rhs.domains;
+    variables = rhs.variables;
+    variable_data = rhs.variable_data;
+}
+MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs)
+{
+    this->name = rhs.name;
+    this->type = rhs.type;
+    this->meshClass = rhs.meshClass;
+    this->format = rhs.format;
+    this->domains = rhs.domains;
+    this->variables = rhs.variables;
+    this->variable_data = rhs.variable_data;
+    return *this;
+}
+VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname ) const
+{
+    for (size_t i=0; i<variables.size(); i++) {
+        if ( variables[i].name == varname )
+            return variables[i];
+    }
+    return VariableDatabase();
+}
+
+
+/****************************************************
+* DatabaseEntry                                     *
+****************************************************/
+std::string DatabaseEntry::write( ) const
+{
+    char tmp[1000];
+    sprintf(tmp,"%s; %s; %lu",name.c_str(),file.c_str(),offset);
+    return std::string(tmp);
+}
+DatabaseEntry::DatabaseEntry( const char* line )
+{
+    std::vector<std::string> list = splitList(line,';');
+    name = list[0];
+    file = list[1];
+    offset = atol(list[2].c_str());
+}
+void DatabaseEntry::read( const char* line )
+{
+    std::vector<std::string> list = splitList(line,';');
+    name = list[0];
+    file = list[1];
+    offset = atol(list[2].c_str());
+}
+void DatabaseEntry::read( const std::string& line )
+{
+    std::vector<std::string> list = splitList(line.c_str(),';');
+    name = list[0];
+    file = list[1];
+    offset = atol(list[2].c_str());
+}
+
+
+// Gather the mesh databases from all processors
+inline int tod( int N ) { return (N+7)/sizeof(double); }
+std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm )
+{
+    #ifdef USE_MPI
+        PROFILE_START("gatherAll");
+        PROFILE_START("gatherAll-pack",2);
+        int size = MPI_WORLD_SIZE();
+        // First pack the mesh data to local buffers
+        int localsize = 0;
+        for (size_t i=0; i<meshes.size(); i++)
+            localsize += tod(packsize(meshes[i]));
+        auto localbuf = new double[localsize];
+        int pos = 0;
+        for (size_t i=0; i<meshes.size(); i++) {
+            pack( meshes[i], (char*) &localbuf[pos] );
+            pos += tod(packsize(meshes[i]));
+        }
+        PROFILE_STOP("gatherAll-pack",2);
+        // Get the number of bytes each processor will be sending/recieving
+        PROFILE_START("gatherAll-send1",2);
+        auto recvsize = new int[size];
+        MPI_Allgather(&localsize,1,MPI_INT,recvsize,1,MPI_INT,comm);
+        int globalsize = recvsize[0];
+        auto disp = new int[size];
+        disp[0] = 0;
+        for (int i=1; i<size; i++) {
+            disp[i] = disp[i-1] + recvsize[i];
+            globalsize += recvsize[i];
+        }
+        PROFILE_STOP("gatherAll-send1",2);
+        // Send/recv the global data
+        PROFILE_START("gatherAll-send2",2);
+        auto globalbuf = new double[globalsize];
+        MPI_Allgatherv(localbuf,localsize,MPI_DOUBLE,globalbuf,recvsize,disp,MPI_DOUBLE,comm);
+        PROFILE_STOP("gatherAll-send2",2);
+        // Unpack the data
+        PROFILE_START("gatherAll-unpack",2);
+        std::map<std::string,MeshDatabase> data;
+        pos = 0;
+        while ( pos < globalsize ) {
+            MeshDatabase tmp;
+            unpack(tmp,(char*)&globalbuf[pos]);
+            pos += tod(packsize(tmp));
+            std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
+            if ( it==data.end() ) {
+                data[tmp.name] = tmp;
+            } else {
+                for (size_t i=0; i<tmp.domains.size(); i++)
+                    it->second.domains.push_back(tmp.domains[i]);
+                for (size_t i=0; i<tmp.variables.size(); i++)
+                    it->second.variables.push_back(tmp.variables[i]);
+                it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
+            }
+        }
+        for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it) {
+            // Get the unique variables
+            std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
+            it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
+        }
+        // Free temporary memory
+        delete [] localbuf;
+        delete [] recvsize;
+        delete [] disp;
+        delete [] globalbuf;
+        // Return the results
+        std::vector<MeshDatabase> data2(data.size());
+        size_t i=0; 
+        for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
+            data2[i] = it->second;
+        PROFILE_STOP("gatherAll-unpack",2);
+        PROFILE_STOP("gatherAll");
+        return data2;
+    #else
+        return meshes;
+    #endif
+}
+
+
+//! Write the mesh databases to a file
+void write( const std::vector<MeshDatabase>& meshes, const std::string& filename )
+{
+    PROFILE_START("write");
+    FILE *fid = fopen(filename.c_str(),"wb");
+    for (size_t i=0; i<meshes.size(); i++) {
+        fprintf(fid,"%s\n",meshes[i].name.c_str());
+        fprintf(fid,"   type: %i\n",static_cast<int>(meshes[i].type));
+        fprintf(fid,"   meshClass: %s\n",meshes[i].meshClass.c_str());
+        fprintf(fid,"   format: %i\n",static_cast<int>(meshes[i].format));
+        for (size_t j=0; j<meshes[i].domains.size(); j++)
+            fprintf(fid,"   domain: %s\n",meshes[i].domains[j].write().c_str());
+        fprintf(fid,"   variables: ");
+        for (size_t j=0; j<meshes[i].variables.size(); j++) {
+            const VariableDatabase& var = meshes[i].variables[j];
+            fprintf(fid,"%s|%i|%i; ",var.name.c_str(),static_cast<int>(var.type),var.dim);
+        }
+        fprintf(fid,"\n");
+        std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
+        for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) {
+            const char* domain = it->first.first.c_str();
+            const char* variable = it->first.second.c_str();
+            fprintf(fid,"   variable(%s,%s): %s\n",domain,variable,it->second.write().c_str());
+        }
+    }
+    fclose(fid);
+    PROFILE_STOP("write");
+}
+
+
+//! Read the mesh databases from a file
+std::vector<MeshDatabase> read( const std::string& filename )
+{
+    std::vector<MeshDatabase> meshes;
+    PROFILE_START("read");
+    FILE *fid = fopen(filename.c_str(),"rb");
+    if ( fid==NULL )
+        ERROR("Error opening file");
+    char *line = new char[10000];
+    while ( std::fgets(line,1000,fid) != NULL ) {
+        if ( line[0]<32 ) {
+            // Empty line
+            continue;
+        } else if ( line[0] != ' ' ) {
+            meshes.resize(meshes.size()+1);
+            std::string name(line);
+            name.resize(name.size()-1);
+            meshes.back().name = name;
+        } else if ( strncmp(line,"   format:",10)==0 ) {
+            meshes.back().format = static_cast<unsigned char>(atoi(&line[10]));
+        } else if ( strncmp(line,"   type:",8)==0 ) {
+            meshes.back().type = static_cast<MeshType>(atoi(&line[8]));
+        } else if ( strncmp(line,"   meshClass:",13)==0 ) {
+            meshes.back().meshClass = deblank(std::string(&line[13]));
+        } else if ( strncmp(line,"   domain:",10)==0 ) {
+            DatabaseEntry data(&line[10]);
+            meshes.back().domains.push_back(data);
+        } else if ( strncmp(line,"   variables:",13)==0 ) {
+            MeshDatabase& mesh = meshes.back();
+            std::vector<std::string> variables = splitList(&line[13],';');
+            mesh.variables.resize(variables.size());
+            for (size_t i=0; i<variables.size(); i++) {
+                std::vector<std::string> tmp = splitList(variables[i].c_str(),'|');
+                ASSERT(tmp.size()==3);
+                mesh.variables[i].name = tmp[0];
+                mesh.variables[i].type = static_cast<VariableType>(atoi(tmp[1].c_str()));
+                mesh.variables[i].dim = atoi(tmp[2].c_str());
+            }
+        } else if ( strncmp(line,"   variable(",12)==0 ) {
+            size_t i1 = find(line,',');
+            size_t i2 = find(line,':');
+            std::string domain = deblank(std::string(line,12,i1-12));
+            std::string variable = deblank(std::string(line,i1+1,i2-i1-2));
+            std::pair<std::string,std::string> key(domain,variable);
+            DatabaseEntry data(&line[i2+1]);
+            meshes.back().variable_data.insert( 
+                std::pair<std::pair<std::string,std::string>,DatabaseEntry>(key,data) );
+        } else {
+            ERROR("Error reading line");
+        }
+    }
+    fclose(fid);
+    delete [] line;
+    PROFILE_STOP("read");
+    return meshes;
+}
+
+
+// Return the mesh type
+IO::MeshType meshType( const IO::Mesh& mesh )
+{
+    IO::MeshType type = IO::Unknown;
+    const std::string meshClass = mesh.className();
+    if ( meshClass=="PointList" ) {
+        type = IO::PointMesh;
+    } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
+        type = IO::SurfaceMesh;
+    } else if ( meshClass=="DomainMesh" ) {
+        type = IO::VolumeMesh;
+    } else {
+        ERROR("Unknown mesh");
+    }
+    return type;
+}
+
+
+} // IO namespace
+

IO/MeshDatabase.h

@@ -0,0 +1,90 @@
+#ifndef MeshDatabase_INC
+#define MeshDatabase_INC
+
+#include "IO/Mesh.h" 
+#include "common/MPI_Helpers.h"
+#include "shared_ptr.h"
+
+#include <iostream>
+#include <string.h>
+#include <vector>
+#include <map>
+
+
+namespace IO {
+
+class Mesh;
+
+
+//! Enum to identify mesh type
+//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
+enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
+
+
+//! Helper struct for containing offsets for the mesh info
+struct DatabaseEntry {
+    std::string name;                   //!< Name of the entry
+    std::string file;                   //!< Name of the file containing the entry
+    size_t offset;                      //!< Offset in the file to start reading
+    std::string write( ) const;         //!< Convert the data to a string
+    void read( const char* line );      //!< Convert the string to data
+    void read( const std::string& line ); //!< Convert the string to data
+    DatabaseEntry( ) {}                 //!< Empty constructor
+    DatabaseEntry( const char* line );  //!< Convert the string to data
+    ~DatabaseEntry() {}                 //!< Destructor
+};
+
+
+//! Structure to hold the info about the variables
+struct VariableDatabase {
+    std::string name;                   //!< Name of the variable
+    IO::VariableType type;              //!< Variable
+    unsigned int dim;                   //!< Number of points per grid point (1: scalar, 3: vector, ...)
+    // Overload key operators
+    bool operator==(const VariableDatabase& rhs ) const;
+    bool operator!=(const VariableDatabase& rhs ) const;
+    bool operator>=(const VariableDatabase& rhs ) const;
+    bool operator<=(const VariableDatabase& rhs ) const;
+    bool operator> (const VariableDatabase& rhs ) const;
+    bool operator< (const VariableDatabase& rhs ) const;
+};
+
+
+//! Structure to hold the info about the meshes
+struct MeshDatabase {
+    typedef  std::pair<std::string,std::string>  variable_id;
+    std::string name;                   //!< Name of the mesh
+    MeshType type;                      //!< Mesh type
+    std::string meshClass;              //!< Mesh class
+    unsigned char format;               //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
+    std::vector<DatabaseEntry> domains; //!< List of the domains
+    std::vector<VariableDatabase> variables; //!< List of the variables
+    std::map<variable_id,DatabaseEntry> variable_data; //!< Data for the variables
+    VariableDatabase getVariableDatabase( const std::string& varname ) const;
+public:
+    MeshDatabase();
+    ~MeshDatabase();
+    MeshDatabase(const MeshDatabase&);
+    MeshDatabase& operator=(const MeshDatabase&);
+};
+
+
+//! Gather the mesh databases from all processors
+std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm );
+
+
+//! Write the mesh databases to a file
+void write( const std::vector<MeshDatabase>& meshes, const std::string& filename );
+
+
+//! Read the mesh databases from a file
+std::vector<MeshDatabase> read( const std::string& filename );
+
+
+//! Return the mesh type
+IO::MeshType meshType( const IO::Mesh& mesh );
+
+
+} // IO namespace
+
+#endif

IO/PIO.cpp

@@ -0,0 +1,169 @@
+#include "IO/PIO.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+
+#include <fstream>
+#include <string>
+#include <cstring>
+
+
+namespace IO {
+
+
+static ParallelStreamBuffer pout_buffer;
+static ParallelStreamBuffer perr_buffer;
+static ParallelStreamBuffer plog_buffer;
+
+
+std::ostream pout(&pout_buffer);
+std::ostream perr(&perr_buffer);
+std::ostream plog(&plog_buffer);
+
+
+
+/****************************************************************************
+*  Functions to control logging                                             *
+****************************************************************************/
+std::ofstream *global_filestream=NULL;
+static void shutdownFilestream( )
+{
+    if ( global_filestream!=NULL ) {
+        global_filestream->flush();
+        global_filestream->close();
+        delete global_filestream;
+        global_filestream = NULL;
+    }
+}
+void Utilities::logOnlyNodeZero( const std::string &filename )
+{
+    int rank = 0;
+    #ifdef USE_MPI
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+    #endif
+    if ( rank == 0 )
+        logAllNodes(filename,true);
+}
+void Utilities::logAllNodes( const std::string &filename, bool singleStream )
+{
+    if ( singleStream )
+        ERROR("Not implimented yet");
+
+    // If the filestream was open, then close it and reset streams
+    shutdownFilestream();
+
+    // Open the log stream and redirect output
+    std::string full_filename = filename;
+    if ( !singleStream ) {
+        int rank = 0;
+        #ifdef USE_MPI
+            MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+        #endif
+        char tmp[100];
+        sprintf(tmp,".%04i",rank);
+        full_filename += std::string(tmp);
+    }
+    global_filestream = new std::ofstream(full_filename.c_str());
+
+    if ( !(*global_filestream) ) {
+        delete global_filestream;
+        global_filestream = NULL;
+        perr << "PIO: Could not open log file ``" << full_filename << "''\n";
+    } else {
+        pout_buffer.setOutputStream(global_filestream);
+        pout_buffer.setOutputStream(&std::cout);
+        perr_buffer.setOutputStream(global_filestream);
+        perr_buffer.setOutputStream(&std::cerr);
+        plog_buffer.setOutputStream(global_filestream);
+    }
+}
+
+
+/****************************************************************************
+*  ParallelStreamBuffer class                                               *
+****************************************************************************/
+void Utilities::stopLogging( )
+{
+    pout_buffer.reset();
+    perr_buffer.reset();
+    plog_buffer.reset();
+    shutdownFilestream();
+    delete global_filestream;
+    global_filestream = NULL;
+}
+
+
+/****************************************************************************
+*  ParallelStreamBuffer class                                               *
+****************************************************************************/
+ParallelStreamBuffer::ParallelStreamBuffer( ):
+    d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL)
+{
+}
+ParallelStreamBuffer:: ~ParallelStreamBuffer()
+{
+    delete [] d_buffer;
+}
+void ParallelStreamBuffer::setOutputStream( std::ostream *stream )
+{
+    d_stream.push_back( stream );
+}
+int ParallelStreamBuffer::sync()
+{
+    for (size_t i=0; i<d_stream.size(); i++) {
+        std::ostream& stream = *d_stream[i];
+        stream << d_buffer;
+    }
+    d_size = 0;
+    memset(d_buffer,0,d_buffer_size);
+    return 0;
+}
+void ParallelStreamBuffer::reserve( size_t size )
+{
+    if ( size > d_buffer_size ) {
+        if ( d_buffer_size==0 ) {
+            d_buffer_size = 1024;
+            d_buffer = new char[d_buffer_size];
+            memset(d_buffer,0,d_buffer_size);
+        }
+        while ( size > d_buffer_size ) {
+            char *tmp = d_buffer;
+            d_buffer_size *= 2;
+            d_buffer = new char[d_buffer_size];
+            memset(d_buffer,0,d_buffer_size);
+            memcpy(d_buffer,tmp,d_size);
+            delete [] tmp;
+        }
+    }
+}
+std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize n )
+{
+    reserve(d_size+n);
+    memcpy(&d_buffer[d_size],text,n);
+    d_size += n;
+    if ( text[n-1]==0 || text[n-1]==10 ) { sync(); }
+    return n;
+}
+int ParallelStreamBuffer::overflow(int ch)
+{
+    reserve(d_size+1);
+    d_buffer[d_size] = ch;
+    d_size++;
+    if ( ch==0 || ch==10 ) { sync(); }
+    return std::char_traits<char>::to_int_type(ch);
+}
+int ParallelStreamBuffer::underflow() 
+{
+    return -1;
+}
+void ParallelStreamBuffer::reset() 
+{
+    sync();
+    d_stream.clear();
+    delete [] d_buffer;
+    d_buffer = NULL;
+    d_buffer_size = 0;
+}
+
+
+} // IO namespace
+

IO/PIO.h

@@ -0,0 +1,135 @@
+#ifndef included_PIO
+#define included_PIO
+
+#include <iostream>
+#include <vector>
+
+
+namespace IO {
+
+
+/*!
+ * Parallel output stream pout writes to the standard output from node zero
+ * only.  Output from other nodes is ignored.  If logging is enabled, then
+ * output is mirrored to the log stream, as well.
+ */
+extern std::ostream pout;
+
+/*!
+ * Parallel output stream perr writes to the standard error from all nodes.
+ * Output is prepended with the processor number.  
+ */
+extern std::ostream perr;
+
+/*!
+ * Parallel output stream plog writes output to the log file.  When logging
+ * from multiple processors, the processor number is appended to the filename.
+ */
+extern std::ostream plog;
+
+/*!
+ * Parallel output printp pout writes to the standard output from node zero
+ * only.  Output from other nodes is ignored.  If logging is enabled, then
+ * output is mirrored to the log stream, as well.
+ * The format matches the format for printf
+ */
+inline int printp( const char *format, ... );
+
+
+/*!
+ * Class ParallelBuffer is a simple I/O stream utility that
+ * intercepts output from an ostream and redirects the output as necessary
+ * for parallel I/O.  This class defines a stream buffer class for an
+ * ostream class.
+ */
+class ParallelStreamBuffer : public std::streambuf
+{
+public:
+
+    /*!
+     * Create a parallel buffer class.  The object will require further
+     * initialization to set up the I/O streams and prefix string.
+     */
+    ParallelStreamBuffer( );
+
+    /*!
+     * Set the output file stream (multiple output streams are supported)
+     * @param stream    Output stream
+     */
+    void setOutputStream( std::ostream *stream );
+
+    /*!
+     * The destructor simply deallocates any internal data
+    * buffers.  It does not modify the output streams.
+     */
+    virtual ~ParallelStreamBuffer();
+
+    /*!
+     * Synchronize the parallel buffer (called from streambuf).
+     */
+   virtual int sync();
+
+    /**
+     * Write the specified number of characters into the output stream (called
+     * from streambuf).
+    */
+    virtual std::streamsize xsputn(const char* text, std::streamsize n);
+
+    /*!
+     * Write an overflow character into the parallel buffer (called from
+     * streambuf).
+     */
+    virtual int overflow(int ch);
+
+    /*!
+     * Read an overflow character from the parallel buffer (called from
+     * streambuf).  This is not implemented.  It is needed by the
+     * MSVC++ stream implementation.
+     */
+    virtual int underflow();
+
+    /*!
+     * Clear the internal buffer's memory
+     */
+    virtual void reset();
+
+private:
+    int d_rank;
+    size_t d_size;
+    size_t d_buffer_size;
+    char *d_buffer;
+    std::vector<std::ostream*> d_stream;
+    inline void reserve( size_t size );
+};
+
+
+namespace Utilities {
+
+    /*!
+     * Log messages for node zero only to the specified filename.  All output
+     * to pout, perr, and plog on node zero will go to the log file.
+     */
+    void logOnlyNodeZero( const std::string &filename );
+
+    /*!
+     * Log messages from all nodes.  The diagnostic data for processor XXXXX
+     * will be sent to a file with the name filename.XXXXX, where filename is
+     * the function argument.
+     */
+    void logAllNodes( const std::string &filename, bool singleStream=false );
+
+    /*!
+     * Stop logging messages, flush buffers, and reset memory.
+     */
+    void stopLogging( );
+
+
+} // namespace Utilities
+
+
+} // namespace IO
+
+
+#include "IO/PIO.hpp"
+
+#endif

IO/PIO.hpp

@@ -0,0 +1,29 @@
+#ifndef included_PIO_hpp
+#define included_PIO_hpp
+
+#include "IO/PIO.h"
+
+#include <iostream>
+#include <stdarg.h>
+#include <cstdio>
+
+
+namespace IO {
+
+
+inline int printp( const char *format, ... )
+{
+    va_list ap; 
+    va_start(ap,format); 
+    char tmp[1024];
+    int n = vsprintf(tmp,format,ap);
+    va_end(ap);
+    pout << tmp;
+    pout.flush();
+    return n;
+}
+
+
+} // IO namespace
+
+#endif

IO/Reader.cpp

@@ -0,0 +1,343 @@
+#include "IO/Reader.h"
+#include "IO/Mesh.h"
+#include "IO/MeshDatabase.h"
+#include "IO/IOHelpers.h"
+#include "common/Utilities.h"
+
+#ifdef USE_SILO
+#include "IO/silo.h"
+#endif
+
+
+#include <ProfilerApp.h>
+#include <iostream>
+#include <string.h>
+#include <vector>
+#include <map>
+#include <cstdio>
+
+
+// Inline function to read line without a return argument
+static inline void fgetl( char * str, int num, FILE * stream )
+{
+    char* ptr = fgets( str, num, stream );
+    if ( 0 ) {char *temp = (char *)&ptr; temp++;}
+}
+
+
+// Get the path to a file
+std::string IO::getPath( const std::string& filename )
+{
+    std::string file(filename);
+    size_t k1 = file.rfind(47);
+    size_t k2 = file.rfind(92);
+    if ( k1==std::string::npos ) { k1=0; }
+    if ( k2==std::string::npos ) { k2=0; }
+    return file.substr(0,std::max(k1,k2));
+}
+
+
+// List the timesteps in the given directors (dumps.LBPM)
+std::vector<std::string> IO::readTimesteps( const std::string& filename )
+{
+    PROFILE_START("readTimesteps");
+    FILE *fid= fopen(filename.c_str(),"rb");
+    if ( fid==NULL )
+        ERROR("Error opening file");
+    auto pos = std::min(filename.find_last_of(47),filename.find_last_of(90));
+    std::vector<std::string> timesteps;
+    char buf[1000];
+    while (fgets(buf,sizeof(buf),fid) != NULL) {
+        std::string line(buf);
+        line.resize(line.size()-1);
+        auto pos = line.find( "summary.silo" );
+        if ( pos != std::string::npos )
+            line.resize(pos);
+        if ( line.empty() )
+            continue;
+        timesteps.push_back(line);
+    }
+    fclose(fid);
+    PROFILE_STOP("readTimesteps");
+    return timesteps;
+}
+
+
+// Read the list of variables for the given timestep
+std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const std::string& timestep )
+{
+    std::string filename = path + "/" + timestep + "/LBM.summary";
+    return IO::read( filename );
+}
+
+
+// Read the given mesh domain
+std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::string& timestep, 
+    const IO::MeshDatabase& meshDatabase, int domain )
+{
+    PROFILE_START("getMesh");
+    std::shared_ptr<IO::Mesh> mesh;
+    if ( meshDatabase.format==1 ) {
+        // Old format (binary doubles)
+        std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file;
+        FILE *fid = fopen(filename.c_str(),"rb");
+        INSIST(fid!=NULL,"Error opening file");
+        fseek( fid, 0, SEEK_END );
+        size_t bytes = ftell(fid);
+        size_t N_max = bytes/sizeof(double)+1000;
+        double *data = new double[N_max];
+        fseek(fid,0,SEEK_SET);
+        size_t count = fread(data,sizeof(double),N_max,fid);
+        fclose(fid);
+        if ( count%3 != 0 )
+            ERROR("Error reading file");
+        if ( meshDatabase.type==IO::PointMesh ) {
+            size_t N = count/3;
+            std::shared_ptr<PointList> pointlist( new PointList(N) );
+            std::vector<Point>& P = pointlist->points;
+            for (size_t i=0; i<N; i++) {
+                P[i].x = data[3*i+0];
+                P[i].y = data[3*i+1];
+                P[i].z = data[3*i+2];
+            }
+            mesh = pointlist;
+        } else if ( meshDatabase.type==IO::SurfaceMesh ) {
+            if ( count%9 != 0 )
+                ERROR("Error reading file (2)");
+            size_t N_tri = count/9;
+            std::shared_ptr<TriList> trilist( new TriList(N_tri) );
+            std::vector<Point>& A = trilist->A;
+            std::vector<Point>& B = trilist->B;
+            std::vector<Point>& C = trilist->C;
+            for (size_t i=0; i<N_tri; i++) {
+                A[i].x = data[9*i+0];
+                A[i].y = data[9*i+1];
+                A[i].z = data[9*i+2];
+                B[i].x = data[9*i+3];
+                B[i].y = data[9*i+4];
+                B[i].z = data[9*i+5];
+                C[i].x = data[9*i+6];
+                C[i].y = data[9*i+7];
+                C[i].z = data[9*i+8];
+            }
+            mesh = trilist;
+        } else if ( meshDatabase.type==IO::VolumeMesh ) {
+            // this was never supported in the old format
+            mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
+        } else {
+            ERROR("Unknown mesh type");
+        }
+        delete [] data;
+    } else if ( meshDatabase.format==2 ) {
+        const DatabaseEntry& database = meshDatabase.domains[domain];
+        std::string filename = path + "/" + timestep + "/" + database.file;
+        FILE *fid = fopen(filename.c_str(),"rb");
+        fseek(fid,database.offset,SEEK_SET);
+        char line[1000];
+        fgetl(line,1000,fid);
+        size_t i1 = find(line,':');
+        size_t i2 = find(&line[i1+1],':')+i1+1;
+        size_t bytes = atol(&line[i2+1]);
+        char *data = new char[bytes];
+        size_t count = fread(data,1,bytes,fid);
+        fclose(fid);
+        ASSERT(count==bytes);
+        if ( meshDatabase.meshClass=="PointList" ) {
+            mesh.reset( new IO::PointList() );
+        } else if ( meshDatabase.meshClass=="TriMesh" ) {
+            mesh.reset( new IO::TriMesh() );
+        } else if ( meshDatabase.meshClass=="TriList" ) {
+            mesh.reset( new IO::TriList() );
+        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+            mesh.reset( new IO::DomainMesh() );
+        } else {
+            ERROR("Unknown mesh class");
+        }
+        mesh->unpack( std::pair<size_t,void*>(bytes,data) );
+        delete [] data;
+    } else if ( meshDatabase.format==4 ) {
+        // Reading a silo file
+#ifdef USE_SILO
+        const DatabaseEntry& database = meshDatabase.domains[domain];
+        std::string filename = path + "/" + timestep + "/" + database.file;
+        int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str());
+        auto fid = silo::open( filename, silo::READ );
+        if ( meshDatabase.meshClass=="PointList" ) {
+            Array<double> coords = silo::readPointMesh<double>( fid, database.name );
+            ASSERT(coords.size(1)==3);
+            std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size(0) ) );
+            for (size_t i=0; i<coords.size(1); i++) {
+                mesh2->points[i].x = coords(i,0);
+                mesh2->points[i].y = coords(i,1);
+                mesh2->points[i].z = coords(i,2);
+            }
+            mesh = mesh2;
+        } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
+            Array<double> coords;
+            Array<int> tri;
+            silo::readTriMesh( fid, database.name, coords, tri );
+            ASSERT( tri.size(1)==3 && coords.size(1)==3 );
+            int N_tri = tri.size(0);
+            int N_point = coords.size(0);
+            std::shared_ptr<IO::TriMesh> mesh2( new IO::TriMesh( N_tri, N_point ) );
+            for (int i=0; i<N_point; i++) {
+                mesh2->vertices->points[i].x = coords(i,0);
+                mesh2->vertices->points[i].y = coords(i,1);
+                mesh2->vertices->points[i].z = coords(i,2);
+            }
+            for (int i=0; i<N_tri; i++) {
+                mesh2->A[i] = tri(i,0);
+                mesh2->B[i] = tri(i,1);
+                mesh2->C[i] = tri(i,2);
+            }
+            if ( meshDatabase.meshClass=="TriMesh" ) {
+                mesh = mesh2;
+            } else if ( meshDatabase.meshClass=="TriList" ) {
+                auto trilist = IO::getTriList( std::dynamic_pointer_cast<IO::Mesh>( mesh2 ) );
+                mesh = trilist;
+            }
+        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+            std::vector<double> range;
+            std::vector<int> N;
+            silo::readUniformMesh( fid, database.name, range, N );
+            auto rankinfo = silo::read<int>( fid, database.name+"_rankinfo" );
+            RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
+            mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) );
+        } else {
+            ERROR("Unknown mesh class");
+        }
+        silo::close( fid );
+#else
+        ERROR("Build without silo support");
+#endif
+    } else {
+        ERROR("Unknown format");
+    }
+    PROFILE_STOP("getMesh");
+    return mesh;
+}
+
+
+// Read the given variable for the given mesh domain
+std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const std::string& timestep, 
+    const MeshDatabase& meshDatabase, int domain, const std::string& variable )
+{
+    std::pair<std::string,std::string> key(meshDatabase.domains[domain].name,variable);
+    std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
+    it = meshDatabase.variable_data.find(key);
+    if ( it==meshDatabase.variable_data.end() )
+        return std::shared_ptr<IO::Variable>();
+    std::shared_ptr<IO::Variable> var;
+    if ( meshDatabase.format == 2 ) {
+        const DatabaseEntry& database = it->second;
+        std::string filename = path + "/" + timestep + "/" + database.file;
+        FILE *fid = fopen(filename.c_str(),"rb");
+        fseek(fid,database.offset,SEEK_SET);
+        char line[1000];
+        fgetl(line,1000,fid);
+        size_t i1 = find(line,':');
+        size_t i2 = find(&line[i1+1],':')+i1+1;
+        std::vector<std::string> values = splitList(&line[i2+1],',');
+        ASSERT(values.size()==5);
+        int dim = atoi(values[0].c_str());
+        int type = atoi(values[1].c_str());
+        size_t N = atol(values[2].c_str());
+        size_t bytes = atol(values[3].c_str());
+        std::string precision  = values[4];
+        var = std::shared_ptr<IO::Variable>( new IO::Variable() );
+        var->dim = dim;
+        var->type = static_cast<IO::VariableType>(type);
+        var->name = variable;
+        var->data.resize(N*dim);
+        if ( precision=="double" ) {
+            size_t count = fread(var->data.data(),sizeof(double),N*dim,fid);
+            ASSERT(count*sizeof(double)==bytes);
+        } else {
+            ERROR("Format not implimented");
+        }
+        fclose(fid);
+    } else if ( meshDatabase.format == 4 ) {
+        // Reading a silo file
+#ifdef USE_SILO
+        const auto& database = meshDatabase.domains[domain];
+        auto variableDatabase = meshDatabase.getVariableDatabase( variable );
+        std::string filename = path + "/" + timestep + "/" + database.file;
+        int rank = std::stoi(database.file.substr(0,database.file.find(".silo")).c_str());
+        auto fid = silo::open( filename, silo::READ );
+        var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
+        if ( meshDatabase.meshClass=="PointList" ) {
+            var->data = silo::readPointMeshVariable<double>( fid, variable );
+        } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
+            var->data = silo::readTriMeshVariable<double>( fid, variable );
+        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+            var->data = silo::readUniformMeshVariable<double>( fid, variable );
+        } else {
+            ERROR("Unknown mesh class");
+        }
+        silo::close( fid );
+#else
+        ERROR("Build without silo support");
+#endif
+
+    } else {
+        ERROR("Unknown format");
+    }
+    return var;
+}
+
+
+/****************************************************
+* Reformat the variable to match the mesh           *
+****************************************************/
+void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
+{
+    if ( mesh.className() == "DomainMesh" ) {
+        const IO::DomainMesh& mesh2 = dynamic_cast<const IO::DomainMesh&>( mesh );
+        if ( var.type == VariableType::NodeVariable ) {
+            size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1));
+            ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() );
+            var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } );
+        } else if ( var.type == VariableType::EdgeVariable ) {
+            ERROR("Not finished");
+        } else if ( var.type == VariableType::SurfaceVariable ) {
+            ERROR("Not finished");
+        } else if ( var.type == VariableType::VolumeVariable ) {
+            size_t N2 = var.data.length() / (mesh2.nx*mesh2.ny*mesh2.nz);
+            ASSERT( mesh2.nx*mesh2.ny*mesh2.nz*N2 == var.data.length() );
+            var.data.reshape( { (size_t) mesh2.nx, (size_t) mesh2.ny, (size_t) mesh2.nz, N2 } );
+        } else {
+            ERROR("Invalid variable type");
+        }
+    } else if ( mesh.className() == "PointList" ) {
+        const IO::PointList& mesh2 = dynamic_cast<const IO::PointList&>( mesh );
+        size_t N = mesh2.points.size();
+        size_t N_var = var.data.length()/N;
+        ASSERT( N*N_var == var.data.length() );
+        var.data.reshape( { N, N_var } );
+    } else if ( mesh.className()=="TriMesh" || mesh.className() == "TriList" ) {
+        std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh*>(&mesh), []( void* ) {} );
+        std::shared_ptr<TriMesh> mesh2 = getTriMesh( mesh_ptr );
+        if ( var.type == VariableType::NodeVariable ) {
+            size_t N = mesh2->vertices->points.size();
+            size_t N_var = var.data.length()/N;
+            ASSERT( N*N_var == var.data.length() );
+            var.data.reshape( { N, N_var } );
+        } else if ( var.type == VariableType::EdgeVariable ) {
+            ERROR("Not finished");
+        } else if ( var.type == VariableType::SurfaceVariable ) {
+            ERROR("Not finished");
+        } else if ( var.type == VariableType::VolumeVariable ) {
+            size_t N = mesh2->A.size();
+            size_t N_var = var.data.length()/N;
+            ASSERT( N*N_var == var.data.length() );
+            var.data.reshape( { N, N_var } );
+        } else {
+            ERROR("Invalid variable type");
+        }
+    } else {
+        ERROR("Unknown mesh type");
+    }
+}
+
+
+

IO/Reader.h

@@ -0,0 +1,58 @@
+#ifndef READER_INC
+#define READER_INC
+
+#include <iostream>
+#include <string.h>
+#include <vector>
+
+#include "IO/Mesh.h"
+#include "IO/MeshDatabase.h"
+#include "shared_ptr.h"
+
+
+namespace IO {
+
+
+//! Get the path to a file
+std::string getPath( const std::string& filename );
+
+
+//! List the timesteps in the given directors (dumps.LBPM)
+std::vector<std::string> readTimesteps( const std::string& filename );
+
+
+//! Read the list of mesh databases for the given timestep
+std::vector<IO::MeshDatabase> getMeshList( const std::string& path, const std::string& timestep );
+
+
+//! Read the given mesh domain
+std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& timestep, 
+    const MeshDatabase& meshDatabase, int domain );
+
+
+/*!
+ * @brief Read the given variable
+ * @details  This function reads the variable data provided for the current timestep
+ * @param[in] path          The path to the file
+ * @param[in] timestep      The timestep iteration
+ * @param[in] meshDatabase  The mesh database (see getMeshList)
+ * @param[in] domain        The index of the domain we want to read
+ * @param[in] variable      The variable name to read
+ * @return                  Returns the variable data as a linear array
+ */
+std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::string& timestep, 
+    const MeshDatabase& meshDatabase, int domain, const std::string& variable );
+
+
+/*!
+ * @brief Reformat the variable to match the mesh
+ * @details  This function modifies the dimensions of the array to match the mesh
+ * @param[in] mesh          The underlying mesh
+ * @param[in/out] variable  The variable name to read
+ */
+void reformatVariable( const IO::Mesh& mesh, IO::Variable& var );
+
+
+} // IO namespace
+
+#endif

IO/Writer.cpp

@@ -0,0 +1,508 @@
+#include "IO/Writer.h"
+#include "IO/MeshDatabase.h"
+#include "IO/IOHelpers.h"
+#include "IO/silo.h"
+#include "common/MPI_Helpers.h"
+#include "common/Utilities.h"
+#include "shared_ptr.h"
+
+#include <sys/stat.h>
+#include <algorithm>
+#include <vector>
+#include <set>
+
+
+
+enum class Format { OLD, NEW, SILO, UNKNOWN }; 
+
+
+
+/****************************************************
+* Initialize the writer                             *
+****************************************************/
+static std::string global_IO_path;
+static Format global_IO_format = Format::UNKNOWN;
+void IO::initialize( const std::string& path, const std::string& format, bool append )
+{
+    if ( path.empty() )
+        global_IO_path = ".";
+    else
+        global_IO_path = path;
+    if ( format == "old" )
+        global_IO_format = Format::OLD;
+    else if ( format == "new" )
+        global_IO_format = Format::NEW;
+    else if ( format == "silo" )
+        global_IO_format = Format::SILO;
+    else
+        ERROR("Unknown format");
+    int rank = comm_rank(MPI_COMM_WORLD);
+    if ( !append && rank==0 ) {
+        mkdir(path.c_str(),S_IRWXU|S_IRGRP);
+        std::string filename;
+        if ( global_IO_format==Format::OLD || global_IO_format==Format::NEW )
+            filename = global_IO_path + "/summary.LBM";
+        else if ( global_IO_format==Format::SILO )
+            filename = global_IO_path + "/LBM.visit";
+        else
+            ERROR("Unknown format");
+        auto fid = fopen(filename.c_str(),"wb");
+        fclose(fid);
+    }
+}
+
+
+// Write the mesh data in the original format
+static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
+{
+    int rank = MPI_WORLD_RANK();
+    std::vector<IO::MeshDatabase> meshes_written;
+    for (size_t i=0; i<meshData.size(); i++) {
+        char domainname[100], filename[100], fullpath[200];
+        sprintf(domainname,"%05i",rank);
+        sprintf(filename,"%s.%05i",meshData[i].meshName.c_str(),rank);
+	    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+        FILE *fid = fopen(fullpath,"wb");
+        INSIST(fid!=NULL,std::string("Error opening file: ")+fullpath);
+        std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
+        IO::MeshDatabase mesh_entry;
+        mesh_entry.name = meshData[i].meshName;
+        mesh_entry.type = meshType(*mesh);
+        mesh_entry.meshClass = meshData[i].mesh->className();
+        mesh_entry.format = 1;
+        IO::DatabaseEntry domain;
+        domain.name = domainname;
+        domain.file = filename;
+        domain.offset = 0;
+        mesh_entry.domains.push_back(domain);
+        if ( !meshData[i].vars.empty() ) {
+            printf("Warning: variables are not supported with this format\n");
+            //for (size_t j=0; j<meshData[i].vars.size(); j++)
+            //    mesh_entry.variables.push_back( meshData[i].vars[j]->name );
+        }
+        const std::string meshClass = mesh->className();
+        if ( meshClass=="PointList" ) {
+            // List of points
+            std::shared_ptr<IO::PointList> pointlist = std::dynamic_pointer_cast<IO::PointList>(mesh);
+            const std::vector<Point>& P = pointlist->points;
+            for (size_t i=0; i<P.size(); i++) {
+                double x[3];
+                x[0] = P[i].x;  x[1] = P[i].y;  x[2] = P[i].z;
+                fwrite(x,sizeof(double),3,fid);
+            }
+        } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
+            // Triangle mesh
+            std::shared_ptr<IO::TriList> trilist = IO::getTriList(mesh);
+            const std::vector<Point>& A = trilist->A;
+            const std::vector<Point>& B = trilist->B;
+            const std::vector<Point>& C = trilist->C;
+            for (size_t i=0; i<A.size(); i++) {
+                double tri[9];
+                tri[0] = A[i].x;  tri[1] = A[i].y;  tri[2] = A[i].z;
+                tri[3] = B[i].x;  tri[4] = B[i].y;  tri[5] = B[i].z;
+                tri[6] = C[i].x;  tri[7] = C[i].y;  tri[8] = C[i].z;
+                fwrite(tri,sizeof(double),9,fid);
+            }
+        } else if ( meshClass=="DomainMesh" ) {
+            // This format was never supported with the old format
+        } else {
+            ERROR("Unknown mesh");
+        }
+        fclose(fid);
+        std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() );
+        mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() );
+        meshes_written.push_back(mesh_entry);
+    }
+    return meshes_written;
+}
+
+
+// Create the database entry for the mesh data
+static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
+{
+    int rank = MPI_WORLD_RANK();
+    char domainname[100];
+    sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank);
+    // Create the MeshDatabase
+    IO::MeshDatabase database;
+    database.name = mesh.meshName;
+    database.type = meshType(*(mesh.mesh));
+    database.meshClass = mesh.mesh->className();
+    database.format = format;
+    // Write the mesh
+    IO::DatabaseEntry domain;
+    domain.name = domainname;
+    domain.file = filename;
+    domain.offset = -1;
+    database.domains.push_back(domain);
+    // Write the variables
+    for (size_t i=0; i<mesh.vars.size(); i++) {
+        // Add basic variable info
+        IO::VariableDatabase info;
+        info.name = mesh.vars[i]->name;
+        info.type = mesh.vars[i]->type;
+        info.dim = mesh.vars[i]->dim;
+        database.variables.push_back(info);
+        // Add domain variable info
+        IO::DatabaseEntry variable;
+        variable.name = mesh.vars[i]->name;
+        variable.file = filename;
+        variable.offset = -1;
+        std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
+        database.variable_data.insert( 
+            std::pair<std::pair<std::string,std::string>,IO::DatabaseEntry>(key,variable) );
+    }
+    return database;
+}
+
+
+// Write a mesh (and variables) to a file
+static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
+    const IO::MeshDataStruct& mesh, int format )
+{
+    const int level = 0;
+    int rank = MPI_WORLD_RANK();
+    // Create the MeshDatabase
+    IO::MeshDatabase database = getDatabase( filename, mesh, format );
+    // Write the mesh
+    IO::DatabaseEntry& domain = database.domains[0];
+    domain.offset = ftell(fid);
+    std::pair<size_t,void*> data = mesh.mesh->pack(level);
+    fprintf(fid,"Mesh: %s-%05i: %lu\n",mesh.meshName.c_str(),rank,data.first);
+    fwrite(data.second,1,data.first,fid);
+    fprintf(fid,"\n");
+    delete [] (char*) data.second;
+    // Write the variables
+    for (size_t i=0; i<mesh.vars.size(); i++) {
+        std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
+        IO::DatabaseEntry& variable = database.variable_data[key];
+        variable.offset = ftell(fid);
+        int dim = mesh.vars[i]->dim;
+        int type = static_cast<int>(mesh.vars[i]->type);
+        size_t N = mesh.vars[i]->data.length();
+        if ( type == static_cast<int>(IO::VariableType::NullVariable) ) {
+            ERROR("Variable type not set");
+        }
+        size_t N_mesh = mesh.mesh->numberPointsVar(mesh.vars[i]->type);
+        ASSERT(N==dim*N_mesh);
+        fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n",
+            database.name.c_str(), rank, variable.name.c_str(),
+            dim, type, N_mesh, N*sizeof(double) );
+        fwrite(mesh.vars[i]->data.data(),sizeof(double),N,fid);
+        fprintf(fid,"\n");
+    }
+    return database;
+}
+
+
+#ifdef USE_SILO
+// Write a PointList mesh (and variables) to a file
+template<class TYPE>
+static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname )
+{
+    const auto& points = mesh.getPoints();
+    std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
+    for (size_t i=0; i<x.size(); i++) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const TYPE *coords[] = { x.data(), y.data(), z.data() };
+    silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
+}
+static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+{
+    const IO::PointList& mesh = dynamic_cast<IO::PointList&>( *meshData.mesh );
+    const std::string meshname = database.domains[0].name;
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeSiloPointMesh<double>( fid, mesh, meshname );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeSiloPointMesh<float>( fid, mesh, meshname );
+    } else {
+        ERROR("Unsupported format");
+    }
+    const auto& points = mesh.getPoints();
+    std::vector<double> x(points.size()), y(points.size()), z(points.size());
+    for (size_t i=0; i<x.size(); i++) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const double *coords[] = { x.data(), y.data(), z.data() };
+    silo::writePointMesh( fid, meshname, 3, points.size(), coords );
+    for (size_t i=0; i<meshData.vars.size(); i++) {
+        const IO::Variable& var = *meshData.vars[i];
+        if ( var.precision == IO::DataType::Double ) {
+            silo::writePointMeshVariable( fid, meshname, var.name, var.data );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+        } else {
+            ERROR("Unsupported format");
+        }
+    }
+}
+// Write a TriMesh mesh (and variables) to a file
+template<class TYPE>
+static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::string& meshname )
+{
+    const auto& points = mesh.vertices->getPoints();
+    std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
+    for (size_t i=0; i<x.size(); i++) {
+        x[i] = points[i].x;
+        y[i] = points[i].y;
+        z[i] = points[i].z;
+    }
+    const TYPE *coords[] = { x.data(), y.data(), z.data() };
+    const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
+    silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
+}
+static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
+    const IO::TriMesh& mesh, IO::MeshDatabase database )
+{
+    const std::string meshname = database.domains[0].name;
+    if ( meshData.precision == IO::DataType::Double ) {
+        writeSiloTriMesh<double>( fid, mesh, meshname );
+    } else if ( meshData.precision == IO::DataType::Float ) {
+        writeSiloTriMesh<float>( fid, mesh, meshname );
+    } else {
+        ERROR("Unsupported format");
+    }
+    for (size_t i=0; i<meshData.vars.size(); i++) {
+        const IO::Variable& var = *meshData.vars[i];
+        auto type = static_cast<silo::VariableType>( var.type );
+        if ( var.precision == IO::DataType::Double ) {
+            silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
+        } else {
+            ERROR("Unsupported format");
+        }
+    }
+}
+static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+{
+    const IO::TriMesh& mesh = dynamic_cast<IO::TriMesh&>( *meshData.mesh );
+    writeSiloTriMesh2( fid, meshData, mesh, database );
+}
+static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+{
+    auto mesh = getTriMesh( meshData.mesh );
+    writeSiloTriMesh2( fid, meshData, *mesh, database );
+}
+// Write a DomainMesh mesh (and variables) to a file
+static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+{
+    const IO::DomainMesh& mesh = dynamic_cast<IO::DomainMesh&>( *meshData.mesh );
+    RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
+    std::array<double,6> range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx, 
+                                   info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny,
+                                   info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz };
+    std::array<int,3> N = { mesh.nx, mesh.ny, mesh.nz };
+    auto meshname = database.domains[0].name;
+    silo::writeUniformMesh<3>( fid, meshname, range, N );
+    silo::write<int>( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
+    for (size_t i=0; i<meshData.vars.size(); i++) {
+        const auto& var = *meshData.vars[i];
+        auto type = static_cast<silo::VariableType>( var.type );
+        if ( var.precision == IO::DataType::Double ) {
+            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type );
+        } else if ( var.precision == IO::DataType::Float ) {
+            Array<float> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
+        } else if ( var.precision == IO::DataType::Int ) {
+            Array<int> data2( var.data.size() );
+            data2.copy( var.data );
+            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
+        } else {
+            ERROR("Unsupported format");
+        }
+    }
+}
+// Write a mesh (and variables) to a file
+static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename,
+    const IO::MeshDataStruct& mesh, int format )
+{
+    // Create the MeshDatabase
+    auto database = getDatabase( filename, mesh, format );
+    if ( database.meshClass=="PointList" ) {
+        writeSiloPointList( fid, mesh, database );
+    } else if ( database.meshClass=="TriMesh" ) {
+        writeSiloTriMesh( fid, mesh, database );
+    } else if ( database.meshClass=="TriList" ) {
+        writeSiloTriList( fid, mesh, database );
+    } else if ( database.meshClass=="DomainMesh" ) {
+        writeSiloDomainMesh( fid, mesh, database );
+    } else {
+        ERROR("Unknown mesh class");
+    }
+    return database;
+}
+// Write the summary file for silo
+std::pair<int,int> getSiloMeshType( const std::string& meshClass )
+{
+    int meshType = 0;
+    int varType = 0;
+    if ( meshClass=="PointList" ) {
+        meshType = DB_POINTMESH;
+        varType  = DB_POINTVAR;
+    } else if ( meshClass=="TriMesh" ) {
+        meshType = DB_UCDMESH;
+        varType  = DB_UCDVAR;
+    } else if ( meshClass=="TriList" ) {
+        meshType = DB_UCDMESH;
+        varType  = DB_UCDVAR;
+    } else if ( meshClass=="DomainMesh" ) {
+        meshType = DB_QUAD_RECT;
+        varType  = DB_QUADVAR;
+    } else {
+        ERROR("Unknown mesh class");
+    }
+    return std::make_pair( meshType, varType );
+}
+void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, const std::string& filename )
+{
+    auto fid = silo::open( filename, silo::CREATE );
+    for ( const auto& data : meshes_written ) {
+        auto type = getSiloMeshType( data.meshClass );
+        std::vector<int> meshTypes( data.domains.size(), type.first );
+        std::vector<int> varTypes( data.domains.size(), type.second );
+        std::vector<std::string> meshNames;
+        for ( const auto& tmp : data.domains )
+            meshNames.push_back( tmp.file + ":" + tmp.name );
+        silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
+        for (const auto& variable : data.variables ) {
+            std::vector<std::string> varnames;
+            for ( const auto& tmp : data.domains )
+                varnames.push_back( tmp.file + ":" + variable.name );
+            silo::writeMultiVar( fid, variable.name, varnames, varTypes );
+        }
+    }
+    silo::close( fid );
+}
+#endif
+
+
+// Write the mesh data in the new format
+static std::vector<IO::MeshDatabase> writeMeshesNewFormat( 
+    const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
+{
+    int rank = MPI_WORLD_RANK();
+    std::vector<IO::MeshDatabase> meshes_written;
+    char filename[100], fullpath[200];
+    sprintf(filename,"%05i",rank);
+    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+    FILE *fid = fopen(fullpath,"wb");
+    for (size_t i=0; i<meshData.size(); i++) {
+        std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
+        meshes_written.push_back( write_domain(fid,filename,meshData[i],format) );
+    }
+    fclose(fid);
+    return meshes_written;
+}
+
+
+// Write the mesh data to silo
+static std::vector<IO::MeshDatabase> writeMeshesSilo( 
+    const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
+{
+#ifdef USE_SILO
+    int rank = MPI_WORLD_RANK();
+    std::vector<IO::MeshDatabase> meshes_written;
+    char filename[100], fullpath[200];
+    sprintf(filename,"%05i.silo",rank);
+    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+    auto fid = silo::open( fullpath, silo::CREATE );
+    for (size_t i=0; i<meshData.size(); i++) {
+        auto mesh = meshData[i].mesh;
+        meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) );
+    }
+    silo::close( fid );
+    return meshes_written;
+#else
+    ERROR("Application built without silo support");
+    return std::vector<IO::MeshDatabase>();
+#endif
+}        
+
+
+/****************************************************
+* Write the mesh data                               *
+****************************************************/
+void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
+{
+    if ( global_IO_path.empty() )
+        IO::initialize( );
+    PROFILE_START("writeData");
+    int rank = comm_rank(comm);
+    // Check the meshData before writing
+    for ( const auto& data : meshData ) {
+        if ( !data.check() )
+            ERROR("Error in meshData");
+    }
+    // Create the output directory
+    std::string path = global_IO_path + "/" + subdir;
+    if ( rank == 0 ) {
+        mkdir(path.c_str(),S_IRWXU|S_IRGRP);
+    }
+    MPI_Barrier(comm);
+    // Write the mesh files
+    std::vector<IO::MeshDatabase> meshes_written;
+    if ( global_IO_format == Format::OLD ) {
+        // Write the original triangle format
+        meshes_written = writeMeshesOrigFormat( meshData, path );
+    } else if ( global_IO_format == Format::NEW ) {
+        // Write the new format (double precision)
+        meshes_written = writeMeshesNewFormat( meshData, path, 2 );
+    } else if ( global_IO_format == Format::SILO ) {
+        // Write silo
+        meshes_written = writeMeshesSilo( meshData, path, 4 );
+    } else {
+        ERROR("Unknown format");
+    }
+    // Gather a complete list of files on rank 0
+    meshes_written = gatherAll(meshes_written,comm);
+    // Write the summary files
+    if ( rank == 0 ) {
+        // Write the summary file for the current timestep
+        char filename[200];
+        sprintf(filename,"%s/LBM.summary",path.c_str());
+        write(meshes_written,filename);
+        // Write summary silo file if needed
+        #ifdef USE_SILO
+        if ( global_IO_format == Format::SILO ) {
+            sprintf(filename,"%s/summary.silo",path.c_str());
+            writeSiloSummary(meshes_written,filename);
+        }
+        #endif
+        // Add the timestep to the global summary file
+        if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) {
+            auto filename = global_IO_path+"/summary.LBM";
+            FILE *fid = fopen(filename.c_str(),"ab");
+            fprintf(fid,"%s/\n",subdir.c_str());
+            fclose(fid);
+        } else if ( global_IO_format == Format::SILO ) {
+            auto filename = global_IO_path+"/LBM.visit";
+            FILE *fid = fopen(filename.c_str(),"ab");
+            fprintf(fid,"%s/summary.silo\n",subdir.c_str());
+            fclose(fid);
+        } else {
+            ERROR("Unknown format");
+        }
+    }
+    PROFILE_STOP("writeData");
+}
+
+

IO/Writer.h

@@ -0,0 +1,57 @@
+#ifndef WRITER_INC
+#define WRITER_INC
+
+#include <iostream>
+#include <string.h>
+#include <vector>
+
+#include "IO/Mesh.h"
+#include "IO/MeshDatabase.h"
+
+
+namespace IO {
+
+
+/*!
+ * @brief  Initialize the writer
+ * @details  This function initializes the writer to the given path.  All subsequent
+ *    writes will occur in this directory.  If this is not called, then it will default
+ *    to the current path.
+ * @param[in] path          The path to use for writes
+ * @param[in] format        The data format to use:
+ *                          old - Old mesh format (provided for backward compatibility, cannot write variables)
+ *                          new - New format, 1 file/process
+ *                          silo - Silo
+ * @param[in] append        Append any existing data (default is false)
+ */
+void initialize( const std::string& path="", const std::string& format="new", bool append=false );
+
+
+/*!
+ * @brief  Write the data for the timestep
+ * @details  This function writes the mesh and variable data provided for the current timestep
+ * @param[in] subdir        The subdirectory to use for the timestep
+ * @param[in] meshData      The data to write
+ * @param[in] comm          The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
+ */
+void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm );
+
+
+/*!
+ * @brief  Write the data for the timestep
+ * @details  This function writes the mesh and variable data provided for the current timestep
+ * @param[in] timestep      The timestep iteration
+ * @param[in] meshData      The data to write
+ * @param[in] comm          The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
+ */
+inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
+{
+    char subdir[100];
+    sprintf(subdir,"vis%03i",timestep);
+    writeData( subdir, meshData, comm );
+}
+
+
+} // IO namespace
+
+#endif

IO/netcdf.cpp

@@ -0,0 +1,471 @@
+#include "IO/netcdf.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+
+#include "ProfilerApp.h"
+
+
+#ifdef USE_NETCDF
+
+
+#include <netcdf.h>
+#include <netcdf_par.h>
+
+
+#define CHECK_NC_ERR( ERR )             \
+    do {                                \
+        if ( ERR != NC_NOERR ) {        \
+            std::string msg = "Error calling netcdf routine: "; \
+            msg += nc_strerror( ERR );  \
+            ERROR( msg );               \
+        }                               \
+    } while (0)
+
+
+namespace netcdf {
+
+
+// Convert nc_type to VariableType
+static inline VariableType convertType( nc_type type )
+{
+    VariableType type2 = UNKNOWN;
+    if ( type == NC_BYTE )
+        type2 = BYTE;
+    else if ( type == NC_CHAR )
+        type2 = STRING;
+    else if ( type == NC_SHORT )
+        type2 = SHORT;
+    else if ( type == NC_USHORT )
+        type2 = USHORT;
+    else if ( type == NC_INT )
+        type2 = INT;
+    else if ( type == NC_UINT )
+        type2 = UINT;
+    else if ( type == NC_INT64 )
+        type2 = INT64;
+    else if ( type == NC_UINT64 )
+        type2 = UINT64;
+    else if ( type == NC_FLOAT )
+        type2 = FLOAT;
+    else if ( type == NC_DOUBLE )
+        type2 = DOUBLE;
+    else
+        ERROR("Unknown type");
+    return type2;
+}
+
+
+// Get nc_type from the template
+template<class T> inline nc_type getType();
+template<> inline nc_type getType<char>()   { return NC_CHAR; }
+template<> inline nc_type getType<short>()  { return NC_SHORT; }
+template<> inline nc_type getType<int>()    { return NC_INT; }
+template<> inline nc_type getType<float>()  { return NC_FLOAT; }
+template<> inline nc_type getType<double>() { return NC_DOUBLE; }
+
+
+// Function to reverse an array
+template<class TYPE>
+inline std::vector<TYPE> reverse( const std::vector<TYPE>& x )
+{
+    std::vector<TYPE> y(x.size());
+    for (size_t i=0; i<x.size(); i++)
+        y[i] = x[x.size()-i-1];
+    return y;
+}
+// Function to reverse an array
+template<class TYPE1, class TYPE2>
+inline std::vector<TYPE2> convert( const std::vector<TYPE1>& x )
+{
+    std::vector<TYPE2> y(x.size());
+    for (size_t i=0; i<x.size(); i++)
+        y[i] = static_cast<TYPE2>(x[i]);
+    return y;
+}
+
+
+/****************************************************
+* Convert the VariableType to a string              *
+****************************************************/
+std::string VariableTypeName( VariableType type )
+{
+    if ( type == BYTE )
+        return "BYTE";
+    else if ( type == SHORT )
+        return "SHORT";
+    else if ( type == USHORT )
+        return "USHORT";
+    else if ( type == INT )
+        return "INT";
+    else if ( type == UINT )
+        return "UINT";
+    else if ( type == INT64 )
+        return "INT64";
+    else if ( type == UINT64 )
+        return "UINT64";
+    else if ( type == FLOAT )
+        return "FLOAT";
+    else if ( type == DOUBLE )
+        return "DOUBLE";
+    else if ( type == STRING )
+        return "STRING";
+    return "Unknown";
+}
+
+
+/****************************************************
+* Open/close a file                                 *
+****************************************************/
+int open( const std::string& filename, FileMode mode, MPI_Comm comm )
+{
+    int fid = 0;
+    if ( comm == MPI_COMM_NULL ) {
+        if ( mode == READ ) {
+            int err = nc_open( filename.c_str(), NC_NOWRITE, &fid );
+            CHECK_NC_ERR( err );
+        } else if ( mode == WRITE ) {
+            int err = nc_open( filename.c_str(), NC_WRITE, &fid );
+            CHECK_NC_ERR( err );
+        } else if ( mode == CREATE ) {
+            int err = nc_create( filename.c_str(), NC_SHARE|NC_64BIT_OFFSET, &fid );
+            CHECK_NC_ERR( err );
+        } else {
+            ERROR("Unknown file mode");
+        }
+    } else {
+        if ( mode == READ ) {
+            int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
+            CHECK_NC_ERR( err );
+        } else if ( mode == WRITE ) {
+            int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
+            CHECK_NC_ERR( err );
+        } else if ( mode == CREATE ) {
+            int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm, MPI_INFO_NULL, &fid );
+            CHECK_NC_ERR( err );
+        } else {
+            ERROR("Unknown file mode");
+        }
+    }
+    return fid;
+}
+void close( int fid )
+{
+    int err = nc_close( fid );
+    if ( err != NC_NOERR )
+        ERROR("Error closing file");
+}
+
+
+/****************************************************
+* Query basic properties                            *
+****************************************************/
+static std::vector<size_t> getDimVar( int fid, int varid )
+{
+    int ndim = 0;
+    int err = nc_inq_varndims( fid, varid, &ndim );
+    CHECK_NC_ERR( err );
+    std::vector<size_t> dims(ndim,0);
+    int dimid[64] = {-1};
+    err = nc_inq_vardimid( fid, varid, dimid );
+    CHECK_NC_ERR( err );
+    for (int i=0; i<ndim; i++) {
+        err = nc_inq_dimlen( fid, dimid[i], &dims[i] );
+        CHECK_NC_ERR( err );
+    }
+    return dims;
+}
+static int getVarID( int fid, const std::string& var )
+{
+    int id = -1;
+    int err = nc_inq_varid( fid, var.c_str(), &id );
+    CHECK_NC_ERR( err );
+    return id;
+}
+std::vector<size_t> getVarDim( int fid, const std::string& var )
+{
+    return getDimVar( fid, getVarID( fid, var ) );
+}
+std::vector<size_t> getAttDim( int fid, const std::string& att )
+{
+    std::vector<size_t> dim(1,0);
+    int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
+    return dim;
+}
+std::vector<std::string> getVarNames( int fid )
+{
+    int nvar;
+    int err = nc_inq( fid, NULL, &nvar, NULL, NULL );
+    CHECK_NC_ERR( err );
+    std::vector<std::string> vars(nvar);
+    for (int i=0; i<nvar; i++) {
+        char name[NC_MAX_NAME+1];
+        err = nc_inq_varname( fid, i, name );
+        CHECK_NC_ERR( err );
+        vars[i] = name;
+    }
+    return vars;
+}
+std::vector<std::string> getAttNames( int fid )
+{
+    int natt;
+    int err = nc_inq( fid, NULL, NULL, &natt, NULL );
+    CHECK_NC_ERR( err );
+    std::vector<std::string> att(natt);
+    for (int i=0; i<natt; i++) {
+        char name[NC_MAX_NAME+1];
+        err = nc_inq_attname( fid,  NC_GLOBAL, i, name );
+        CHECK_NC_ERR( err );
+        att[i] = name;
+    }
+    return att;
+}
+VariableType getVarType( int fid, const std::string& var )
+{
+    int varid = -1;
+    int err = nc_inq_varid( fid, var.c_str(), &varid );
+    CHECK_NC_ERR( err );
+    nc_type type=0;
+    err = nc_inq_vartype( fid, varid, &type );
+    CHECK_NC_ERR( err );
+    return convertType(type);
+}
+VariableType getAttType( int fid, const std::string& att )
+{
+    nc_type type=0;
+    int err = nc_inq_atttype( fid,  NC_GLOBAL, att.c_str(), &type );
+    CHECK_NC_ERR( err );
+    return convertType(type);
+}
+
+
+
+/****************************************************
+* Read a variable                                   *
+****************************************************/
+template<>
+Array<unsigned short> getVar<unsigned short>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<unsigned short>");
+    Array<unsigned short> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_ushort( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<unsigned short>");
+    return x.reverseDim();
+}
+template<>
+Array<short> getVar<short>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<short>");
+    Array<short> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_short( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<short>");
+    return x.reverseDim();
+}
+template<>
+Array<unsigned int> getVar<unsigned int>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<unsigned int>");
+    Array<unsigned int> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_uint( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<unsigned int>");
+    return x.reverseDim();
+}
+template<>
+Array<int> getVar<int>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<int>");
+    Array<int> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_int( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<int>");
+    return x.reverseDim();
+}
+template<>
+Array<float> getVar<float>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<float>");
+    Array<float> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_float( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<float>");
+    return x.reverseDim();
+}
+template<>
+Array<double> getVar<double>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<double>");
+    Array<double> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_double( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<double>");
+    return x.reverseDim();
+}
+template<>
+Array<char> getVar<char>( int fid, const std::string& var )
+{ 
+    PROFILE_START("getVar<char>");
+    Array<char> x( reverse(getVarDim(fid,var)) );
+    int err = nc_get_var_text( fid, getVarID(fid,var), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<char>");
+    return x.reverseDim();
+}
+template<>
+Array<std::string> getVar<std::string>( int fid, const std::string& var )
+{
+    PROFILE_START("getVar<std::string>");
+    Array<char> tmp = getVar<char>( fid, var );
+    std::vector<size_t> dim = tmp.size();
+    if ( dim.size() == 1 )
+        dim[0] = 1;
+    else
+        dim.erase( dim.begin() );
+    Array<std::string> text(dim);
+    for (size_t i=0; i<text.length(); i++)
+        text(i) = &(tmp(0,i));
+    PROFILE_STOP("getVar<std::string>");
+    return text;
+}
+static inline void get_stride_args( const std::vector<int>& start,
+    const std::vector<int>& count, const std::vector<int>& stride,
+    size_t *startp, size_t *countp, ptrdiff_t *stridep )
+{
+    for (size_t i=0; i<start.size(); i++)
+        startp[i] = start[i];
+    for (size_t i=0; i<count.size(); i++)
+        countp[i] = count[i];
+    for (size_t i=0; i<stride.size(); i++)
+        stridep[i] = stride[i];
+}
+template<class TYPE>
+int nc_get_vars_TYPE( int fid, int varid, const size_t start[],
+    const size_t count[], const ptrdiff_t stride[], TYPE *ptr );
+template<>
+int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[],
+    const size_t count[], const ptrdiff_t stride[], short *ptr )
+{
+    return nc_get_vars_short( fid, varid, start, count, stride, ptr );
+}
+template<>
+int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[],
+    const size_t count[], const ptrdiff_t stride[], int *ptr )
+{
+    return nc_get_vars_int( fid, varid, start, count, stride, ptr );
+}
+template<>
+int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[],
+    const size_t count[], const ptrdiff_t stride[], float *ptr )
+{
+    return nc_get_vars_float( fid, varid, start, count, stride, ptr );
+}
+template<>
+int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[],
+    const size_t count[], const ptrdiff_t stride[], double *ptr )
+{
+    return nc_get_vars_double( fid, varid, start, count, stride, ptr );
+}
+template<class TYPE>
+Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
+    const std::vector<int>& count, const std::vector<int>& stride )
+{
+    PROFILE_START("getVar<> (strided)");
+    std::vector<size_t> var_size = getVarDim( fid, var );
+    for (int d=0; d<(int)var_size.size(); d++) {
+        if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) {
+            int rank = comm_rank(MPI_COMM_WORLD);
+            char tmp[1000];
+            sprintf(tmp,"%i: Range exceeded array dimension:\n"
+                "   start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",
+                rank,d,start[d],d,count[d],d,stride[d],d,(int)var_size[d]);
+            ERROR(tmp);
+        }
+    }
+    Array<TYPE> x( reverse(convert<int,size_t>(count)) );
+    size_t startp[10], countp[10];
+    ptrdiff_t stridep[10];
+    get_stride_args( start, count, stride, startp, countp, stridep );
+    int err = nc_get_vars_TYPE<TYPE>( fid, getVarID(fid,var), startp, countp, stridep, x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getVar<> (strided)");
+    return x.reverseDim();
+}
+template Array<short>  getVar<short>(  int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+template Array<int>    getVar<int>(    int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+template Array<float>  getVar<float>(  int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+template Array<double> getVar<double>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+
+
+/****************************************************
+* Read an attribute                                 *
+****************************************************/
+template<>
+Array<double> getAtt<double>( int fid, const std::string& att )
+{
+    PROFILE_START("getAtt<double>");
+    Array<double> x( getAttDim(fid,att) );
+    int err = nc_get_att_double( fid, NC_GLOBAL, att.c_str(), x.data() );
+    CHECK_NC_ERR( err );
+    PROFILE_STOP("getAtt<double>");
+    return x;
+}
+template<>
+Array<std::string> getAtt<std::string>( int fid, const std::string& att )
+{
+    PROFILE_START("getAtt<std::string>");
+    char *tmp = new char[getAttDim(fid,att)[0]];
+    Array<std::string> x(1);
+    x(0) = tmp;
+    delete [] tmp;
+    PROFILE_STOP("getAtt<std::string>");
+    return x;
+}
+
+
+/****************************************************
+* Write an array to a file                          *
+****************************************************/
+std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims )
+{
+    std::vector<int> dimid(names.size(),0);
+    for (size_t i=0; i<names.size(); i++) {
+        int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i]);
+        CHECK_NC_ERR( err );
+    }
+    return dimid;
+}
+template<class TYPE>
+void write( int fid, const std::string& var, const std::vector<int>& dimids,
+    const Array<TYPE>& data, const RankInfoStruct& info )
+{
+    // Define the variable
+    int varid = 0;
+    int err = nc_def_var( fid, var.c_str(), getType<TYPE>(), data.ndim(), dimids.data(), &varid );
+    CHECK_NC_ERR( err );
+    // exit define mode 
+    err = nc_enddef( fid );
+    CHECK_NC_ERR( err );
+    // set the access method to use MPI/PnetCDF collective I/O 
+    err = nc_var_par_access( fid, varid, NC_INDEPENDENT );
+    CHECK_NC_ERR( err );
+    // parallel write: each process writes its subarray to the file
+    auto x = data.reverseDim();
+    std::vector<size_t> count = data.size();
+    std::vector<size_t> start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) };
+    nc_put_vara( fid, varid, start.data(), count.data(), x.data() );
+}
+template void write<short>(  int fid, const std::string& var, const std::vector<int>& dimids, const Array<short>& data,  const RankInfoStruct& info );
+template void write<int>(    int fid, const std::string& var, const std::vector<int>& dimids, const Array<int>& data,    const RankInfoStruct& info );
+template void write<float>(  int fid, const std::string& var, const std::vector<int>& dimids, const Array<float>& data,  const RankInfoStruct& info );
+template void write<double>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<double>& data, const RankInfoStruct& info );
+
+
+
+}; // netcdf namespace
+
+#else
+
+#endif
+
+

IO/netcdf.h

@@ -0,0 +1,141 @@
+#ifndef NETCDF_READER
+#define NETCDF_READER
+
+#include <string>
+#include <vector>
+
+#include "common/Array.h"
+#include "common/MPI_Helpers.h"
+#include "common/Communication.h"
+
+
+
+namespace netcdf {
+
+
+//! Enum to hold variable type
+enum VariableType { BYTE, SHORT, USHORT, INT, UINT, INT64, UINT64, FLOAT, DOUBLE, STRING, UNKNOWN };
+
+//! Enum to hold variable type
+enum FileMode { READ, WRITE, CREATE };
+
+
+//! Convert the VariableType to a string
+std::string VariableTypeName( VariableType type );
+
+
+/*!
+ * @brief  Open netcdf file
+ * @detailed  This function opens a netcdf file
+ * @return This function returns a handle to the file
+ * @param filename      File to open
+ * @param mode          Open the file for reading or writing
+ * @param comm          MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf)
+*/
+int open( const std::string& filename, FileMode mode, MPI_Comm comm=MPI_COMM_NULL );
+
+
+/*!
+ * @brief  Close netcdf file
+ * @detailed  This function closes a netcdf file
+ * @param fid           Handle to the open file
+*/
+void close( int fid );
+
+
+/*!
+ * @brief  Read the variable names
+ * @detailed  This function reads a list of the variable names in the file
+ * @param fid           Handle to the open file
+*/
+std::vector<std::string> getVarNames( int fid );
+
+
+/*!
+ * @brief  Read the attribute names
+ * @detailed  This function reads a list of the attribute names in the file
+ * @param fid           Handle to the open file
+*/
+std::vector<std::string> getAttNames( int fid );
+
+
+/*!
+ * @brief  Return the variable type
+ * @detailed  This function returns the type for a variable
+ * @param fid           Handle to the open file
+ * @param var           Variable to read
+*/
+VariableType getVarType( int fid, const std::string& var );
+
+
+/*!
+ * @brief  Return the attribute type
+ * @detailed  This function returns the type for an attribute
+ * @param fid           Handle to the open file
+ * @param att           Attribute to read
+*/
+VariableType getAttType( int fid, const std::string& att );
+
+
+/*!
+ * @brief  Return the variable dimensions
+ * @detailed  This function returns the die for a variable
+ * @param fid           Handle to the open file
+ * @param var           Variable to read
+*/
+std::vector<size_t> getVarDim( int fid, const std::string& var );
+
+
+/*!
+ * @brief  Read a variable
+ * @detailed  This function reads a variable with the given name from the file
+ * @param fid           Handle to the open file
+ * @param var           Variable to read
+*/
+template<class TYPE>
+Array<TYPE> getVar( int fid, const std::string& var );
+
+
+/*!
+ * @brief  Read a strided variable
+ * @detailed  This function reads a strided variable with the given name from the file
+ * @param fid           Handle to the open file
+ * @param var           Variable to read
+ * @param start         Starting corner for the read
+ * @param count         Number of elements to read
+ * @param stride        Stride size for the read
+*/
+template<class TYPE>
+Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
+    const std::vector<int>& count, const std::vector<int>& stride );
+
+
+/*!
+ * @brief  Read an attribute
+ * @detailed  This function reads an attribute with the given name from the file
+ * @param fid           Handle to the open file
+ * @param att           Attribute to read
+*/
+template<class TYPE>
+Array<TYPE> getAtt( int fid, const std::string& att );
+
+
+/*!
+ * @brief  Write the dimensions
+ * @detailed  This function writes the grid dimensions to netcdf. 
+ * @param fid           Handle to the open file
+*/
+std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims );
+
+
+/*!
+ * @brief  Write a variable
+ * @detailed  This function writes a variable to netcdf. 
+ * @param fid           Handle to the open file
+*/
+template<class TYPE>
+void write( int fid, const std::string& var, const std::vector<int>& dimids, const Array<TYPE>& data, const RankInfoStruct& rank_info );
+
+
+}; // netcdf namespace
+#endif

IO/silo.cpp

@@ -0,0 +1,112 @@
+#include "IO/silo.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+
+#include "ProfilerApp.h"
+
+
+#ifdef USE_SILO
+
+#include <silo.h>
+
+
+
+namespace silo {
+
+
+/****************************************************
+* Open/close a file                                 *
+****************************************************/
+DBfile* open( const std::string& filename, FileMode mode )
+{
+    DBfile *fid = nullptr;
+    if ( mode == CREATE ) {
+        fid = DBCreate( filename.c_str(), DB_CLOBBER, DB_LOCAL, nullptr, DB_HDF5 );
+    } else if ( mode == WRITE ) {
+        fid = DBOpen( filename.c_str(), DB_HDF5, DB_APPEND );
+    } else if ( mode == READ ) {
+        fid = DBOpen( filename.c_str(), DB_HDF5, DB_READ );
+    }
+    return fid;
+}
+void close( DBfile* fid )
+{
+    DBClose( fid );
+}
+
+
+/****************************************************
+* Helper functions                                  *
+****************************************************/
+VariableDataType varDataType( DBfile *fid, const std::string& name )
+{
+    auto type = DBGetVarType( fid, name.c_str() );
+    VariableDataType type2 = VariableDataType::UNKNOWN;
+    if ( type == DB_DOUBLE )
+        type2 = VariableDataType::DOUBLE;
+    else if ( type == DB_FLOAT )
+        type2 = VariableDataType::FLOAT;
+    else if ( type == DB_INT )
+        type2 = VariableDataType::INT;
+    return type2;
+}
+
+
+/****************************************************
+* Write/read a uniform mesh to silo                 *
+****************************************************/
+void readUniformMesh( DBfile* fid, const std::string& meshname,
+    std::vector<double>& range, std::vector<int>& N )
+{
+    DBquadmesh* mesh = DBGetQuadmesh( fid, meshname.c_str() );
+    int ndim = mesh->ndims;
+    range.resize(2*ndim);
+    N.resize(ndim);
+    for (int d=0; d<ndim; d++) {
+        N[d] = mesh->dims[d]-1;
+        range[2*d+0] = mesh->min_extents[d];
+        range[2*d+1] = mesh->max_extents[d];
+    }
+    DBFreeQuadmesh( mesh );
+}
+
+
+/****************************************************
+* Write a multimesh                                 *
+****************************************************/
+void writeMultiMesh( DBfile* fid, const std::string& meshname,
+    const std::vector<std::string>& meshNames,
+    const std::vector<int>& meshTypes )
+{
+    std::vector<char*> meshnames(meshNames.size());
+    for ( size_t i = 0; i < meshNames.size(); ++i )
+        meshnames[i] = (char *) meshNames[i].c_str();
+    std::string tree_name = meshname + "_tree";
+    DBoptlist *optList    = DBMakeOptlist( 1 );
+    DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() );
+    DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr );
+    DBFreeOptlist( optList );
+}
+
+
+/****************************************************
+* Write a multivariable                             *
+****************************************************/
+void writeMultiVar( DBfile* fid, const std::string& varname,
+    const std::vector<std::string>& varNames,
+    const std::vector<int>& varTypes )
+{
+    std::vector<char*> varnames(varNames.size(),nullptr);
+    for (size_t j=0; j<varNames.size(); j++)
+        varnames[j] = const_cast<char*>(varNames[j].c_str());
+    DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr );
+}
+
+
+
+}; // silo namespace
+
+
+#else
+
+#endif

IO/silo.h

@@ -0,0 +1,267 @@
+#ifndef SILO_INTERFACE
+#define SILO_INTERFACE
+
+#include <string>
+#include <vector>
+#include <array>
+
+#include "common/Array.h"
+#include "common/MPI_Helpers.h"
+#include "common/Communication.h"
+
+
+#ifdef USE_SILO
+    #include <silo.h>
+#else
+    typedef int DBfile;
+#endif
+
+
+
+namespace silo {
+
+
+enum FileMode { READ, WRITE, CREATE };
+
+enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
+
+enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
+
+
+/*!
+ * @brief  Open silo file
+ * @detailed  This function opens a silo file
+ * @param[in] filename      File to open
+ * @param[in] mode          Open the file for reading or writing
+ * @return This function returns a handle to the file
+*/
+DBfile* open( const std::string& filename, FileMode mode );
+
+
+/*!
+ * @brief  Close silo file
+ * @detailed  This function closes a silo file
+ * @param[in] fid           Handle to the open file
+*/
+void close( DBfile* fid );
+
+
+/*!
+ * @brief  Get the variable type
+ * @detailed  This function returns the type of variable data
+ * @param[in] fid           Handle to the open file
+ * @param[in] name          Name of variable
+*/
+VariableDataType varDataType( DBfile *dbfile, const std::string& name );
+
+
+/*!
+ * @brief  Write data to silo
+ * @detailed  This function writes an arbitrary array to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Variable name
+ * @param[in] data          Data to write
+*/
+template<class TYPE>
+void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data );
+
+
+/*!
+ * @brief  Write data to silo
+ * @detailed  This function writes an arbitrary array to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Variable name
+ * @return                  Data read
+*/
+template<class TYPE>
+std::vector<TYPE> read( DBfile* fid, const std::string& varname );
+
+
+/*!
+ * @brief  Write a uniform grid
+ * @detailed  This function writes a uniform grid to silo as a Quadmesh
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] range         Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
+ * @param[in] N             Number of cells in each direction
+*/
+template<int NDIM>
+void writeUniformMesh( DBfile* fid, const std::string& meshname,
+    const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N );
+
+
+/*!
+ * @brief  Read a uniform grid
+ * @detailed  This function reads a uniform grid from silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[out] range         Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
+ * @param[out] N             Number of cells in each direction
+*/
+void readUniformMesh( DBfile* fid, const std::string& meshname,
+    std::vector<double>& range, std::vector<int>& N );
+
+
+/*!
+ * @brief  Write a uniform grid variable
+ * @detailed  This function writes a uniform grid variable to silo as a Quadmesh
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] N             Number of cells in each direction
+ * @param[in] varname       Variable name
+ * @param[in] data          Variable data
+ * @param[in] type          Variable type
+*/
+template< int NDIM, class TYPE >
+void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
+    const std::string& varname, const Array<TYPE>& data, VariableType type );
+
+
+/*!
+ * @brief  Read a uniform mesh grid variable
+ * @detailed  This function read a uniform mesh variable to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Variable name
+ * @return                  Variable data
+*/
+template<class TYPE>
+Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
+
+
+/*!
+ * @brief  Write a pointmesh
+ * @detailed  This function writes a pointmesh to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] ndim          Number of dimensions
+ * @param[in] N             Number of points
+ * @param[in] coords        Coordinates of the points
+*/
+template<class TYPE>
+void writePointMesh( DBfile* fid, const std::string& meshname,
+    int ndim, int N, const TYPE *coords[] );
+
+
+/*!
+ * @brief  Read a pointmesh
+ * @detailed  This function reads a pointmesh from silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @return                  Returns the coordinates as a N x ndim array 
+*/
+template<class TYPE>
+Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
+
+
+/*!
+ * @brief  Write a pointmesh grid variable
+ * @detailed  This function writes a pointmesh variable to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] varname       Variable name
+ * @param[in] data          Variable data
+*/
+template<class TYPE>
+void writePointMeshVariable( DBfile* fid, const std::string& meshname,
+    const std::string& varname, const Array<TYPE>& data );
+
+
+/*!
+ * @brief  Read a pointmesh grid variable
+ * @detailed  This function reads a pointmesh variable from silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Variable name
+ * @return                  Variable data
+*/
+template<class TYPE>
+Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
+
+
+/*!
+ * @brief  Write a triangle mesh
+ * @detailed  This function writes a triangle (or simplex) based mesh to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] ndim          Number of dimensions for the coordinates
+ * @param[in] ndim_tri      Number of dimensions for the triangles (2: surface, 3: volume)
+ * @param[in] N             Number of points
+ * @param[in] coords        Coordinates of the points
+ * @param[in] N_tri         Number of triangles
+ * @param[in] tri           Coordinates of the points
+*/
+template<class TYPE>
+void writeTriMesh( DBfile* fid, const std::string& meshname,
+    int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] );
+
+
+/*!
+ * @brief  Read a triangle mesh
+ * @detailed  This function reads a triangle (or simplex) based mesh to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] coords        Coordinates of the points
+ * @param[in] tri           Coordinates of the points
+*/
+template<class TYPE>
+void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri );
+
+
+/*!
+ * @brief  Write a triangle mesh grid variable
+ * @detailed  This function writes a triangle mesh variable to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] varname       Variable name
+ * @param[in] data          Variable data
+ * @param[in] type          Variable type
+*/
+template<class TYPE>
+void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
+    const std::string& varname, const Array<TYPE>& data, VariableType type );
+
+
+/*!
+ * @brief  Read a triangle mesh grid variable
+ * @detailed  This function read a triangle mesh variable to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Variable name
+ * @return                  Variable data
+*/
+template<class TYPE>
+Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
+
+
+/*!
+ * @brief  Write a multimesh
+ * @detailed  This function writes a multimesh to silo
+ * @param[in] fid           Handle to the open file
+ * @param[in] meshname      Mesh name
+ * @param[in] subMeshNames  Names of the sub meshes in the form "filename:meshname"
+ * @param[in] subMeshTypes  Type of each submesh
+*/
+void writeMultiMesh( DBfile* fid, const std::string& meshname,
+    const std::vector<std::string>& subMeshNames,
+    const std::vector<int>& subMeshTypes );
+
+
+/*!
+ * @brief  Write a multivariable
+ * @detailed  This function writes a multivariable to silo
+ * @return This function returns a handle to the file
+ * @param[in] fid           Handle to the open file
+ * @param[in] varname       Mesh name
+ * @param[in] subVarNames   Names of the sub meshes in the form "filename:meshname"
+ * @param[in] subVarTypes   Type of each submesh
+ * @param[in] ndim          Dimension of variable (used to determine suffix)
+ * @param[in] nvar          Number of subvariables (used to determine suffix)
+*/
+void writeMultiVar( DBfile* fid, const std::string& varname,
+    const std::vector<std::string>& subVarNames,
+    const std::vector<int>& subVarTypes );
+
+
+}; // silo namespace
+#endif
+
+#include "IO/silo.hpp"
+

IO/silo.hpp

@@ -0,0 +1,392 @@
+#ifndef SILO_INTERFACE_HPP
+#define SILO_INTERFACE_HPP
+
+#include "IO/silo.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+
+#include "ProfilerApp.h"
+
+
+#ifdef USE_SILO
+
+#include <silo.h>
+
+
+
+namespace silo {
+
+
+/****************************************************
+* Helper functions                                  *
+****************************************************/
+template<class TYPE> static constexpr int getType();
+template<> constexpr int getType<double>() { return DB_DOUBLE; }
+template<> constexpr int getType<float>() { return DB_FLOAT; }
+template<> constexpr int getType<int>() { return DB_INT; }
+template<class TYPE>
+inline void copyData( Array<TYPE>& data, int type, const void *src )
+{
+    if ( type == getType<TYPE>() )
+        memcpy( data.data(), src, data.length()*sizeof(TYPE) );
+    else if ( type == DB_DOUBLE )
+        data.copy( static_cast<const double*>(src) );
+    else if ( type == DB_FLOAT )
+        data.copy( static_cast<const float*>(src) );
+    else if ( type == DB_INT )
+        data.copy( static_cast<const int*>(src) );
+    else
+        ERROR("Unknown type");
+}
+
+
+/****************************************************
+* Write/read an arbitrary vector                    *
+****************************************************/
+template<class TYPE> constexpr int getSiloType();
+template<> constexpr int getSiloType<int>() { return DB_INT; }
+template<> constexpr int getSiloType<float>() { return DB_FLOAT; }
+template<> constexpr int getSiloType<double>() { return DB_DOUBLE; }
+template<class TYPE>
+void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data )
+{
+    int dims = data.size();
+    int err = DBWrite( fid, varname.c_str(), (void*) data.data(), &dims, 1, getSiloType<TYPE>() );
+    ASSERT( err == 0 );
+}
+template<class TYPE>
+std::vector<TYPE> read( DBfile* fid, const std::string& varname )
+{
+    int N = DBGetVarLength( fid, varname.c_str() );
+    std::vector<TYPE> data(N);
+    int err = DBReadVar( fid, varname.c_str(), data.data() );
+    ASSERT( err == 0 );
+    return data;
+}
+
+
+/****************************************************
+* Helper function to get variable suffixes          *
+****************************************************/
+inline std::vector<std::string> getVarSuffix( int ndim, int nvars )
+{
+    std::vector<std::string> suffix(nvars);
+    if ( nvars == 1 ) {
+        suffix[0] = "";
+    } else if ( nvars == ndim ) {
+        if ( ndim==2 ) {
+            suffix[0] = "_x";
+            suffix[1] = "_y";
+        } else if ( ndim==3 ) {
+            suffix[0] = "_x";
+            suffix[1] = "_y";
+            suffix[2] = "_z";
+        } else {
+            ERROR("Not finished");
+        }
+    } else if ( nvars == ndim*ndim ) {
+        if ( ndim==2 ) {
+            suffix[0] = "_xx";
+            suffix[1] = "_xy";
+            suffix[2] = "_yx";
+            suffix[3] = "_yy";
+        } else if ( ndim==3 ) {
+            suffix[0] = "_xx";
+            suffix[1] = "_xy";
+            suffix[2] = "_xz";
+            suffix[3] = "_yx";
+            suffix[4] = "_yy";
+            suffix[5] = "_yz";
+            suffix[6] = "_zx";
+            suffix[7] = "_zy";
+            suffix[8] = "_zz";
+        } else {
+            ERROR("Not finished");
+        }
+    } else {
+        for (int i=0; i<nvars; i++)
+            suffix[i] = "_" + std::to_string(i+1);
+    }
+    return suffix;
+}
+
+
+/****************************************************
+* Write/read a uniform mesh to silo                 *
+****************************************************/
+template<int NDIM>
+void writeUniformMesh( DBfile* fid, const std::string& meshname,
+    const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N )
+{
+    PROFILE_START("writeUniformMesh",2);
+    int dims[NDIM];
+    for (size_t d=0; d<N.size(); d++)
+        dims[d] = N[d]+1;
+    float *x = nullptr;
+    if ( NDIM >= 1 ) {
+        x = new float[dims[0]];
+        for (int i=0; i<N[0]; i++)
+            x[i] = range[0] + i*(range[1]-range[0])/N[0];
+        x[N[0]] = range[1];
+    }
+    float *y = nullptr;
+    if ( NDIM >= 2 ) {
+        y = new float[dims[1]];
+        for (int i=0; i<N[1]; i++)
+            y[i] = range[2] + i*(range[3]-range[2])/N[1];
+        y[N[1]] = range[3];
+    }
+    float *z = nullptr;
+    if ( NDIM >= 3 ) {
+        z = new float[dims[2]];
+        for (int i=0; i<N[2]; i++)
+            z[i] = range[4] + i*(range[5]-range[4])/N[2];
+        z[N[2]] = range[5];
+    }
+    float *coords[] = { x, y, z };
+    int err = DBPutQuadmesh( fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
+    ASSERT( err == 0 );
+    PROFILE_STOP("writeUniformMesh",2);
+}
+
+
+/****************************************************
+* Write a vector/tensor quad variable               *
+****************************************************/
+template<int NDIM,class TYPE>
+void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
+    const std::string& varname, const Array<TYPE>& data, VariableType type )
+{
+    PROFILE_START("writeUniformMeshVariable",2);
+    int nvars=1, dims[NDIM]={1};
+    const TYPE *vars[NDIM] = { nullptr };
+    int vartype = 0;
+    if ( type == VariableType::NodeVariable ) {
+        ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
+        for (int d=0; d<NDIM; d++)
+            ASSERT(N[d]+1==(int)data.size(d));
+        vartype = DB_NODECENT;
+        nvars = data.size(NDIM);
+        size_t N = data.length()/nvars;
+        for (int d=0; d<NDIM; d++)
+            dims[d] = data.size(d);
+        for (int i=0; i<nvars; i++)
+            vars[i] = &data(i*N);
+    } else if ( type == VariableType::EdgeVariable ) {
+        ERROR("Not finished");
+    } else if ( type == VariableType::SurfaceVariable ) {
+        ERROR("Not finished");
+    } else if ( type == VariableType::VolumeVariable ) {
+        ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
+        for (int d=0; d<NDIM; d++)
+            ASSERT(N[d]==(int)data.size(d));
+        vartype = DB_ZONECENT;
+        nvars = data.size(NDIM);
+        size_t N = data.length()/nvars;
+        for (int d=0; d<NDIM; d++)
+            dims[d] = data.size(d);
+        for (int i=0; i<nvars; i++)
+            vars[i] = &data(i*N);
+    } else {
+        ERROR("Invalid variable type");
+    }
+    auto suffix = getVarSuffix( NDIM, nvars );
+    std::vector<std::string> var_names(nvars);
+    for (int i=0; i<nvars; i++)
+        var_names[i] = varname + suffix[i];
+    std::vector<char*> varnames(nvars,nullptr);
+    for (int i=0; i<nvars; i++)
+        varnames[i] = const_cast<char*>(var_names[i].c_str());
+    int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars,
+        varnames.data(), vars, dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
+    ASSERT( err == 0 );
+    PROFILE_STOP("writeUniformMeshVariable",2);
+}
+template<class TYPE> 
+Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
+{
+    auto var = DBGetQuadvar( fid, varname.c_str() );
+    ASSERT( var != nullptr );
+    Array<TYPE> data( var->nels, var->nvals );
+    int type = var->datatype;
+    for (int i=0; i<var->nvals; i++) {
+        Array<TYPE> data2( var->nels );
+        copyData<TYPE>( data2, type, var->vals[i] );
+        memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
+    }
+    DBFreeQuadvar( var );
+    std::vector<size_t> dims( var->ndims+1, var->nvals );
+    for (int d=0; d<var->ndims; d++)
+        dims[d] = var->dims[d];
+    data.reshape( dims );
+    return data;
+}
+
+
+/****************************************************
+* Read/write a point mesh/variable to silo          *
+****************************************************/
+template<class TYPE>
+void writePointMesh( DBfile* fid, const std::string& meshname,
+    int ndim, int N, const TYPE *coords[] )
+{
+    int err = DBPutPointmesh( fid, meshname.c_str(), ndim, coords, N, getType<TYPE>(), nullptr );
+    ASSERT( err == 0 );
+}
+template<class TYPE> 
+Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname )
+{
+    auto mesh = DBGetPointmesh( fid, meshname.c_str() );
+    int N = mesh->nels;
+    int ndim = mesh->ndims;
+    Array<TYPE> coords(N,ndim);
+    int type = mesh->datatype;
+    for (int d=0; d<ndim; d++) {
+        Array<TYPE> data2( N );
+        copyData<TYPE>( data2, type, mesh->coords[d] );
+        memcpy( &coords(0,d), data2.data(), N*sizeof(TYPE) );
+    }
+    DBFreePointmesh( mesh );
+    return coords;
+}
+template<class TYPE>
+void writePointMeshVariable( DBfile* fid, const std::string& meshname,
+    const std::string& varname, const Array<TYPE>& data )
+{
+    int N = data.size(0);
+    int nvars = data.size(1);
+    std::vector<const TYPE*> vars(nvars);
+    for (int i=0; i<nvars; i++)
+        vars[i] = &data(0,i);
+    int err = DBPutPointvar( fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
+    ASSERT( err == 0 );
+}
+template<class TYPE> 
+Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
+{
+    auto var = DBGetPointvar( fid, varname.c_str() );
+    ASSERT( var != nullptr );
+    Array<TYPE> data( var->nels, var->nvals );
+    int type = var->datatype;
+    for (int i=0; i<var->nvals; i++) {
+        Array<TYPE> data2( var->nels );
+        copyData<TYPE>( data2, type, var->vals[i] );
+        memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
+    }
+    DBFreeMeshvar( var );
+    return data;
+}
+
+
+/****************************************************
+* Read/write a triangle mesh                        *
+****************************************************/
+template<class TYPE>
+void writeTriMesh( DBfile* fid, const std::string& meshName,
+    int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] )
+{
+    auto zoneName = meshName + "_zones";
+    std::vector<int> nodelist( (ndim_tri+1)*N_tri );
+    for (int i=0, j=0; i<N_tri; i++) {
+        for (int d=0; d<ndim_tri+1; d++, j++)
+            nodelist[j] = tri[d][i];
+    }
+    int shapetype = 0;
+    if ( ndim_tri==1 )
+        shapetype = DB_ZONETYPE_BEAM;
+    else if ( ndim_tri==2 )
+        shapetype = DB_ZONETYPE_TRIANGLE;
+    else if ( ndim_tri==3 )
+        shapetype = DB_ZONETYPE_PYRAMID;
+    else
+        ERROR("Unknown shapetype");
+    int shapesize = ndim_tri+1;
+    int shapecnt = N_tri;
+    DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(),
+        nodelist.size(), 0, 0, 0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
+    DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N,
+        nodelist.size(), zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
+}
+template<class TYPE>
+void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri )
+{
+    auto mesh = DBGetUcdmesh( fid, meshname.c_str() );
+    int ndim = mesh->ndims;
+    int N_nodes = mesh->nnodes;
+    coords.resize(N_nodes,ndim);
+    int mesh_type = mesh->datatype;
+    for (int d=0; d<ndim; d++) {
+        Array<TYPE> data2( N_nodes );
+        copyData<TYPE>( data2, mesh_type, mesh->coords[d] );
+        memcpy( &coords(0,d), data2.data(), N_nodes*sizeof(TYPE) );
+    }
+    auto zones = mesh->zones;
+    int N_zones = zones->nzones;
+    int ndim_zones = zones->ndims;
+    ASSERT( zones->nshapes==1 );
+    int shape_type = zones->shapetype[0];
+    int shapesize = zones->shapesize[0];
+    tri.resize(N_zones,shapesize);
+    for (int i=0; i<N_zones; i++) {
+        for (int j=0; j<shapesize; j++)
+            tri(i,j) = zones->nodelist[i*shapesize+j];
+    }
+    DBFreeUcdmesh( mesh );
+}
+template<class TYPE>
+void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
+    const std::string& varname, const Array<TYPE>& data, VariableType type )
+{
+    int nvars = 0;
+    int vartype = 0;
+    const TYPE *vars[10] = { nullptr };
+    if ( type == VariableType::NodeVariable ) {
+        vartype = DB_NODECENT;
+        nvars = data.size(1);
+        for (int i=0; i<nvars; i++)
+            vars[i] = &data(0,i);
+    } else if ( type == VariableType::EdgeVariable ) {
+        ERROR("Not finished");
+    } else if ( type == VariableType::SurfaceVariable ) {
+        ERROR("Not finished");
+    } else if ( type == VariableType::VolumeVariable ) {
+        vartype = DB_ZONECENT;
+        nvars = data.size(1);
+        for (int i=0; i<nvars; i++)
+            vars[i] = &data(0,i);
+    } else {
+        ERROR("Invalid variable type");
+    }
+    auto suffix = getVarSuffix( ndim, nvars );
+    std::vector<std::string> var_names(nvars);
+    for (int i=0; i<nvars; i++)
+        var_names[i] = varname + suffix[i];
+    std::vector<char*> varnames(nvars,nullptr);
+    for (int i=0; i<nvars; i++)
+        varnames[i] = const_cast<char*>(var_names[i].c_str());
+    DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars,
+        varnames.data(), vars, data.size(0), nullptr, 0, getType<TYPE>(), vartype, nullptr );
+}
+template<class TYPE>
+Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname )
+{
+    auto var = DBGetUcdvar( fid, varname.c_str() );
+    ASSERT( var != nullptr );
+    Array<TYPE> data( var->nels, var->nvals );
+    int type = var->datatype;
+    for (int i=0; i<var->nvals; i++) {
+        Array<TYPE> data2( var->nels );
+        copyData<TYPE>( data2, type, var->vals[i] );
+        memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
+    }
+    DBFreeUcdvar( var );
+    return data;
+}
+
+
+}; // silo namespace
+
+
+#endif
+
+#endif

LICENSE.bsd2

@@ -0,0 +1,23 @@
+Copyright (c) 2015, JamesEMcClure
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,31 @@
+LBPM-WIA
+========
+
+Lattice Boltzmann Methods for Porous Media with Integrated Averaging
+
+========
+
+Notes on installation
+
+* example configure scripts for cmake are in the sample_scripts directory
+* required dependencies - MPI, C++ 11
+* optional dependencies - NetCDF, CUDA, TimerUtility, VisIT
+
+Configure, build & install procedure
+* create an empty directory to install (do not build in source!)
+
+   mkdir /path/to/my/install
+
+   cd /path/to/my/install
+
+* edit configure script from sample_scripts directory and configure (e.g.)
+
+   /path/to/LBPM-WIA/sample_scripts/configure_desktop 
+
+* compile and install
+
+   make -j4 && make install
+
+* run the unit tests to make sure it works
+   ctest
+

README.titan

@@ -0,0 +1,53 @@
+# INSTRUCTIONS FOR COMPILING AND RUNNING ON TITAN
+# Note: there are 3 directories of interest:
+#    source directory - assumed to be located at ${HOME}/LBPM-WIA for these instructions, but may be anywhere
+#    build directory  - current directory used for building, may not be the source directory, generally should be on lustre
+#    install directory - installation directory, optional, may be the same as the build directory, 
+#                        generally should be on lustre, specified with "-D LBPM_INSTALL_DIR:PATH="path_tot_install"
+
+
+# load the module for cuda
+module load cudatoolkit
+
+
+# load the module for cmake
+module load cmake
+
+
+# Change to a directory build (outside the repository)
+cd BUILD_DIR
+
+
+# configure
+rm -rf CMake*
+cmake                                    \
+    -D CMAKE_C_COMPILER:PATH=cc          \
+    -D CMAKE_CXX_COMPILER:PATH=CC        \
+    -D CMAKE_CXX_COMPILER:PATH=CC        \
+    -D CFLAGS="-DCBUB"                   \
+    -D CXXFLAGS="-DCBUB"                 \
+    -D MPI_COMPILER:BOOL=TRUE            \
+    -D MPIEXEC=aprun                     \
+    -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
+    -D CMAKE_BUILD_TYPE:STRING=Debug     \
+    -D CUDA_FLAGS="-arch sm_35"          \
+    -D CUDA_HOST_COMPILER="/usr/bin/gcc" \
+    -D USE_CUDA=1                        \
+    ${HOME}/LBPM-WIA
+
+
+# Build and install (may be done in parallel)
+make install -j 8
+
+
+# Run the tests (optional, requires that were are in an interactive session)
+# Note: to run the tests and submit the results to CDash, see "run_ctest_titan"
+#    in sample_scripts, which uses the "config_titan" and "ctest_titan.ctest"
+# Run the fast tests:
+ctest -E WEEKLY
+# Run all tests
+ctest
+
+
+
+

README.visit

@@ -0,0 +1,64 @@
+To compile the visit plug-in and run in parallel on computational back-end:
+
+Checkout code and create a cmake script to compile plug-in.
+Example commands for Rhea:
+
+Load the proper modules
+   module unload intel gcc
+   module load gcc/4.8.2
+   module load cmake
+
+
+Configure cmake to build the plug-in
+    cmake                                       \
+        -D CMAKE_BUILD_TYPE:STRING=Release      \
+        -D CMAKE_C_COMPILER:PATH=gcc            \
+        -D CMAKE_CXX_COMPILER:PATH=g++          \
+        -D USE_MPI=false                        \
+        -D USE_CUDA=false                       \
+        -D USE_VISIT=true                       \
+            -D VISIT_ROOT_DIR=/sw/redhat6/visit \
+        -D USE_TIMER=false                      \
+        ../../LBPM-WIA
+
+
+Build the visit plug-in
+    make install -j 8
+    make visit
+
+Check that the visit plug-in installed in ~/.visit/2.9.0/linux-x86_64/plugins/databases
+where 2.9.0 is the current version of visit on rhea
+It should have created several shared libraries:
+    libELBMDatabase_par.so  libELBMDatabase_ser.so  libILBMDatabase.so  libMLBMDatabase.so
+
+Install visit on local machine (2.9.0)
+
+Start visit on local machine
+
+First time:
+    Go to: Options - Host profiles
+    Click Remote Profiles - Update
+    After list populates, drag host_ornl_rhea.xml to Hosts
+    Select ORNL_Rhea from Hosts, click on Machines
+    Set account username
+    Click on Launch Profiles tab
+    Change profile name and timeout if desired
+    Click on Parallel tab
+    Add Default Bank / Account with project number, set default nodes/processors if desired
+    Click Apply, then close window
+    Go to: Options - Save Settings
+
+Open remote file / launch parallel job
+    Click Open file
+    Change Host to ORNL_Rhea, enter passcode when prompted
+    Open visualization file
+        /lustre/atlas/proj-shared/geo106/vis_test/dumps.LBM
+    New window will open prompting for # of nodes, processors, Bank (project number) and time limit
+        Rhea as 16 physical-cores/node and 64 GB/node
+        for example file above I used 1 node, 16 cores
+    Visit will call qsub and submit a parallel job that will start when ready
+
+Visualize desired quantities
+
+
+

ValgrindSuppresionFile

@@ -0,0 +1,52 @@
+# ACML suppressions
+{
+   IdentifyCPUCond
+   Memcheck:Cond
+   ...
+   fun:acmlcpuid2
+   ...
+}
+{
+   IdentifyCPUValue
+   Memcheck:Value8
+   ...
+   fun:acmlcpuid_once
+   fun:acmlcpuid2
+   ...
+}
+
+
+# MPI suppressions
+{
+   HYD_pmci_wait_for_completion
+   Memcheck:Leak
+   ...
+   fun:HYD_pmci_wait_for_completion
+   fun:main
+}
+{
+   HYDT_dmxu_poll_wait_for_event
+   Memcheck:Leak
+   ...
+   fun:HYDT_dmxu_poll_wait_for_event
+   fun:main
+}
+{
+   PMPI_Init
+   Memcheck:Leak
+   ...
+   fun:PMPI_Init
+   fun:main
+}
+
+
+# System suppressions
+{
+   expand_dynamic_string_token
+   Memcheck:Cond
+   fun:index
+   fun:expand_dynamic_string_token
+   ...
+}
+
+

analysis/PointList.h

@@ -0,0 +1,178 @@
+#ifndef PointList_INC
+#define PointList_INC
+
+#include <math.h>
+
+struct LBPM_Point {
+    LBPM_Point() : x(0.0), y(0.0), z(0.0) {}
+    LBPM_Point(double xv,double yv,double zv) : x(xv), y(yv), z(zv) {}
+    LBPM_Point(const LBPM_Point& rhs): x(rhs.x), y(rhs.y), z(rhs.z) {}
+    //Point& operator=(const Point& rhs) { this->x=rhs.x; this->y=rhs.y; this->z=rhs.z; return *this; }
+    //~Point() {}
+    double x,y,z;
+};
+typedef LBPM_Point Point;
+
+inline Point operator+(const Point &A,const Point &B) {return Point(A.x+B.x,A.y+B.y,A.z+B.z);}
+inline Point operator-(const Point &A,const Point &B) {return Point(A.x-B.x,A.y-B.y,A.z-B.z);}
+inline Point operator*(const Point &A,double v) {return Point(A.x*v,A.y*v,A.z*v);}
+inline Point operator*(double v,const Point &A) {return Point(A.x*v,A.y*v,A.z*v);}
+inline Point operator/(const Point &A,double v) {return Point(A.x/v,A.y/v,A.z/v);}
+inline Point operator-(const Point &A) {return Point(-A.x,-A.y,-A.z);}
+
+inline bool operator==(const Point &A,const Point &B) {return (A.x==B.x && A.y==B.y && A.z==B.z);}
+inline bool operator!=(const Point &A,const Point &B) {return (A.x!=B.x || A.y!=B.y || A.z!=B.z);}
+
+inline double Norm(const Point &A) {return sqrt(A.x*A.x+A.y*A.y+A.z*A.z);}
+inline Point Cross(const Point &A,const Point &B) {return Point(A.y*B.z-A.z*B.y,B.x*A.z-A.x*B.z,A.x*B.y-A.y*B.x);}
+inline double Dot(const Point &A,const Point &B) {return (A.x*B.x+A.y*B.y+A.z*B.z);}
+inline double Distance(const Point &A,const Point &B) {return sqrt((A.x-B.x)*(A.x-B.x)+(A.y-B.y)*(A.y-B.y)+(A.z-B.z)*(A.z-B.z));}
+
+/*
+class  PointList{
+public:
+	int length;
+	int m;
+	int index;
+
+	Point pt;
+
+	double *data;
+	PointList();
+	PointList(int size);
+	~PointList();
+
+	void New(int size);
+
+	Point & operator()(int idx)
+	{
+		index = idx;
+		pt.x = data[3*index];
+		pt.y = data[3*index+1];
+		pt.z = data[3*index+2];
+		return pt;
+	}
+	
+	Point &operator=(Point &P){
+		pt.x = P.x;
+		pt.y = P.y;
+		pt.z = P.z;
+		data[3*index]=pt.x;
+		data[3*index+1]=pt.y;
+		data[3*index+2]=pt.z;
+	}
+};
+
+// *****************************************
+// ******** class PointList **************
+// *****************************************
+PointList::PointList()
+{
+	m=length=0;
+}
+
+PointList::PointList(int size)
+{
+	m=size;
+	data = new double [3*size];
+	length = size;
+}
+
+void PointList::New(int size)
+{
+	m=size;
+	data = new double [3*size];
+	length = size;
+}
+
+
+PointList::~PointList()
+{
+	delete data;
+}
+*/
+template <class T>
+class DTList {
+public:
+    DTList() : Data(0), length(0), refCount(new size_t(1)), outOfRange() {}
+    DTList(const DTList<T> &A) : Data(A.Data), length(A.length), refCount(A.refCount), outOfRange() {++(*refCount);}
+protected:
+    DTList(size_t len) : Data(len<=0 ? 0 : new T[len]), length(len<=0 ? 0 : len), refCount(new size_t(1)), outOfRange() {}
+public:
+    
+    virtual ~DTList() {
+      --(*refCount);
+      if (*refCount==0) {delete [] Data; delete refCount;}
+      Data = 0; refCount = 0; length=0;
+    }
+	
+    DTList<T> &operator=(const DTList<T> &A) {
+        if (A.refCount!=refCount) { // Otherwise doing A=A.
+            --(*refCount);
+            if (*refCount==0) {delete [] Data; delete refCount;}
+            refCount = A.refCount;
+            ++(*refCount);
+            length = A.length;
+            Data = A.Data;
+        }
+        return *this;
+    }
+	
+    size_t MemoryUsed(void) const {return length*sizeof(T);}
+	
+    const T *Pointer(void) const {return Data;}
+	
+    size_t IsEmpty(void) const {return (Data==0);}
+    size_t Length(void) const {return length;}
+
+    const T operator()(size_t i) const  {return Data[i];}
+	
+protected:
+    T *Data;
+    size_t length;
+    size_t *refCount;
+	
+    // Should be static.
+    T outOfRange;
+};
+
+template <class T>
+class DTMutableList : public DTList<T> {
+public:
+    DTMutableList() : DTList<T>() {}
+    DTMutableList(size_t len) : DTList<T>(len) {}
+    DTMutableList(const DTMutableList<T> &A) : DTList<T>(A) {}
+	
+    DTMutableList<T> &operator=(const DTMutableList<T> &A) {DTList<T>::operator=(A); return *this;}
+	
+    T *Pointer(void) {return DTList<T>::Data;}
+    const T *Pointer(void) const {return DTList<T>::Data;}
+    T &operator()(size_t i) {return DTList<T>::Data[i];}
+    T operator()(size_t i) const  {return DTList<T>::Data[i];}
+	
+    DTMutableList<T> &operator=(T v) {for (size_t i=0;i<DTList<T>::length;i++) DTList<T>::Data[i] = v; return *this;}
+};
+
+template <class T> DTMutableList<T> TruncateSize(const DTList<T> &A,size_t length)
+{
+    if (length>A.Length()) length = A.Length();
+    DTMutableList<T> toReturn(length);
+    const T *fromP = A.Pointer();
+    T *toP = toReturn.Pointer();
+    for (size_t i=0;i<length;i++) toP[i] = fromP[i];
+    return toReturn;
+}
+
+template <class T> DTMutableList<T> IncreaseSize(const DTList<T> &A,size_t addLength)
+{
+    DTMutableList<T> toReturn(A.Length()+(addLength>=0 ? addLength : 0));
+    size_t len = A.Length();
+    const T *fromP = A.Pointer();
+    T *toP = toReturn.Pointer();
+    for (size_t i=0;i<len;i++) toP[i] = fromP[i];
+    return toReturn;
+}
+
+#endif
+
+

analysis/TwoPhase.h

@@ -0,0 +1,170 @@
+// Header file for two-phase averaging class
+#ifndef TwoPhase_INC
+#define TwoPhase_INC
+
+#include <vector>
+
+#include "analysis/pmmc.h"
+#include "common/Domain.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+
+#include "shared_ptr.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+
+
+class TwoPhase{
+
+	//...........................................................................
+	int n_nw_pts,n_ns_pts,n_ws_pts,n_nws_pts,n_local_sol_pts,n_local_nws_pts;
+	int n_nw_tris,n_ns_tris,n_ws_tris,n_nws_seg,n_local_sol_tris;
+	//...........................................................................
+	int nc;
+	int kstart,kfinish;
+
+	double fluid_isovalue, solid_isovalue;
+	double Volume;
+	// initialize lists for vertices for surfaces, common line
+	DTMutableList<Point> nw_pts;
+	DTMutableList<Point> ns_pts;
+	DTMutableList<Point> ws_pts;
+	DTMutableList<Point> nws_pts;
+	DTMutableList<Point> local_sol_pts;
+	DTMutableList<Point> local_nws_pts;
+	DTMutableList<Point> tmp;
+
+	// initialize triangle lists for surfaces
+	IntArray nw_tris;
+	IntArray ns_tris;
+	IntArray ws_tris;
+	IntArray nws_seg;
+	IntArray local_sol_tris;
+
+	// Temporary storage arrays
+	DoubleArray CubeValues;
+	DoubleArray Values;
+	DoubleArray DistanceValues;
+	DoubleArray KGwns_values;
+	DoubleArray KNwns_values;
+	DoubleArray InterfaceSpeed;
+	DoubleArray NormalVector;
+
+	DoubleArray RecvBuffer;
+
+	char *TempID;
+
+	// CSV / text file where time history of averages is saved
+	FILE *TIMELOG;
+	FILE *NWPLOG;
+	FILE *WPLOG;
+
+public:
+	//...........................................................................
+	Domain& Dm;
+	int NumberComponents_WP,NumberComponents_NWP;
+	//...........................................................................
+	// Averaging variables
+	//...........................................................................
+	// local averages (to each MPI process)
+	double trimdist; 						// pixel distance to trim surface for specified averages
+	double porosity,poreVol;
+	double awn,ans,aws,lwns;
+	double wp_volume,nwp_volume;
+	double As, dummy;
+	double vol_w, vol_n;						// volumes the exclude the interfacial region
+	double sat_w, sat_w_previous;
+	double pan,paw;								// local phase averaged pressure
+	// Global averages (all processes)
+	double pan_global,paw_global;				// local phase averaged pressure
+	double vol_w_global, vol_n_global;			// volumes the exclude the interfacial region
+	double awn_global,ans_global,aws_global;
+	double lwns_global;
+	double efawns,efawns_global;				// averaged contact angle
+	double euler,Kn,Jn,An;
+	double euler_global,Kn_global,Jn_global,An_global;
+
+	double Jwn,Jwn_global;						// average mean curavture - wn interface
+	double Kwn,Kwn_global;						// average Gaussian curavture - wn interface
+	double KNwns,KNwns_global;					// wns common curve normal curavture
+	double KGwns,KGwns_global;					// wns common curve geodesic curavture
+	double trawn,trawn_global;					// trimmed interfacial area
+	double trJwn,trJwn_global;					// trimmed interfacial area
+	double trRwn,trRwn_global;					// trimmed interfacial area
+	double nwp_volume_global;					// volume for the non-wetting phase
+	double wp_volume_global;					// volume for the wetting phase
+	double As_global;
+	double wwndnw, wwndnw_global;
+	double wwnsdnwn, wwnsdnwn_global;
+	double Jwnwwndnw, Jwnwwndnw_global;
+	double dEs,dAwn,dAns;						// Global surface energy (calculated by rank=0)
+	DoubleArray van;
+	DoubleArray vaw;
+	DoubleArray vawn;
+	DoubleArray vawns;
+	DoubleArray Gwn;
+	DoubleArray Gns;
+	DoubleArray Gws;
+	DoubleArray van_global;
+	DoubleArray vaw_global;
+	DoubleArray vawn_global;
+	DoubleArray vawns_global;
+	DoubleArray Gwn_global;
+	DoubleArray Gns_global;
+	DoubleArray Gws_global;
+	//...........................................................................
+	//...........................................................................
+        int Nx,Ny,Nz;
+	IntArray PhaseID;	// Phase ID array (solid=0, non-wetting=1, wetting=2)
+	BlobIDArray Label_WP;   // Wetting phase label
+	BlobIDArray Label_NWP;  // Non-wetting phase label index (0:nblobs-1)
+	std::vector<BlobIDType> Label_NWP_map;  // Non-wetting phase label for each index
+	DoubleArray SDn;
+	DoubleArray SDs;
+	DoubleArray Phase;
+	DoubleArray Press;
+	DoubleArray dPdt;
+	DoubleArray MeanCurvature;
+	DoubleArray GaussCurvature;
+	DoubleArray SDs_x;		// Gradient of the signed distance
+	DoubleArray SDs_y;
+	DoubleArray SDs_z;
+	DoubleArray SDn_x;		// Gradient of the signed distance
+	DoubleArray SDn_y;
+	DoubleArray SDn_z;
+	DoubleArray DelPhi;		// Magnitude of Gradient of the phase indicator field
+	DoubleArray Phase_tplus;
+	DoubleArray Phase_tminus;
+	DoubleArray Vel_x;		// Velocity
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+	//	Container for averages;
+	DoubleArray ComponentAverages_WP;
+	DoubleArray ComponentAverages_NWP;
+	//...........................................................................
+	TwoPhase(Domain &dm);
+	~TwoPhase();
+	void Initialize();
+//	void SetupCubes(Domain &Dm);
+	void UpdateMeshValues();
+	void UpdateSolid();
+	void ComputeDelPhi();
+	void ColorToSignedDistance(double Beta, DoubleArray &ColorData, DoubleArray &DistData);
+	void ComputeLocal();
+	void AssignComponentLabels();
+	void ComponentAverages();
+	void Reduce();
+	void WriteSurfaces(int logcount);
+	void NonDimensionalize(double D, double viscosity, double IFT);
+	void PrintAll(int timestep);
+	int GetCubeLabel(int i, int j, int k, IntArray &BlobLabel);
+	void SortBlobs();
+	void PrintComponents(int timestep);
+};
+
+
+#endif
+

analysis/analysis.cpp

@@ -0,0 +1,833 @@
+#include "analysis/analysis.h"
+#include "ProfilerApp.h"
+#include <iostream>
+
+
+template<class TYPE>
+inline TYPE* getPtr( std::vector<TYPE>& x ) { return x.empty() ? NULL:&x[0]; }
+template<class TYPE>
+inline const TYPE* getPtr( const std::vector<TYPE>& x ) { return x.empty() ? NULL:&x[0]; }
+
+
+/******************************************************************
+* Compute the blobs                                               *
+******************************************************************/
+int ComputeBlob( const Array<bool>& isPhase, BlobIDArray& LocalBlobID, bool periodic, int start_id )
+{
+    PROFILE_START("ComputeBlob",1);
+    ASSERT(isPhase.size()==LocalBlobID.size());
+    const int Nx = isPhase.size(0);  // maxima for the meshes
+    const int Ny = isPhase.size(1);
+    const int Nz = isPhase.size(2);
+    std::vector<int> map;
+    map.reserve(128);
+    // Get the list of neighbors we need to check
+    int N_neighbors = 0;
+    int d[26][3];
+    bool include_corners = false;   // Do we need to include cells that only touch at a corder/edge
+    if ( include_corners ) {
+        // Include corners/edges as neighbors, check all cells
+        N_neighbors = 26;
+        const int tmp[26][3] = {{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
+                                {1,1,0},{1,-1,0},{-1,1,0},{-1,-1,0},{1,0,1},{-1,0,1},
+                                {1,0,-1},{-1,0,-1},{0,1,1},{0,-1,1},{0,1,-1},{0,-1,-1},
+                                {1,1,1},{1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1},{-1,1,-1},
+                                {-1,-1,1},{-1,-1,-1}};   // directions to neighbors
+        memcpy(d,tmp,sizeof(tmp));
+    } else {
+        // Do not include corners/edges as neighbors
+        if ( periodic ) {
+            // Include all neighbors for periodic problems
+            N_neighbors = 6;
+            const int tmp[6][3] = {{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1}};   // directions to neighbors
+            memcpy(d,tmp,sizeof(tmp));
+        } else {
+            // We only need to include the lower neighbors for non-periodic problems
+            N_neighbors = 3;
+            const int tmp[3][3] = {{-1,0,0},{0,-1,0},{0,0,-1}};   // directions to neighbors
+            memcpy(d,tmp,sizeof(tmp));
+        }
+    }
+    // Loop through all the points
+    int last = start_id-1;
+    std::vector<int> neighbor_ids;
+    neighbor_ids.reserve(N_neighbors);
+    const bool *isPhasePtr = isPhase.data();
+    BlobIDType *LocalBlobIDPtr = LocalBlobID.data();
+    for (int z=0; z<Nz; z++) {
+        for (int y=0; y<Ny; y++) {
+            for (int x=0; x<Nx; x++) {
+                int index = x + y*Nx + z*Nx*Ny;
+                if ( !isPhasePtr[index] )
+                    continue;
+                // Get all neighbor indicies
+                int N_list=0, neighbor_ids[26];
+                for (int p=0; p<N_neighbors; p++) {
+                    // Get the neighbor index
+                    int x2 = x+d[p][0];
+                    int y2 = y+d[p][1];
+                    int z2 = z+d[p][2];
+                    if ( periodic ) {
+                        x2 = x2<0 ? Nx-1:x2;         // Periodic BC for x
+                        x2 = x2>Nx-1 ? 0:x2;
+                        y2 = y2<0 ? Ny-1:y2;         // Periodic BC for x
+                        y2 = y2>Ny-1 ? 0:y2;
+                        z2 = z2<0 ? Nz-1:z2;         // Periodic BC for x
+                        z2 = z2>Nz-1 ? 0:z2;
+                    } else {
+                        if ( x2<0 || x2>=Nx || y2<0 || y2>=Ny || z2<0 || z2>=Nz )
+                            continue;
+                    }
+                    // Check if a neighbor has a known blob id
+                    size_t index2 = x2 + y2*Nx + z2*Nx*Ny;
+                    int id = LocalBlobIDPtr[index2];
+                    if ( !isPhasePtr[index2] || id<0 )
+                        continue;
+                    neighbor_ids[N_list] = id;
+                    N_list++;
+                }
+                if ( N_list==0 ) {
+                    // No neighbors with a blob id, create a new one
+                    LocalBlobIDPtr[index] = last+1;
+                    map.push_back(last+1);
+                    last++;
+                } else if ( N_list==1 ) {
+                    // We have one neighbor
+                    LocalBlobIDPtr[index] = neighbor_ids[0];
+                } else {
+                    // We have multiple neighbors
+                    int id = neighbor_ids[0];
+                    for (int i=1; i<N_list; i++)
+                        id = std::min(id,neighbor_ids[i]);
+                    LocalBlobIDPtr[index] = id;
+                    for (int i=0; i<N_list; i++)
+                        map[neighbor_ids[i]-start_id] = std::min(map[neighbor_ids[i]-start_id],id);
+                }
+            }
+        }
+    }
+    // Collapse the ids that map to another id
+    last = start_id-1;
+    for (int i=0; i<(int)map.size(); i++) {
+        if ( map[i] == i+start_id ) {
+            map[i] = last+1;
+            last++;
+        } else {
+            ASSERT(map[i]<i+start_id);
+            map[i] = map[map[i]-start_id];
+        }
+    }
+    for (int i=0; i<Nx*Ny*Nz; i++) {
+        if ( isPhasePtr[i] ) {
+            LocalBlobIDPtr[i] = map[LocalBlobIDPtr[i]-start_id];
+        }
+    }
+    PROFILE_STOP("ComputeBlob",1);
+    return last-start_id+1;
+}
+
+
+/******************************************************************
+* Compute the local blob ids                                      *
+******************************************************************/
+int ComputeLocalBlobIDs( const DoubleArray& Phase, const DoubleArray& SignDist, 
+    double vF, double vS, BlobIDArray& LocalBlobID, bool periodic )
+{
+    PROFILE_START("ComputeLocalBlobIDs");
+    ASSERT(SignDist.size()==Phase.size());
+    size_t Nx = Phase.size(0);
+    size_t Ny = Phase.size(1);
+    size_t Nz = Phase.size(2);
+    // Initialize output
+    LocalBlobID.resize(Nx,Ny,Nz);
+    // Compute the local blob ids
+    size_t N = Nx*Ny*Nz;
+    Array<bool> isPhase(Nx,Ny,Nz);
+    memset(isPhase.data(),0,Nx*Ny*Nz*sizeof(bool));
+    for (size_t i=0; i<N; i++) {
+        if ( SignDist(i) <= vS) {
+            // Solid phase 
+            LocalBlobID(i) = -2;
+        } else {
+            LocalBlobID(i) = -1;
+            if ( Phase(i)>vF && SignDist(i)>vS )
+                isPhase(i) = true;
+        }
+    }
+    int nblobs = ComputeBlob( isPhase, LocalBlobID, periodic, 0 );
+    PROFILE_STOP("ComputeLocalBlobIDs");
+    return nblobs;
+}
+int ComputeLocalPhaseComponent(const IntArray &PhaseID, int &VALUE, BlobIDArray &ComponentLabel, bool periodic )
+{
+    PROFILE_START("ComputeLocalPhaseComponent");
+    size_t Nx = PhaseID.size(0);
+    size_t Ny = PhaseID.size(1);
+    size_t Nz = PhaseID.size(2);
+    size_t N = Nx*Ny*Nz;
+    // Compute the local blob ids
+    ComponentLabel.resize(Nx,Ny,Nz);
+    Array<bool> isPhase(Nx,Ny,Nz);
+    for (size_t i=0; i<N; i++) {
+        if ( PhaseID(i) == VALUE) {
+            ComponentLabel(i) = -1;
+            isPhase(i) = true;
+        } else{
+            ComponentLabel(i) = -2;
+            isPhase(i) = false;
+        }
+    }
+    int ncomponents = ComputeBlob( isPhase, ComponentLabel, periodic, 0 );
+    PROFILE_STOP("ComputeLocalPhaseComponent");
+    return ncomponents;
+}
+
+
+/******************************************************************
+* Reorder the global blob ids                                     *
+******************************************************************/
+static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, MPI_Comm comm )
+{
+    if ( N_blobs==0 )
+        return 0;
+    PROFILE_START("ReorderBlobIDs2",1);
+    ASSERT(sizeof(long long int)==sizeof(int64_t));
+    double *local_size = new double[N_blobs];
+    double *global_size = new double[N_blobs];
+    for (int i=0; i<N_blobs; i++)
+        local_size[i] = 0;
+    for (int i=0; i<N_blobs; i++)
+        global_size[i] = 0;
+    int max_id = -1;
+    for (size_t k=ngz; k<ID.size(2)-ngz; k++) {
+        for (size_t j=ngy; j<ID.size(1)-ngy; j++) {
+            for (size_t i=ngx; i<ID.size(0)-ngx; i++) {
+                int id = ID(i,j,k);
+                if ( id >= 0 ) 
+                    local_size[id] += 1;
+                max_id = std::max(max_id,id);
+            }
+        }
+    }
+    ASSERT(max_id<N_blobs);
+    MPI_Allreduce(local_size,global_size,N_blobs,MPI_DOUBLE,MPI_SUM,comm);
+    std::vector<std::pair<double,int> > map1(N_blobs);
+    int N_blobs2 = 0;
+    for (int i=0; i<N_blobs; i++) {
+        map1[i].first = global_size[i];
+        map1[i].second = i;
+        if ( global_size[i] > 0 )
+            N_blobs2++;
+    }
+    std::sort( map1.begin(), map1.end() );
+    std::vector<int> map2(N_blobs,-1);
+    for (int i=0; i<N_blobs; i++) {
+        map2[map1[N_blobs-i-1].second] = i;
+    }
+    for (size_t i=0; i<ID.length(); i++) {
+        if ( ID(i) >= 0 )
+            ID(i) = map2[ID(i)];
+    }
+    delete [] local_size;
+    delete [] global_size;
+    PROFILE_STOP("ReorderBlobIDs2",1);
+    return N_blobs2;
+}
+void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm )
+{
+    PROFILE_START("ReorderBlobIDs");
+    int tmp = ID.max()+1;
+    int N_blobs = 0;
+    MPI_Allreduce(&tmp,&N_blobs,1,MPI_INT,MPI_MAX,comm);
+    ReorderBlobIDs2(ID,N_blobs,1,1,1,comm);
+    PROFILE_STOP("ReorderBlobIDs");
+}
+
+
+/******************************************************************
+* Compute the global blob ids                                     *
+******************************************************************/
+struct global_id_info_struct {
+    int64_t new_id;
+    std::set<int64_t> remote_ids;
+};
+// Send the local ids and their new value to all neighbors
+static void updateRemoteIds( 
+    const std::map<int64_t,global_id_info_struct>& map,
+    const std::vector<int>& neighbors,
+    int N_send, const std::vector<int>& N_recv,
+    int64_t *send_buf, std::vector<int64_t*>& recv_buf,
+    std::map<int64_t,int64_t>& remote_map,
+    MPI_Comm comm )
+{
+    std::vector<MPI_Request> send_req(neighbors.size());
+    std::vector<MPI_Request> recv_req(neighbors.size());
+    std::vector<MPI_Status> status(neighbors.size());
+    std::map<int64_t,global_id_info_struct>::const_iterator it = map.begin();
+    ASSERT(N_send==(int)map.size());
+    for (size_t i=0; i<map.size(); i++, ++it) {
+        send_buf[2*i+0] = it->first;
+        send_buf[2*i+1] = it->second.new_id;
+    }
+    for (size_t i=0; i<neighbors.size(); i++) {
+        MPI_Isend( send_buf,    2*N_send,    MPI_LONG_LONG, neighbors[i], 0, comm, &send_req[i] );
+        MPI_Irecv( recv_buf[i], 2*N_recv[i], MPI_LONG_LONG, neighbors[i], 0, comm, &recv_req[i] );
+    }
+    for (it=map.begin(); it!=map.end(); ++it) {
+        remote_map[it->first] = it->second.new_id;
+    }
+    for (size_t i=0; i<neighbors.size(); i++) {
+        MPI_Wait(&recv_req[i],&status[i]);
+        for (int j=0; j<N_recv[i]; j++)
+            remote_map[recv_buf[i][2*j+0]] = recv_buf[i][2*j+1];
+    }
+    MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
+}
+// Compute a new local id for each local id
+static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map, 
+    std::map<int64_t,global_id_info_struct>& map )
+{
+    bool changed = false;
+    std::map<int64_t,global_id_info_struct>::iterator it;
+    for (it=map.begin(); it!=map.end(); ++it) {
+        int64_t id = it->second.new_id;
+        std::set<int64_t>::const_iterator it2;
+        for (it2=it->second.remote_ids.begin(); it2!=it->second.remote_ids.end(); ++it2) {
+            int64_t id2 = remote_map.find(*it2)->second;
+            id = std::min(id,id2);
+        }
+        changed = changed || it->second.new_id!=id;
+        it->second.new_id = id;
+    }
+    return changed;
+}
+static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, 
+    int nblobs, BlobIDArray& IDs, MPI_Comm comm )
+{
+    PROFILE_START("LocalToGlobalIDs",1);
+    const int rank = rank_info.rank[1][1][1];
+    int nprocs = comm_size(comm);
+    const int ngx = (IDs.size(0)-nx)/2;
+    const int ngy = (IDs.size(1)-ny)/2;
+    const int ngz = (IDs.size(2)-nz)/2;
+    // Get the number of blobs for each rank
+    std::vector<int> N_blobs(nprocs,0);
+    PROFILE_START("LocalToGlobalIDs-Allgather",1);
+    MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,comm);
+    PROFILE_STOP("LocalToGlobalIDs-Allgather",1);
+    int64_t N_blobs_tot = 0;
+    int offset = 0;
+    for (int i=0; i<rank; i++)
+        offset += N_blobs[i];
+    for (int i=0; i<nprocs; i++)
+        N_blobs_tot += N_blobs[i];
+    INSIST(N_blobs_tot<0x80000000,"Maximum number of blobs exceeded");
+    // Compute temporary global ids
+    for (size_t i=0; i<IDs.length(); i++) {
+        if ( IDs(i) >= 0 )
+            IDs(i) += offset;
+    }
+    const BlobIDArray LocalIDs = IDs;
+    // Copy the ids and get the neighbors through the halos
+    fillHalo<BlobIDType> fillData(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
+    fillData.fill(IDs);
+    // Create a list of all neighbor ranks (excluding self)
+    std::vector<int> neighbors;
+    neighbors.push_back( rank_info.rank[0][1][1] );
+    neighbors.push_back( rank_info.rank[2][1][1] );
+    neighbors.push_back( rank_info.rank[1][0][1] );
+    neighbors.push_back( rank_info.rank[1][2][1] );
+    neighbors.push_back( rank_info.rank[1][1][0] );
+    neighbors.push_back( rank_info.rank[1][1][2] );
+    std::sort( neighbors.begin(), neighbors.end() );
+    neighbors.erase( std::unique( neighbors.begin(), neighbors.end() ), neighbors.end() );
+    // Create a map of all local ids to the neighbor ids
+    std::map<int64_t,global_id_info_struct> map;
+    std::set<int64_t> local;
+    for (size_t i=0; i<LocalIDs.length(); i++) {
+        if ( LocalIDs(i)>=0 ) {
+            local.insert(LocalIDs(i));
+            if ( LocalIDs(i)!=IDs(i) && IDs(i)>=0 )
+                map[LocalIDs(i)].remote_ids.insert(IDs(i));
+        }
+    }
+    std::map<int64_t,global_id_info_struct>::iterator it;
+    for (it=map.begin(); it!=map.end(); ++it) {
+        it->second.new_id = it->first;
+        local.erase(it->first);
+    }
+    // Get the number of ids we will recieve from each rank
+    int N_send = map.size();
+    std::vector<int> N_recv(neighbors.size(),0);
+    std::vector<MPI_Request> send_req(neighbors.size());
+    std::vector<MPI_Request> recv_req(neighbors.size());
+    std::vector<MPI_Status> status(neighbors.size());
+    for (size_t i=0; i<neighbors.size(); i++) {
+        MPI_Isend( &N_send,    1, MPI_INT, neighbors[i], 0, comm, &send_req[i] );
+        MPI_Irecv( &N_recv[i], 1, MPI_INT, neighbors[i], 0, comm, &recv_req[i] );
+    }
+    MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
+    MPI_Waitall(neighbors.size(),getPtr(recv_req),getPtr(status));
+    // Allocate memory for communication
+    int64_t *send_buf = new int64_t[2*N_send];
+    std::vector<int64_t*> recv_buf(neighbors.size());
+    for (size_t i=0; i<neighbors.size(); i++)
+        recv_buf[i] = new int64_t[2*N_recv[i]];
+    // Compute a map for the remote ids, and new local id for each id
+    std::map<int64_t,int64_t> remote_map;
+    for (it=map.begin(); it!=map.end(); ++it) {
+        int64_t id = it->first;
+        std::set<int64_t>::const_iterator it2;
+        for (it2=it->second.remote_ids.begin(); it2!=it->second.remote_ids.end(); ++it2) {
+            int64_t id2 = *it2;
+            id = std::min(id,id2);
+            remote_map.insert(std::pair<int64_t,int64_t>(id2,id2));
+        }
+        it->second.new_id = id;
+    }
+    // Iterate until we are done
+    int iteration = 1;
+    PROFILE_START("LocalToGlobalIDs-loop",1);
+    while ( 1 ) {
+        iteration++;
+        // Send the local ids and their new value to all neighbors
+        updateRemoteIds( map, neighbors, N_send, N_recv,send_buf, recv_buf, remote_map, comm );
+        // Compute a new local id for each local id
+        bool changed = updateLocalIds( remote_map, map );
+        // Check if we are finished
+        int test = changed ? 1:0;
+        int result = 0;
+        MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,comm);
+        if ( result==0 )
+            break;
+    }
+    PROFILE_STOP("LocalToGlobalIDs-loop",1);
+    // Relabel the ids
+    std::vector<int> final_map(nblobs,-1);
+    for (it=map.begin(); it!=map.end(); ++it)
+        final_map[it->first-offset] = it->second.new_id;
+    for (std::set<int64_t>::const_iterator it2=local.begin(); it2!=local.end(); ++it2)
+        final_map[*it2-offset] = *it2;
+    for (size_t i=0; i<final_map.size(); i++)
+        ASSERT(final_map[i]>=0);
+    for (size_t k=ngz; k<IDs.size(2)-ngz; k++) {
+        for (size_t j=ngy; j<IDs.size(1)-ngy; j++) {
+            for (size_t i=ngx; i<IDs.size(0)-ngx; i++) {
+                BlobIDType id = IDs(i,j,k);
+                if ( id >= 0 )
+                    IDs(i,j,k) = final_map[id-offset];
+            }
+        }
+    }
+    // Fill the ghosts
+    fillHalo<BlobIDType> fillData2(comm,rank_info,nx,ny,nz,1,1,1,0,1,true,true,true);
+    fillData2.fill(IDs);
+    // Reorder based on size (and compress the id space
+    int N_blobs_global = ReorderBlobIDs2(IDs,N_blobs_tot,ngx,ngy,ngz,comm);
+    // Finished
+    delete [] send_buf;
+    for (size_t i=0; i<neighbors.size(); i++)
+        delete [] recv_buf[i];
+    PROFILE_STOP("LocalToGlobalIDs",1);
+    return N_blobs_global;
+}
+int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, 
+    const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
+    BlobIDArray& GlobalBlobID, MPI_Comm comm )
+{
+    PROFILE_START("ComputeGlobalBlobIDs");
+    // First compute the local ids
+    int nblobs = ComputeLocalBlobIDs(Phase,SignDist,vF,vS,GlobalBlobID,false);
+    // Compute the global ids
+    int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID, comm );
+    PROFILE_STOP("ComputeGlobalBlobIDs");
+    return nglobal;
+}
+int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
+    const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm )
+{
+    PROFILE_START("ComputeGlobalPhaseComponent");
+    // First compute the local ids
+    int nblobs = ComputeLocalPhaseComponent(PhaseID,VALUE,GlobalBlobID,false);
+    // Compute the global ids
+    int nglobal = LocalToGlobalIDs( nx, ny, nz, rank_info, nblobs, GlobalBlobID, comm );
+    PROFILE_STOP("ComputeGlobalPhaseComponent");
+    return nglobal;
+}
+
+
+/******************************************************************
+* Compute the mapping of blob ids between timesteps               *
+******************************************************************/
+typedef std::map<BlobIDType,std::map<BlobIDType,int64_t> > map_type;
+template<class TYPE> inline MPI_Datatype getMPIType();
+template<> inline MPI_Datatype getMPIType<int32_t>() { return MPI_INT; }
+template<> inline MPI_Datatype getMPIType<int64_t>() { 
+    if ( sizeof(int64_t)==sizeof(long int) )
+        return MPI_LONG;
+    else if ( sizeof(int64_t)==sizeof(double) )
+        return MPI_DOUBLE;
+}
+template<class TYPE>
+void gatherSet( std::set<TYPE>& set, MPI_Comm comm )
+{
+    int nprocs = comm_size(comm);
+    MPI_Datatype type = getMPIType<TYPE>();
+    std::vector<TYPE> send_data(set.begin(),set.end());
+    int send_count = send_data.size();
+    std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
+    MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
+    for (int i=1; i<nprocs; i++)
+        recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
+    std::vector<TYPE> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
+    MPI_Allgatherv(getPtr(send_data),send_count,type,
+        getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
+    for (size_t i=0; i<recv_data.size(); i++)
+        set.insert(recv_data[i]);
+}
+void gatherSrcIDMap( map_type& src_map, MPI_Comm comm )
+{
+    int nprocs = comm_size(comm);
+    MPI_Datatype type = getMPIType<int64_t>();
+    std::vector<int64_t> send_data;
+    for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) {
+        int id = it->first;
+        const std::map<BlobIDType,int64_t>& src_ids = it->second;
+        send_data.push_back(id);
+        send_data.push_back(src_ids.size());
+        std::map<BlobIDType,int64_t>::const_iterator it2;
+        for (it2=src_ids.begin(); it2!=src_ids.end(); ++it2) {
+            send_data.push_back(it2->first);
+            send_data.push_back(it2->second);
+        }
+    }
+    int send_count = send_data.size();
+    std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
+    MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
+    for (int i=1; i<nprocs; i++)
+        recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
+    std::vector<int64_t> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
+    MPI_Allgatherv(getPtr(send_data),send_count,type,
+        getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
+    size_t i=0;
+    src_map.clear();
+    while ( i < recv_data.size() ) {
+        BlobIDType id = recv_data[i];
+        size_t count = recv_data[i+1];
+        i += 2;
+        std::map<BlobIDType,int64_t>& src_ids = src_map[id];
+        for (size_t j=0; j<count; j++,i+=2) {
+            std::map<BlobIDType,int64_t>::iterator it = src_ids.find(recv_data[i]);
+            if ( it == src_ids.end() )
+                src_ids.insert(std::pair<BlobIDType,int64_t>(recv_data[i],recv_data[i+1]));
+            else
+                it->second += recv_data[i+1];
+        }
+    }
+}
+void addSrcDstIDs( BlobIDType src_id, map_type& src_map, map_type& dst_map, 
+    std::set<BlobIDType>& src, std::set<BlobIDType>& dst )
+{
+    src.insert(src_id);
+    const std::map<BlobIDType,int64_t>& dst_ids = dst_map[src_id];
+    for (std::map<BlobIDType,int64_t>::const_iterator it=dst_ids.begin(); it!=dst_ids.end(); ++it) {
+        if ( dst.find(it->first)==dst.end() )
+            addSrcDstIDs(it->first,dst_map,src_map,dst,src);
+    }
+}
+ID_map_struct computeIDMap( int nx, int ny, int nz, 
+    const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm )
+{
+    ASSERT(ID1.size()==ID2.size());
+    PROFILE_START("computeIDMap");
+    const int ngx = (ID1.size(0)-nx)/2;
+    const int ngy = (ID1.size(1)-ny)/2;
+    const int ngz = (ID1.size(2)-nz)/2;
+
+    // Get a global list of all src/dst ids and the src map for each local blob
+    std::set<BlobIDType> src_set, dst_set;
+    map_type src_map;   // Map of the src ids for each dst id
+    for (int k=ngz; k<ngz+nz; k++) {
+        for (int j=ngy; j<ngy+ny; j++) {
+            for (int i=ngx; i<ngx+nx; i++) {
+                int id1 = ID1(i,j,k);
+                int id2 = ID2(i,j,k);
+                if ( id1>=0 )
+                    src_set.insert(id1);
+                if ( id2>=0 )
+                    dst_set.insert(id2);
+                if ( id1>=0 && id2>=0 ) {
+                    std::map<BlobIDType,int64_t>& src_ids = src_map[id2];
+                    std::map<BlobIDType,int64_t>::iterator it = src_ids.find(id1);
+                    if ( it == src_ids.end() ) {
+                        src_ids.insert(std::pair<BlobIDType,int64_t>(id1,0));
+                        it = src_ids.find(id1);
+                    }
+                    it->second++;
+                }
+            }
+        }
+    }
+    // Communicate the src/dst ids and src id map to all processors and reduce
+    gatherSet( src_set, comm );
+    gatherSet( dst_set, comm );
+    gatherSrcIDMap( src_map, comm );
+    // Compute the dst id map
+    map_type dst_map;   // Map of the dst ids for each src id
+    for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) {
+        BlobIDType id = it->first;
+        const std::map<BlobIDType,int64_t>& src_ids = it->second;
+        for (std::map<BlobIDType,int64_t>::const_iterator it2=src_ids.begin(); it2!=src_ids.end(); ++it2) {
+            std::map<BlobIDType,int64_t>& dst_ids = dst_map[it2->first];
+            dst_ids.insert(std::pair<BlobIDType,int64_t>(id,it2->second));
+        }
+    }
+
+    // Perform the mapping of ids
+    ID_map_struct id_map;
+    // Find new blobs
+    for (std::set<BlobIDType>::const_iterator it=dst_set.begin(); it!=dst_set.end(); ++it) {
+        if ( src_map.find(*it)==src_map.end() )
+            id_map.created.push_back(*it);
+    }
+    // Fine blobs that disappeared
+    for (std::set<BlobIDType>::const_iterator it=src_set.begin(); it!=src_set.end(); ++it) {
+        if ( dst_map.find(*it)==dst_map.end() )
+            id_map.destroyed.push_back(*it);
+    }
+    // Find blobs with a 1-to-1 mapping
+    std::vector<BlobIDType> dst_list;
+    dst_list.reserve(src_map.size());
+    for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it)
+        dst_list.push_back(it->first);
+    for (size_t i=0; i<dst_list.size(); i++) {
+        int dst_id = dst_list[i];
+        const std::map<BlobIDType,int64_t>& src_ids = src_map[dst_id];
+        if ( src_ids.size()==1 ) {
+            int src_id = src_ids.begin()->first;
+            const std::map<BlobIDType,int64_t>& dst_ids = dst_map[src_id];
+            if ( dst_ids.size()==1 ) {
+                ASSERT(dst_ids.begin()->first==dst_id);
+                src_map.erase(dst_id);
+                dst_map.erase(src_id);
+                id_map.src_dst.push_back(std::pair<BlobIDType,BlobIDType>(src_id,dst_id));
+            }
+        }
+    }
+    // Handle merge/splits
+    while ( !dst_map.empty() ) {
+        // Get a lit of the src-dst ids
+        std::set<BlobIDType> src, dst;
+        addSrcDstIDs( dst_map.begin()->first, src_map, dst_map, src, dst );
+        if ( src.size()==1 ) {
+            // Bubble split
+            id_map.split.push_back( BlobIDSplitStruct(*src.begin(),std::vector<BlobIDType>(dst.begin(),dst.end())) );
+        } else if ( dst.size()==1 ) {
+            // Bubble merge
+            id_map.merge.push_back( BlobIDMergeStruct(std::vector<BlobIDType>(src.begin(),src.end()),*dst.begin()) );
+        } else {
+            // Bubble split/merge
+            id_map.merge_split.push_back( BlobIDMergeSplitStruct(
+                std::vector<BlobIDType>(src.begin(),src.end()), std::vector<BlobIDType>(dst.begin(),dst.end() ) ) );
+        }
+        // Add the overlaps
+        for (std::set<BlobIDType>::const_iterator it1=src.begin(); it1!=src.end(); ++it1) {
+            const std::map<BlobIDType,int64_t>& dst_ids = dst_map[*it1];
+            for (std::set<BlobIDType>::const_iterator it2=dst.begin(); it2!=dst.end(); ++it2) {
+                std::pair<BlobIDType,BlobIDType> id(*it1,*it2);
+                int64_t overlap = 0;
+                const std::map<BlobIDType,int64_t>::const_iterator it = dst_ids.find(*it2);
+                if ( it != dst_ids.end() ) { overlap = it->second; }
+                id_map.overlap.insert(std::pair<OverlapID,int64_t>(id,overlap));
+            }
+        }
+        // Clear the mapped entries
+        for (std::set<BlobIDType>::const_iterator it=src.begin(); it!=src.end(); ++it)
+            dst_map.erase(*it);
+        for (std::set<BlobIDType>::const_iterator it=dst.begin(); it!=dst.end(); ++it)
+            src_map.erase(*it);
+    }
+    ASSERT(src_map.empty());
+
+    PROFILE_STOP("computeIDMap");
+    return id_map;
+}
+
+
+/******************************************************************
+* Renumber the ids                                                *
+******************************************************************/
+typedef std::vector<BlobIDType> IDvec;
+inline void renumber( const std::vector<BlobIDType>& id1, const std::vector<BlobIDType>& id2,
+    const std::map<OverlapID,int64_t>& overlap, std::vector<BlobIDType>& new_ids, BlobIDType& id_max )
+{
+    if ( id2.empty() ) {
+        // No dst ids to set
+    } else if ( id1.empty() ) {
+        // No src ids
+        for (size_t i=0; i<id2.size(); i++) {
+            id_max++;
+            if ( (BlobIDType) new_ids.size() < id2[i]+1 )
+                new_ids.resize(id2[i]+1,-1);
+            new_ids[id2[i]] = id_max;
+        }
+    } else if ( id1.size()==1 && id2.size()==1 ) {
+        // Direct src-dst mapping
+        if ( (BlobIDType) new_ids.size() < id2[0]+1 )
+            new_ids.resize(id2[0]+1,-1);
+        new_ids[id2[0]] = id1[0];
+    } else {
+        // General N to M mapping
+        // Get the overlap weights
+        Array<int64_t> cost(id1.size(),id2.size());
+        for (size_t j=0; j<id2.size(); j++) {
+            for (size_t i=0; i<id1.size(); i++) {
+                cost(i,j) = overlap.find(std::pair<BlobIDType,BlobIDType>(id1[i],id2[j]))->second;
+            }
+        }
+        // While we have not mapped all dst ids
+        while ( 1 ) {
+            size_t index = 1;
+            int64_t cost2 = -1;
+            for (size_t i=0; i<cost.length(); i++) {
+                if ( cost(i) > cost2 ) {
+                    cost2 = cost(i);
+                    index = i;
+                }
+            }
+            if ( cost2 <= 0 )
+                break;
+            // Use id1[i] for id2[j]
+            int i = index%id1.size();
+            int j = index/id1.size();
+            if ( (BlobIDType) new_ids.size() < id2[j]+1 )
+                new_ids.resize(id2[j]+1,-1);
+            new_ids[id2[j]] = id1[i];
+            for (size_t k=0; k<id2.size(); k++)
+                cost(i,k) = -1;
+            for (size_t k=0; k<id1.size(); k++)
+                cost(k,j) = -1;
+        }
+        // No remaining src overlap with dst, create new ids for all remaining dst
+        for (size_t i=0; i<id2.size(); i++) {
+            if ( (BlobIDType) new_ids.size() < id2[i]+1 )
+                new_ids.resize(id2[i]+1,-1);
+            if ( new_ids[id2[i]] == -1 ) {
+                id_max++;
+                new_ids[id2[i]] = id_max;
+            }
+        }
+    }
+}
+inline void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDType& id )
+{
+    id = new_ids[id];
+}
+inline void renumberIDs( const std::vector<BlobIDType>& new_ids, std::vector<BlobIDType>& ids )
+{
+    for (size_t i=0; i<ids.size(); i++)
+        ids[i] = new_ids[ids[i]];
+}
+void getNewIDs( ID_map_struct& map, BlobIDType& id_max, std::vector<BlobIDType>& new_ids )
+{
+    new_ids.resize(0);
+    // Get the new id numbers for each map type
+    for (size_t i=0; i<map.src_dst.size(); i++)
+        renumber(IDvec(1,map.src_dst[i].first),IDvec(1,map.src_dst[i].second),map.overlap,new_ids,id_max);
+    for (size_t i=0; i<map.created.size(); i++)
+        renumber(std::vector<BlobIDType>(),IDvec(1,map.created[i]),map.overlap,new_ids,id_max);
+    for (size_t i=0; i<map.destroyed.size(); i++)
+        renumber(IDvec(1,map.destroyed[i]),std::vector<BlobIDType>(),map.overlap,new_ids,id_max);
+    for (size_t i=0; i<map.split.size(); i++)
+        renumber(IDvec(1,map.split[i].first),map.split[i].second,map.overlap,new_ids,id_max);
+    for (size_t i=0; i<map.merge.size(); i++)
+        renumber(map.merge[i].first,IDvec(1,map.merge[i].second),map.overlap,new_ids,id_max);
+    for (size_t i=0; i<map.merge_split.size(); i++)
+        renumber(map.merge_split[i].first,map.merge_split[i].second,map.overlap,new_ids,id_max);
+    // Renumber the ids in the map
+    for (size_t i=0; i<map.src_dst.size(); i++)
+        renumberIDs( new_ids, map.src_dst[i].second );
+    renumberIDs( new_ids, map.created );
+    for (size_t i=0; i<map.split.size(); i++)
+        renumberIDs( new_ids, map.split[i].second );
+    for (size_t i=0; i<map.merge.size(); i++)
+        renumberIDs( new_ids, map.merge[i].second );
+    for (size_t i=0; i<map.merge_split.size(); i++)
+        renumberIDs( new_ids, map.merge_split[i].second );
+    std::map<OverlapID,int64_t> overlap2;
+    for (std::map<OverlapID,int64_t>::const_iterator it=map.overlap.begin(); it!=map.overlap.begin(); ++it) {
+        OverlapID id = it->first;
+        renumberIDs( new_ids, id.second );
+        overlap2.insert( std::pair<OverlapID,int64_t>(id,it->second) );
+    }
+}
+void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDArray& IDs )
+{
+    size_t N = IDs.length();
+    BlobIDType* ids = IDs.data();
+    for (size_t i=0; i<N; i++) {
+        BlobIDType id = ids[i];
+        if ( id>=0 )
+            ids[i] = new_ids[id];
+    }
+}
+
+
+/******************************************************************
+* Write the id map for the given timestep                         *
+******************************************************************/
+void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename )
+{
+    int rank = MPI_WORLD_RANK();
+    if ( rank!=0 )
+        return;
+    bool empty = map.created.empty() && map.destroyed.empty() &&
+        map.split.empty() && map.merge.empty() && map.merge_split.empty();
+    for (size_t i=0; i<map.src_dst.size(); i++)
+        empty = empty && map.src_dst[i].first==map.src_dst[i].second;
+    if ( timestep!=0 && empty )
+        return;
+    FILE *fid = NULL;
+    if ( timestep==0 )
+        fid = fopen(filename.c_str(),"wb");
+    else
+        fid = fopen(filename.c_str(),"ab");
+    INSIST(fid!=NULL,std::string("Error opening file: ")+filename);
+    if ( empty ) {
+        fclose(fid);
+        return;
+    }
+    fprintf(fid,"%lli:",timestep);
+    for (size_t i=0; i<map.created.size(); i++)
+        fprintf(fid," -%lli",static_cast<long long int>(map.created[i]));
+    for (size_t i=0; i<map.destroyed.size(); i++)
+        fprintf(fid," %lli-",static_cast<long long int>(map.destroyed[i]));
+    for (size_t i=0; i<map.src_dst.size(); i++) {
+        if ( map.src_dst[i].first!=map.src_dst[i].second )
+            fprintf(fid," %lli-%lli",static_cast<long long int>(map.src_dst[i].first),
+                                    static_cast<long long int>(map.src_dst[i].second));
+    }
+    for (size_t i=0; i<map.split.size(); i++) {
+        fprintf(fid," %lli-%lli",static_cast<long long int>(map.split[i].first),
+                              static_cast<long long int>(map.split[i].second[0]));
+        for (size_t j=1; j<map.split[i].second.size(); j++)
+            fprintf(fid,"/%lli",static_cast<long long int>(map.split[i].second[j]));
+    }
+    for (size_t i=0; i<map.merge.size(); i++) {
+        fprintf(fid," %lli",static_cast<long long int>(map.merge[i].first[0]));
+        for (size_t j=1; j<map.merge[i].first.size(); j++)
+            fprintf(fid,"/%lli",static_cast<long long int>(map.merge[i].first[j]));
+        fprintf(fid,"-%lli",static_cast<long long int>(map.merge[i].second));
+    }
+    for (size_t i=0; i<map.merge_split.size(); i++) {
+        fprintf(fid," %lli",static_cast<long long int>(map.merge_split[i].first[0]));
+        for (size_t j=1; j<map.merge_split[i].first.size(); j++)
+            fprintf(fid,"/%lli",static_cast<long long int>(map.merge_split[i].first[j]));
+        fprintf(fid,"-%lli",static_cast<long long int>(map.merge_split[i].second[0]));
+        for (size_t j=1; j<map.merge_split[i].second.size(); j++)
+            fprintf(fid,"/%lli",static_cast<long long int>(map.merge_split[i].second[j]));
+    }
+    fprintf(fid,"\n");
+    fclose(fid);
+}
+

analysis/analysis.h

@@ -0,0 +1,160 @@
+#ifndef Analysis_H_INC
+#define Analysis_H_INC
+
+#include "common/Array.h"
+#include "common/Communication.h"
+
+#include <set>
+#include <map>
+#include <vector>
+
+
+// Define types to use for blob ids
+typedef int32_t BlobIDType;
+typedef Array<BlobIDType> BlobIDArray;
+
+
+/*!
+ * @brief  Compute the blob
+ * @details  Compute the blob (F>vf|S>vs) starting from (i,j,k) - oil blob
+ * @return  Returns the number of cubes in the blob
+ * @param[in] Phase         Phase
+ * @param[in] SignDist      SignDist
+ * @param[in] vF            vF
+ * @param[in] vS            vS
+ * @param[in] S             S
+ * @param[out] LocalBlobID  The ids of the blobs
+ * @return  Returns the number of blobs
+ */
+int ComputeLocalBlobIDs( const DoubleArray& Phase, const DoubleArray& SignDist, 
+    double vF, double vS, BlobIDArray& LocalBlobID, bool periodic=true );
+
+/*!
+ *  @brief Compute blob of an arbitrary phase
+ *  @details Compute the connected components for Phase(i,j,k)=VALUE
+ *  @return the number of connected components of the phase
+ *  @param[in] PhaseID
+ *  @param[in] VALUE
+ *  @param[out] ComponentLabel
+ *  @param[in] periodic
+ */
+int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &ComponentLabel, bool periodic );
+
+
+/*!
+ * @brief  Compute the blob
+ * @details  Compute the blob (F>vf|S>vs) starting from (i,j,k) - oil blob
+ * @return  Returns the number of cubes in the blob
+ * @param[in] nx            Number of elements in the x-direction
+ * @param[in] ny            Number of elements in the y-direction
+ * @param[in] nz            Number of elements in the z-direction
+ * @param[in] Phase         Phase
+ * @param[in] SignDist      SignDist
+ * @param[in] vF            vF
+ * @param[in] vS            vS
+ * @param[in] S             S
+ * @param[out] LocalBlobID  The ids of the blobs
+ * @return  Returns the number of blobs
+ */
+int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info, 
+    const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS, 
+    BlobIDArray& GlobalBlobID, MPI_Comm comm );
+
+
+/*!
+ * @brief Compute component of the specified phase
+ * @details Compute component of specified phase PhaseID=VALUE
+ * @return  Returns the number of cubes in the blob
+ * @param[in] nx            Number of elements in the x-direction
+ * @param[in] ny            Number of elements in the y-direction
+ * @param[in] nz            Number of elements in the z-direction
+ * @param[in] rank_in       MPI communication info
+ * @param[in] PhaseID       Array that identifies the phases
+ * @param[in] VALUE         Identifier for the phase to decompose
+ * @param[out] GlobalBlobID The ids of the blobs for the phase
+ * @return Return the number of components in the specified phase
+ */
+int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
+    const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm );
+
+
+/*!
+ * @brief  Reorder the blobs
+ * @details  Reorder the blobs based on the number of cells they contain
+ *    largest first.
+ * @param[in] nx            Number of elements in the x-direction
+ * @param[in] ny            Number of elements in the y-direction
+ * @param[in] nz            Number of elements in the z-direction
+ * @param[in/out] ID        The ids of the blobs
+ */
+void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm );
+
+
+typedef std::pair<BlobIDType,std::vector<BlobIDType> > BlobIDSplitStruct;
+typedef std::pair<std::vector<BlobIDType>,BlobIDType> BlobIDMergeStruct;
+typedef std::pair<std::vector<BlobIDType>,std::vector<BlobIDType> > BlobIDMergeSplitStruct;
+typedef std::pair<BlobIDType,BlobIDType> OverlapID;
+struct ID_map_struct {
+    std::vector<BlobIDType> created;            // list of new blobs that were created
+    std::vector<BlobIDType> destroyed;          // list of blobs that disappeared
+    std::vector<std::pair<BlobIDType,BlobIDType> > src_dst;   // one-one mapping of blobs (first,second timestep id)
+    std::vector<BlobIDSplitStruct> split;       // list of blobs that split
+    std::vector<BlobIDMergeStruct> merge;       // list of blobs that merged
+    std::vector<BlobIDMergeSplitStruct> merge_split; // list of blobs that both merged and split
+    std::map<OverlapID,int64_t> overlap;         // for ids that are not a 1-1 mapping, this is a list of the overlaps <src,dst>
+    //! Empty constructor
+    ID_map_struct() {}
+    //! Create initial map from N blobs (ordered 1:N-1)
+    ID_map_struct( int N ) {
+        created.resize(N);
+        for (int i=0; i<N; i++) { created[i]=i; }
+    }
+};
+
+
+/*!
+ * @brief  Get the mapping of blob ids between iterations
+ * @details  This functions computes the map of blob ids between iterations
+ * @return  Returns the map of the blob ids.  Each final blob may have no source
+ *    ids, one parent, or multiple parents.  Each src id may be a parent for multiple blobs.
+ * @param[in] ID1           The blob ids at the first timestep
+ * @param[in] ID2           The blob ids at the second timestep
+ */
+ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm );
+
+
+/*!
+ * @brief  Compute the new global ids based on the map
+ * @details  This functions computes the time-consistent global ids for the
+ *     current global id index
+ * @param[in/out] map       The timestep mapping for the ids
+ * @param[in] id_max        The globally largest id used previously
+ * @param[out] new_ids      The newly renumbered blob ids (0:ids.max())
+ */
+void getNewIDs( ID_map_struct& map, BlobIDType& id_max, std::vector<BlobIDType>& new_ids );
+
+
+/*!
+ * @brief  Update the blob ids based on mapping
+ * @details  This functions computes the map of blob ids between iterations.
+ *    Note: we also update the map to reflect the new ids
+ * @param[out] new_ids          The newly renumbered blob ids (0:ids.max())
+ * @param[in/out] IDs           The blob ids to renumber
+ */
+void renumberIDs( const std::vector<BlobIDType>& new_id_list, BlobIDArray& IDs );
+
+
+/*!
+ * @brief  Write the ID map
+ * @details  This functions writes the id map fo an iteration.
+ *    If no ids changed, then nothing will be written
+ *    Note: only rank 0 writes, and the file is created on timestep 0.
+ * @param[in] map           The timestep mapping for the ids
+ * @param[in] timestep      The current timestep (timestep 0 creates the file)
+ * @param[in] filename      The filename to write/append
+ */
+void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename );
+
+
+
+#endif

analysis/eikonal.h

@@ -0,0 +1,58 @@
+#ifndef Eikonal_H_INC
+#define Eikonal_H_INC
+
+
+
+/*!
+ * @brief  Calculate the distance solving the Eikonal equation
+ * @details  This routine converts the data in the Distance array to a signed distance
+ * by solving the equation df/dt = sign*(1-|grad f|), where Distance provides
+ * the values of f on the mesh associated with domain Dm
+ * It has been tested with segmented data initialized to values [-1,1]
+ * and will converge toward the signed distance to the surface bounding the associated phases
+ *
+ * Reference:
+ * Min C (2010) On reinitializing level set functions, Journal of Computational Physics	229
+ *
+ * @return  Returns the number of cubes in the blob
+ * @param[in/out] Distance  Distance function
+ * @param[in] ID            Segmentation id
+ * @param[in] DM            Domain information
+ * @param[in] timesteps     Maximum number of timesteps to process
+ * @return  Returns the global variation
+ */
+inline double Eikonal(DoubleArray &Distance, const char *ID, const Domain &Dm, int timesteps);
+
+float Eikonal3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm, const int timesteps);
+
+
+/*!
+ * @brief  Calculate the distance using a simple method
+ * @details  This routine calculates the distance using a very simple method working off the segmentation id.
+ *
+ * @return  Returns the number of cubes in the blob
+ * @param[in/out] Distance  Distance function
+ * @param[in] ID            Segmentation id
+ * @param[in] DM            Domain information
+ * @return  Returns the global variation
+ */
+void CalcDist3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm );
+
+
+/*!
+ * @brief  Calculate the distance using a multi-level method
+ * @details  This routine calculates the distance using a multi-grid method
+ *
+ * @return  Returns the number of cubes in the blob
+ * @param[in/out] Distance  Distance function
+ * @param[in] ID            Segmentation id
+ * @param[in] DM            Domain information
+ * @return  Returns the global variation
+ */
+void CalcDistMultiLevel( Array<float> &Distance, const Array<char> &ID, const Domain &Dm );
+
+
+
+#include "analysis/eikonal.hpp"
+
+#endif

analysis/eikonal.hpp

@@ -0,0 +1,519 @@
+#ifndef Eikonal_HPP_INC
+#define Eikonal_HPP_INC
+
+#include "analysis/eikonal.h"
+#include "analysis/imfilter.h"
+
+
+
+
+inline float minmod(float &a, float &b)
+{
+    float value = a;
+    if     ( a*b < 0.0)
+        value=0.0;
+    else if (fabs(a) > fabs(b))
+        value = b;
+    return value;
+}
+
+
+inline double minmod(double &a, double &b){
+
+	double value;
+
+	value = a;
+	if 	( a*b < 0.0)	    value=0.0;
+	else if (fabs(a) > fabs(b)) value = b;
+
+	return value;
+}
+
+
+/******************************************************************
+* Solve the eikonal equation                                      *
+******************************************************************/
+
+
+inline double Eikonal(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
+
+	/*
+	 * This routine converts the data in the Distance array to a signed distance
+	 * by solving the equation df/dt = sign(1-|grad f|), where Distance provides
+	 * the values of f on the mesh associated with domain Dm
+	 * It has been tested with segmented data initialized to values [-1,1]
+	 * and will converge toward the signed distance to the surface bounding the associated phases
+	 *
+	 * Reference:
+	 * Min C (2010) On reinitializing level set functions, Journal of Computational Physics	229
+	 */
+
+	int i,j,k;
+	double dt=0.1;
+	double Dx,Dy,Dz;
+	double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
+	double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
+	double sign,norm;
+	double LocalVar,GlobalVar,LocalMax,GlobalMax;
+
+	int xdim,ydim,zdim;
+	xdim=Dm.Nx-2;
+	ydim=Dm.Ny-2;
+	zdim=Dm.Nz-2;
+	fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
+
+	// Arrays to store the second derivatives
+	DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
+	DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
+	DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
+
+	int count = 0;
+	while (count < timesteps){
+
+		// Communicate the halo of values
+		fillData.fill(Distance);
+
+		// Compute second order derivatives
+		for (k=1;k<Dm.Nz-1;k++){
+			for (j=1;j<Dm.Ny-1;j++){
+				for (i=1;i<Dm.Nx-1;i++){
+					Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
+					Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
+					Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
+				}
+			}
+		}
+		fillData.fill(Dxx);
+		fillData.fill(Dyy);
+		fillData.fill(Dzz);
+
+		LocalMax=LocalVar=0.0;
+		// Execute the next timestep
+		for (k=1;k<Dm.Nz-1;k++){
+			for (j=1;j<Dm.Ny-1;j++){
+				for (i=1;i<Dm.Nx-1;i++){
+
+					int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
+
+					sign = 1;
+					if (ID[n] == 0) sign = -1;
+
+					// local second derivative terms
+					Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
+					Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
+					Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
+					Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
+					Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
+					Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
+
+					/* //............Compute upwind derivatives ...................
+                    Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
+                    Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
+                    Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
+                    Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+                    Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+                    Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+					 */
+					Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
+					Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
+					Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
+
+					Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+					Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+					Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+
+					// Compute upwind derivatives for Godunov Hamiltonian
+					if (sign < 0.0){
+						if (Dxp + Dxm > 0.f)  	Dx = Dxp*Dxp;
+						else					Dx = Dxm*Dxm;
+
+						if (Dyp + Dym > 0.f)  	Dy = Dyp*Dyp;
+						else					Dy = Dym*Dym;
+
+						if (Dzp + Dzm > 0.f)  	Dz = Dzp*Dzp;
+						else					Dz = Dzm*Dzm;
+					}
+					else{
+
+						if (Dxp + Dxm < 0.f)  	Dx = Dxp*Dxp;
+						else					Dx = Dxm*Dxm;
+
+						if (Dyp + Dym < 0.f)  	Dy = Dyp*Dyp;
+						else					Dy = Dym*Dym;
+
+						if (Dzp + Dzm < 0.f)  	Dz = Dzp*Dzp;
+						else					Dz = Dzm*Dzm;
+					}
+
+					//Dx = max(Dxp*Dxp,Dxm*Dxm);
+					//Dy = max(Dyp*Dyp,Dym*Dym);
+					//Dz = max(Dzp*Dzp,Dzm*Dzm);
+
+					norm=sqrt(Dx + Dy + Dz);
+					if (norm > 1.0) norm=1.0;
+
+					Distance(i,j,k) += dt*sign*(1.0 - norm);
+					LocalVar +=  dt*sign*(1.0 - norm);
+
+					if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
+						LocalMax = fabs(dt*sign*(1.0 - norm));
+				}
+			}
+		}
+
+		MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
+		MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
+		GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz;
+		count++;
+
+
+		if (count%50 == 0 && Dm.rank==0 ){
+			printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
+			fflush(stdout);
+		}
+
+		if (fabs(GlobalMax) < 1e-5){
+			if (Dm.rank==0) printf("Exiting with max tolerance of 1e-5 \n");
+			count=timesteps;
+		}
+	}
+	return GlobalVar;
+}
+
+inline float Eikonal3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm, const int timesteps)
+{
+    PROFILE_START("Eikonal3D");
+
+    /*
+     * This routine converts the data in the Distance array to a signed distance
+     * by solving the equation df/dt = sign*(1-|grad f|), where Distance provides
+     * the values of f on the mesh associated with domain Dm
+     * It has been tested with segmented data initialized to values [-1,1]
+     * and will converge toward the signed distance to the surface bounding the associated phases
+     *
+     * Reference:
+     * Min C (2010) On reinitializing level set functions, Journal of Computational Physics    229
+     */
+
+    int i,j,k;
+    float dt=0.1;
+    float Dx,Dy,Dz;
+    float Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
+    float Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
+    float sign,norm;
+    float LocalVar,GlobalVar,LocalMax,GlobalMax;
+
+    int xdim,ydim,zdim;
+    xdim=Dm.Nx-2;
+    ydim=Dm.Ny-2;
+    zdim=Dm.Nz-2;
+    fillHalo<float> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
+
+    // Arrays to store the second derivatives
+    Array<float> Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
+    Array<float> Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
+    Array<float> Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
+
+    int count = 0;
+    while (count < timesteps){
+
+        // Communicate the halo of values
+        fillData.fill(Distance);
+
+        // Compute second order derivatives
+        for (k=1;k<Dm.Nz-1;k++){
+            for (j=1;j<Dm.Ny-1;j++){
+                for (i=1;i<Dm.Nx-1;i++){
+                    Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
+                    Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
+                    Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
+                }
+            }
+        }
+        fillData.fill(Dxx);
+        fillData.fill(Dyy);
+        fillData.fill(Dzz);
+
+        LocalMax=LocalVar=0.0;
+        // Execute the next timestep
+        //  f(n+1) = f(n) + dt*sign(1-|grad f|)
+        for (k=1;k<Dm.Nz-1;k++){
+            for (j=1;j<Dm.Ny-1;j++){
+                for (i=1;i<Dm.Nx-1;i++){
+
+                    int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
+
+                    sign = -1;
+                    if (ID(i,j,k) == 1) sign = 1;
+
+                    // local second derivative terms
+                    Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
+                    Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
+                    Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
+                    Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
+                    Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
+                    Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
+
+                    /* //............Compute upwind derivatives ...................
+                    Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
+                    Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
+                    Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
+
+                    Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
+                    Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
+                    Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
+                     */
+                    Dxp = Distance(i+1,j,k);
+                    Dyp = Distance(i,j+1,k);
+                    Dzp = Distance(i,j,k+1);
+
+                    Dxm = Distance(i-1,j,k);
+                    Dym = Distance(i,j-1,k);
+                    Dzm = Distance(i,j,k-1);
+
+                    // Compute upwind derivatives for Godunov Hamiltonian
+                    if (sign < 0.0){
+                        if (Dxp > Dxm)  Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp;
+                        else            Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm;
+
+                        if (Dyp > Dym)  Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp;
+                        else            Dy = Distance(i,j,k) - Dym + 0.5*Dyym;
+
+                        if (Dzp > Dzm)  Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp;
+                        else            Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm;
+                    }
+                    else{
+                        if (Dxp < Dxm)  Dx = Dxp - Distance(i,j,k) + 0.5*Dxxp;
+                        else            Dx = Distance(i,j,k) - Dxm + 0.5*Dxxm;
+
+                        if (Dyp < Dym)  Dy = Dyp - Distance(i,j,k) + 0.5*Dyyp;
+                        else            Dy = Distance(i,j,k) - Dym + 0.5*Dyym;
+
+                        if (Dzp < Dzm)  Dz = Dzp - Distance(i,j,k) + 0.5*Dzzp;
+                        else            Dz = Distance(i,j,k) - Dzm + 0.5*Dzzm;
+                    }
+
+                    norm=sqrt(Dx*Dx+Dy*Dy+Dz*Dz);
+                    if (norm > 1.0) norm=1.0;
+
+                    Distance(i,j,k) += dt*sign*(1.0 - norm);
+                    LocalVar +=  dt*sign*(1.0 - norm);
+
+                    if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
+                        LocalMax = fabs(dt*sign*(1.0 - norm));
+                }
+            }
+        }
+
+        MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_FLOAT,MPI_SUM,Dm.Comm);
+        MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_FLOAT,MPI_MAX,Dm.Comm);
+        GlobalVar /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2)*Dm.nprocx*Dm.nprocy*Dm.nprocz;
+        count++;
+
+        if (count%50 == 0 && Dm.rank==0 )
+            printf("    Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
+
+        if (fabs(GlobalMax) < 1e-5){
+            if (Dm.rank==0) printf("    Exiting with max tolerance of 1e-5 \n");
+            count=timesteps;
+        }
+    }
+    PROFILE_STOP("Eikonal3D");
+    return GlobalVar;
+
+}
+
+
+/******************************************************************
+* A fast distance calculation                                     *
+******************************************************************/
+inline bool CalcDist3DIteration( Array<float> &Distance, const Domain &Dm )
+{
+    const float sq2 = sqrt(2.0f);
+    const float sq3 = sqrt(3.0f);
+    float dist0[27];
+    dist0[0] = sq3;   dist0[1] = sq2;   dist0[2] = sq3;
+    dist0[3] = sq2;   dist0[4] = 1;     dist0[5] = sq2;
+    dist0[6] = sq3;   dist0[7] = sq2;   dist0[8] = sq3;
+    dist0[9] = sq2;   dist0[10] = 1;    dist0[11] = sq2;
+    dist0[12] = 1;    dist0[13] = 0;    dist0[14] = 1;
+    dist0[15] = sq2;  dist0[16] = 1;    dist0[17] = sq2;
+    dist0[18] = sq3;  dist0[19] = sq2;  dist0[20] = sq3;
+    dist0[21] = sq2;  dist0[22] = 1;    dist0[23] = sq2;
+    dist0[24] = sq3;  dist0[25] = sq2;  dist0[26] = sq3;
+    bool changed = false;
+    for (size_t k=1; k<Distance.size(2)-1; k++) {
+        for (size_t j=1; j<Distance.size(1)-1; j++) {
+            for (size_t i=1; i<Distance.size(0)-1; i++) {
+                float dist[27];
+                dist[0] = Distance(i-1,j-1,k-1);   dist[1] = Distance(i,j-1,k-1);   dist[2] = Distance(i+1,j-1,k-1);
+                dist[3] = Distance(i-1,j,k-1);     dist[4] = Distance(i,j,k-1);     dist[5] = Distance(i+1,j,k-1);
+                dist[6] = Distance(i-1,j+1,k-1);   dist[7] = Distance(i,j+1,k-1);   dist[8] = Distance(i+1,j+1,k-1);
+                dist[9] = Distance(i-1,j-1,k);     dist[10] = Distance(i,j-1,k);    dist[11] = Distance(i+1,j-1,k);
+                dist[12] = Distance(i-1,j,k);      dist[13] = Distance(i,j,k);      dist[14] = Distance(i+1,j,k);
+                dist[15] = Distance(i-1,j+1,k);    dist[16] = Distance(i,j+1,k);    dist[17] = Distance(i+1,j+1,k);
+                dist[18] = Distance(i-1,j-1,k+1);  dist[19] = Distance(i,j-1,k+1);  dist[20] = Distance(i+1,j-1,k+1);
+                dist[21] = Distance(i-1,j,k+1);    dist[22] = Distance(i,j,k+1);    dist[23] = Distance(i+1,j,k+1);
+                dist[24] = Distance(i-1,j+1,k+1);  dist[25] = Distance(i,j+1,k+1);  dist[26] = Distance(i+1,j+1,k+1);
+                float tmp = 1e100;
+                for (int k=0; k<27; k++)
+                    tmp = std::min(tmp,dist[k]+dist0[k]);
+                if ( tmp < Distance(i,j,k) ) {
+                    Distance(i,j,k) = tmp;
+                    changed = true;
+                }
+            }
+        }
+    }
+    return changed;
+}
+inline void CalcDist3D( Array<float> &Distance, const Array<char> &ID, const Domain &Dm )
+{
+    PROFILE_START("Calc Distance");
+    // Initialize the distance to be 0 fore the cells adjacent to the interface
+    Distance.fill(1e100);
+    for (size_t k=1; k<ID.size(2)-1; k++) {
+        for (size_t j=1; j<ID.size(1)-1; j++) {
+            for (size_t i=1; i<ID.size(0)-1; i++) {
+                char id = ID(i,j,k);
+                if ( id!=ID(i-1,j,k) || id!=ID(i+1,j,k) || id!=ID(i,j-1,k) || id!=ID(i,j+1,k) || id!=ID(i,j,k-1) || id!=ID(i,j,k+1) )
+                    Distance(i,j,k) = 0.5;
+            }
+        }
+    }
+    // Compute the distance everywhere
+    fillHalo<float> fillData(Dm.Comm, Dm.rank_info,Dm.Nx,Dm.Ny,Dm.Nz,1,1,1,0,1);
+    while ( true ) {
+        // Communicate the halo of values
+        fillData.fill(Distance);
+        // The distance of the cell is the minimum of the distance of the neighbors plus the distance to that node
+        bool changed = CalcDist3DIteration( Distance, Dm );
+        changed = sumReduce(Dm.Comm,changed);
+        if ( !changed )
+            break;
+    }
+    // Update the sign of the distance
+    for (size_t i=0; i<ID.length(); i++)
+        Distance(i) *= ID(i)>0 ? 1:-1;
+    PROFILE_STOP("Calc Distance");
+}
+
+
+/******************************************************************
+* A fast distance calculation                                     *
+******************************************************************/
+inline void CalcDistMultiLevelHelper( Array<float> &Distance, const Domain &Dm )
+{
+    size_t ratio = 4;
+    std::function<float(const Array<float>&)> coarsen = [ratio]( const Array<float>& data )
+    {
+        float tmp = 1e100;
+        int nx = data.size(0);
+        int ny = data.size(1);
+        int nz = data.size(2);
+        for (int k=0; k<nz; k++) {
+            float z = k-0.5*(nz-1);
+            for (int j=0; j<ny; j++) {
+                float y = j-0.5*(ny-1);
+                for (int i=0; i<nx; i++) {
+                    float x = i-0.5*(nx-1);
+                    tmp = std::min(data(i,j,k)+sqrt(x*x+y*y+z*z),tmp);
+                }
+            }
+        }
+        return tmp/ratio;
+    };
+    int Nx = Dm.Nx-2;
+    int Ny = Dm.Ny-2;
+    int Nz = Dm.Nz-2;
+    ASSERT(int(Distance.size(0))==Nx+2&&int(Distance.size(1))==Ny+2&&int(Distance.size(2))==Nz+2);
+    fillHalo<float> fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
+    if ( Nx%ratio==0 && Nx>8 && Ny%ratio==0 && Ny>8 && Nz%ratio==0 && Nz>8 ) {
+        // Use recursive version
+        int Nr = std::max(std::max(ratio,ratio),ratio);
+        // Run Nr iterations, communicate, run Nr iterations
+        for (int i=0; i<Nr; i++)
+            CalcDist3DIteration( Distance, Dm );
+        /*fillData.fill(Distance);
+        for (int i=0; i<Nr; i++)
+            CalcDist3DIteration( Distance, Dm );*/
+        // Coarsen
+        Array<float> dist(Nx,Ny,Nz);
+        fillData.copy(Distance,dist);
+        Domain Dm2(Nx/ratio,Ny/ratio,Nz/ratio,Dm.rank,Dm.nprocx,Dm.nprocy,Dm.nprocz,Dm.Lx,Dm.Ly,Dm.Lz,0);
+        Dm2.CommInit(Dm.Comm);
+        fillHalo<float> fillData2(Dm2.Comm,Dm2.rank_info,Nx/ratio,Ny/ratio,Nz/ratio,1,1,1,0,1);
+        auto dist2 = dist.coarsen( {ratio,ratio,ratio}, coarsen );
+        Array<float> Distance2(Nx/ratio+2,Ny/ratio+2,Nz/ratio+2);
+        fillData2.copy(dist2,Distance2);
+        // Solve
+        CalcDistMultiLevelHelper( Distance2, Dm2 );
+        // Interpolate the coarse grid to the fine grid
+        fillData2.copy(Distance2,dist2);
+        for (int k=0; k<Nz; k++) {
+            int k2 = k/ratio;
+            float z = (k-k2*ratio)-0.5*(ratio-1);
+            for (int j=0; j<Ny; j++) {
+                int j2 = j/ratio;
+                float y = (j-j2*ratio)-0.5*(ratio-1);
+                for (int i=0; i<Nx; i++) {
+                    int i2 = i/ratio;
+                    float x = (i-i2*ratio)-0.5*(ratio-1);
+                    dist(i,j,k) = std::min(dist(i,j,k),ratio*dist2(i2,j2,k2)+sqrt(x*x+y*y+z*z));
+                }
+            }
+        }
+        fillData.copy(dist,Distance);
+        // Run Nr iterations, communicate, run Nr iterations
+        for (int i=0; i<Nr; i++)
+            CalcDist3DIteration( Distance, Dm );
+        fillData.fill(Distance);
+        for (int i=0; i<Nr; i++)
+            CalcDist3DIteration( Distance, Dm );
+    } else {
+        // Use coarse-grid version
+        while ( true ) {
+            // Communicate the halo of values
+            fillData.fill(Distance);
+            // The distance of the cell is the minimum of the distance of the neighbors plus the distance to that node
+            bool changed = CalcDist3DIteration( Distance, Dm );
+            changed = sumReduce(Dm.Comm,changed);
+            if ( !changed )
+                break;
+        }
+    }
+}
+inline void CalcDistMultiLevel( Array<float> &Distance, const Array<char> &ID, const Domain &Dm )
+{
+    PROFILE_START("Calc Distance Multilevel");
+    int Nx = Dm.Nx-2;
+    int Ny = Dm.Ny-2;
+    int Nz = Dm.Nz-2;
+    ASSERT(int(Distance.size(0))==Nx+2&&int(Distance.size(1))==Ny+2&&int(Distance.size(2))==Nz+2);
+    fillHalo<float> fillData(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
+    // Initialize the distance to be 0 fore the cells adjacent to the interface
+    Distance.fill(1e100);
+    for (size_t k=1; k<ID.size(2)-1; k++) {
+        for (size_t j=1; j<ID.size(1)-1; j++) {
+            for (size_t i=1; i<ID.size(0)-1; i++) {
+                char id = ID(i,j,k);
+                if ( id!=ID(i-1,j,k) || id!=ID(i+1,j,k) || id!=ID(i,j-1,k) || id!=ID(i,j+1,k) || id!=ID(i,j,k-1) || id!=ID(i,j,k+1) )
+                    Distance(i,j,k) = 0.5;
+            }
+        }
+    }
+    // Solve the for the distance using a recursive method
+    CalcDistMultiLevelHelper( Distance, Dm );
+    // Update the sign of the distance
+    for (size_t i=0; i<ID.length(); i++)
+        Distance(i) *= ID(i)>0 ? 1:-1;
+    fillData.fill(Distance);
+    // Run a quick filter to smooth the data
+    float sigma = 0.6;
+    Array<float> H = imfilter::create_filter<float>( { 1 }, "gaussian", &sigma );
+    std::vector<imfilter::BC> BC(3,imfilter::BC::replicate);
+    Distance = imfilter::imfilter_separable<float>( Distance, {H,H,H}, BC );
+    PROFILE_STOP("Calc Distance Multilevel");
+}
+
+#endif

analysis/filters.cpp

@@ -0,0 +1,149 @@
+#include "analysis/filters.h"
+#include "math.h"
+#include "ProfilerApp.h"
+
+
+void Med3D( const Array<float> &Input, Array<float> &Output )
+{
+	PROFILE_START("Med3D");
+	// Perform a 3D Median filter on Input array with specified width
+	int i,j,k,ii,jj,kk;
+	int imin,jmin,kmin,imax,jmax,kmax;
+
+	float *List;
+	List=new float[27];
+
+	int Nx = int(Input.size(0));
+	int Ny = int(Input.size(1));
+	int Nz = int(Input.size(2));
+
+	for (k=1; k<Nz-1; k++){
+		for (j=1; j<Ny-1; j++){
+			for (i=1; i<Nx-1; i++){
+
+				// Just use a 3x3x3 window (hit recursively if needed)
+				imin = i-1;
+				jmin = j-1;
+				kmin = k-1;
+				imax = i+2;
+				jmax = j+2;
+				kmax = k+2;
+
+				// Populate the list with values in the window
+				int Number=0;
+				for (kk=kmin; kk<kmax; kk++){
+					for (jj=jmin; jj<jmax; jj++){
+						for (ii=imin; ii<imax; ii++){
+							List[Number++] = Input(ii,jj,kk);
+						}
+					}
+				}
+				// Sort the first 5 entries and return the median
+				for (ii=0; ii<14; ii++){
+					for (jj=ii+1; jj<27; jj++){
+						if (List[jj] < List[ii]){
+							float tmp = List[ii];
+							List[ii] = List[jj];
+							List[jj] = tmp;
+						}
+					}
+				}
+				// Return the median
+				Output(i,j,k) = List[13];
+			}
+		}
+	}
+	PROFILE_STOP("Med3D");
+}
+
+
+int NLM3D( const Array<float> &Input, Array<float> &Mean, 
+    const Array<float> &Distance, Array<float> &Output, const int d, const float h)
+{
+	PROFILE_START("NLM3D");
+	// Implemenation of 3D non-local means filter
+	// 		d determines the width of the search volume
+	// 		h is a free parameter for non-local means (i.e. 1/sigma^2)
+	// 		Distance is the signed distance function
+	// 		If Distance(i,j,k) > THRESHOLD_DIST then don't compute NLM
+
+	float THRESHOLD_DIST = float(d);
+	float weight, sum;
+	int i,j,k,ii,jj,kk;
+	int imin,jmin,kmin,imax,jmax,kmax;
+	int returnCount=0;
+
+	int Nx = int(Input.size(0));
+	int Ny = int(Input.size(1));
+	int Nz = int(Input.size(2));
+
+	// Compute the local means
+	for (k=1; k<Nz-1; k++){
+		for (j=1; j<Ny-1; j++){
+			for (i=1; i<Nx-1; i++){
+
+				imin = std::max(0,i-d);
+				jmin = std::max(0,j-d);
+				kmin = std::max(0,k-d);
+				imax = std::min(Nx-1,i+d);
+				jmax = std::min(Ny-1,j+d);
+				kmax = std::min(Nz-1,k+d);
+
+				// Populate the list with values in the window
+				sum = 0; weight=0;
+				for (kk=kmin; kk<kmax; kk++){
+					for (jj=jmin; jj<jmax; jj++){
+						for (ii=imin; ii<imax; ii++){
+							sum += Input(ii,jj,kk);
+							weight++;
+						}
+					}
+				}
+
+				Mean(i,j,k) = sum / weight;
+			}
+		}
+	}
+
+	// Compute the non-local means
+	for (k=1; k<Nz-1; k++){
+		for (j=1; j<Ny-1; j++){
+			for (i=1; i<Nx-1; i++){
+
+
+				if (fabs(Distance(i,j,k)) < THRESHOLD_DIST){
+					// compute the expensive non-local means
+					sum = 0; weight=0;
+
+					imin = std::max(0,i-d);
+					jmin = std::max(0,j-d);
+					kmin = std::max(0,k-d);
+					imax = std::min(Nx-1,i+d);
+					jmax = std::min(Ny-1,j+d);
+					kmax = std::min(Nz-1,k+d);
+
+					for (kk=kmin; kk<kmax; kk++){
+						for (jj=jmin; jj<jmax; jj++){
+							for (ii=imin; ii<imax; ii++){
+								float tmp = Mean(i,j,k) - Mean(ii,jj,kk);
+								sum += exp(-tmp*tmp*h)*Input(ii,jj,kk);
+								weight += exp(-tmp*tmp*h);
+							}
+						}
+					}
+
+					returnCount++;
+					//Output(i,j,k) = Mean(i,j,k);
+					Output(i,j,k) = sum / weight;
+				}
+				else{
+					// Just return the mean
+					Output(i,j,k) = Mean(i,j,k);
+				}
+			}
+		}
+	}
+	// Return the number of sites where NLM was applied
+	PROFILE_STOP("NLM3D");
+	return returnCount;
+}

analysis/filters.h

@@ -0,0 +1,28 @@
+#ifndef Filters_H_INC
+#define Filters_H_INC
+
+
+#include "common/Array.h"
+
+
+/*!
+ * @brief  Filter image
+ * @details  This routine performs a median filter
+ * @param[in] Input     Input image
+ * @param[out] Output   Output image
+ */
+void Med3D( const Array<float> &Input, Array<float> &Output );
+
+
+/*!
+ * @brief  Filter image
+ * @details  This routine performs a non-linear local means filter
+ * @param[in] Input     Input image
+ * @param[in] Mean      Mean value
+ * @param[out] Output   Output image
+ */
+int NLM3D( const Array<float> &Input, Array<float> &Mean, 
+    const Array<float> &Distance, Array<float> &Output, const int d, const float h);
+
+
+#endif

analysis/histogram.h

@@ -0,0 +1,47 @@
+/*
+ *  Generate a histogram for volumetric, interfacial and common curve properties
+ *  copyright 2014, James E. McClure
+ */
+
+#define HISTOGRAM_RESOLUTION 1000
+
+struct Histogram{
+	Histogram(double v1, double v2){
+		data = new double[HISTOGRAM_RESOLUTION];
+		minimum = v1;
+		maximum = v2;
+		delta = (maximum-minimum)/HISTOGRAM_RESOLUTION;
+	}
+	~Histogram{
+		delete *data;
+	}
+	double *data;
+	double minimum,maximum,delta;
+	
+	// Adds value into the histogram
+	void IncludeValue(double value, double weight){
+		idx = floor((value-min)/delta);
+		if (idx > HISTOGRAM_RESOLUTION) ;
+		else if (idx < 0) ;
+		else data[idx] += weight;
+	}
+	
+	// Returns the maximum value in the histogram
+	void GetMax(){
+		max = minimum;
+		for (idx=1; idx<HISTOGRAM_RESOLUTION; idx++){
+			if (data[idx] > max){
+				max = minimum+idx*delta;
+			}
+		}
+	}
+	
+	// Resets the histogram to zero
+	void Reset(){
+		for (idx=0; idx<HISTOGRAM_RESOLUTION; idx++) data[idx] = 0.0;
+	}
+	
+private:
+	int idx;
+	double max,min;
+};

analysis/imfilter.h

@@ -0,0 +1,159 @@
+// These functions mimic the behavior of imfilter in MATLAB
+#ifndef included_imfilter
+#define included_imfilter
+
+#include "common/Utilities.h"
+#include "common/Array.h"
+#include <vector>
+
+
+namespace imfilter {
+
+
+//! enum to store the BC type
+enum class BC { fixed=0, symmetric=1, replicate=2, circular=3 };
+
+
+/*!
+ * @brief  N-D filtering of multidimensional images
+ * @details  imfilter filters the multidimensional array A with the
+ *    multidimensional filter H.  The result B has the same size and class as A.
+ * @param[in]  A            The input array (Nx,Ny,Nz)
+ * @param[in]  H            The filter (2*Nhx+1,2*Nhy+1,...)
+ * @param[in]  boundary     The boundary conditions to apply (ndim):
+ *                          fixed     - Input array values outside the bounds of the array are
+ *                                      implicitly assumed to have the value X
+ *                          symmetric - Input array values outside the bounds of the array are
+ *                                      computed by mirror-reflecting the array across the array border
+ *                          replicate - Input array values outside the bounds of the array are
+ *                                      assumed to equal the nearest array border value
+ *                          circular  - Input array values outside the bounds of the array are
+ *                                      computed by implicitly assuming the input array is periodic.
+ * @param[in]  X            The value to use for boundary conditions (only used if boundary==fixed)
+ */
+template<class TYPE>
+Array<TYPE> imfilter( const Array<TYPE>& A, const Array<TYPE>& H, const std::vector<imfilter::BC>& boundary, const TYPE X=0 );
+
+
+/*!
+ * @brief  N-D filtering of multidimensional images
+ * @details  imfilter filters the multidimensional array A with the
+ *    multidimensional filter H.  The result B has the same size and class as A.
+ * @param[in]  A            The input array (Nx,Ny,Nz)
+ * @param[in]  Nh           The size of the filter
+ * @param[in]  H            The filter function to use ( y = H(data) )
+ *                          Note that the data passed to this function will be of size 2*Nh+1
+ * @param[in]  boundary     The boundary conditions to apply (ndim):
+ *                          fixed     - Input array values outside the bounds of the array are
+ *                                      implicitly assumed to have the value X
+ *                          symmetric - Input array values outside the bounds of the array are
+ *                                      computed by mirror-reflecting the array across the array border
+ *                          replicate - Input array values outside the bounds of the array are
+ *                                      assumed to equal the nearest array border value
+ *                          circular  - Input array values outside the bounds of the array are
+ *                                      computed by implicitly assuming the input array is periodic.
+ * @param[in]  X            The value to use for boundary conditions (only used if boundary==fixed)
+ */
+template<class TYPE>
+Array<TYPE> imfilter( const Array<TYPE>& A, const std::vector<int>& Nh,
+    std::function<TYPE(const Array<TYPE>&)> H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X=0 );
+
+
+/*!
+ * @brief  N-D filtering of multidimensional images
+ * @details  imfilter filters the multidimensional array A with the
+ *    multidimensional filter H.  The result B has the same size and class as A.
+ *    This version works with separable filters and is more efficient than a single filter.
+ * @param[in]  A            The input array (Nx,Ny,Nz)
+ * @param[in]  H            The filter [2*Nhx+1,2*Nhy+1,...]
+ * @param[in]  boundary     The boundary conditions to apply (ndim):
+ *                          fixed     - Input array values outside the bounds of the array are
+ *                                      implicitly assumed to have the value X
+ *                          symmetric - Input array values outside the bounds of the array are
+ *                                      computed by mirror-reflecting the array across the array border
+ *                          replicate - Input array values outside the bounds of the array are
+ *                                      assumed to equal the nearest array border value
+ *                          circular  - Input array values outside the bounds of the array are
+ *                                      computed by implicitly assuming the input array is periodic.
+ * @param[in]  X            The value to use for boundary conditions (only used if boundary==fixed)
+ */
+template<class TYPE>
+Array<TYPE> imfilter_separable( const Array<TYPE>& A, const std::vector<Array<TYPE>>& H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X=0 );
+
+
+/*!
+ * @brief  N-D filtering of multidimensional images
+ * @details  imfilter filters the multidimensional array A with the
+ *    multidimensional filter H.  The result B has the same size and class as A.
+ *    This version works with separable filters and is more efficient than a single filter.
+ * @param[in]  A            The input array (Nx,Ny,Nz)
+ * @param[in]  H            The filter [2*Nhx+1,2*Nhy+1,...]
+ * @param[in]  boundary     The boundary conditions to apply (ndim):
+ *                          fixed     - Input array values outside the bounds of the array are
+ *                                      implicitly assumed to have the value X
+ *                          symmetric - Input array values outside the bounds of the array are
+ *                                      computed by mirror-reflecting the array across the array border
+ *                          replicate - Input array values outside the bounds of the array are
+ *                                      assumed to equal the nearest array border value
+ *                          circular  - Input array values outside the bounds of the array are
+ *                                      computed by implicitly assuming the input array is periodic.
+ * @param[in]  X            The value to use for boundary conditions (only used if boundary==fixed)
+ */
+template<class TYPE>
+Array<TYPE> imfilter_separable( const Array<TYPE>& A, const std::vector<int>& Nh,
+    std::vector<std::function<TYPE(const Array<TYPE>&)>> H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X=0 );
+
+
+/*!
+ * @brief  N-D filtering of multidimensional images
+ * @details  imfilter filters the multidimensional array A with the
+ *    multidimensional filter H.  The result B has the same size and class as A.
+ *    This version works with separable filters and is more efficient than a single filter.
+ * @param[in]  A            The input array (Nx,Ny,Nz)
+ * @param[in]  H            The filter [2*Nhx+1,2*Nhy+1,...]
+ * @param[in]  boundary     The boundary conditions to apply (ndim):
+ *                          fixed     - Input array values outside the bounds of the array are
+ *                                      implicitly assumed to have the value X
+ *                          symmetric - Input array values outside the bounds of the array are
+ *                                      computed by mirror-reflecting the array across the array border
+ *                          replicate - Input array values outside the bounds of the array are
+ *                                      assumed to equal the nearest array border value
+ *                          circular  - Input array values outside the bounds of the array are
+ *                                      computed by implicitly assuming the input array is periodic.
+ * @param[in]  X            The value to use for boundary conditions (only used if boundary==fixed)
+ */
+template<class TYPE>
+Array<TYPE> imfilter_separable( const Array<TYPE>& A, const std::vector<int>& Nh,
+    std::vector<std::function<TYPE(int, const TYPE*)>> H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X=0 );
+
+
+/**
+ * @brief Create a filter to use with imfilter
+ * @details  This function creates one of several predefined filters
+ *   to use with imfilter.  The filter will always sum to 1.
+ *   Note: this function allocates memory with the new command, the user must call delete.
+ *
+ * @param[in]  N            The stencil size in each direction
+ * @param[in]  type         The type of filter to create
+ *                          average  - Simple averaging filter
+ *                          gaussian - Gaussian filter with given standard deviation.
+ *                                     Optional argument is a double array of size ndim
+ *                                     giving the standard deviation in each direction.
+ *                                     A default value of 0.5 is used if not provided.
+ * \param[in] args          An optional argument that some of the filters use
+ */
+template<class TYPE>
+Array<TYPE> create_filter( const std::vector<int>& N, const std::string &type, const void *args = NULL );
+
+
+}
+
+
+#include "analysis/imfilter.hpp"
+
+#endif
+

analysis/imfilter.hpp

@@ -0,0 +1,378 @@
+#include "analysis/imfilter.h"
+#include "ProfilerApp.h"
+#include <math.h>
+#include <string.h>
+
+
+#define IMFILTER_INSIST INSIST
+#define IMFILTER_ASSERT ASSERT
+#define IMFILTER_ERROR  ERROR
+
+
+// Function to convert an index
+static inline int imfilter_index( int index, const int N, const imfilter::BC bc )
+{
+    if ( index < 0 || index >= N ) {
+        if ( bc == imfilter::BC::symmetric ) {
+            index = ( 2 * N - index ) % N;
+        } else if ( bc == imfilter::BC::replicate ) {
+            index = index < 0 ? 0 : N - 1;
+        } else if ( bc == imfilter::BC::circular ) {
+            index = ( index + N ) % N;
+        } else if ( bc == imfilter::BC::fixed ) {
+            index = -1;
+        }
+    }
+    return index;
+}
+
+
+// Function to copy a 1D array and pad with the appropriate BC
+template<class TYPE>
+static inline void copy_array( const int N, const int Ns, const int Nh,
+    const TYPE *A, const imfilter::BC BC, const TYPE X, TYPE *B )
+{
+    // Fill the center with a memcpy
+    for (int i=0; i<N; i++ )
+        B[i+Nh] = A[i*Ns];
+    // Fill the boundaries
+    for (int i=0; i<Nh; i++ ) {
+        int j1 = imfilter_index( -(i+1), N, BC );
+        int j2 = imfilter_index(   N+i, N, BC );
+        B[Nh-i-1] = j1==-1 ? X : B[Nh+j1];
+        B[N+Nh+i] = j2==-1 ? X : B[Nh+j2];
+    }
+}
+
+
+/********************************************************
+* Perform a 1D filter in a single direction             *
+********************************************************/
+template<class TYPE>
+static void filter_direction( int Ns, int N, int Ne, int Nh, const TYPE *H,
+    imfilter::BC boundary, TYPE X, TYPE *A )
+{
+    if ( Nh < 0 )
+        IMFILTER_ERROR("Invalid filter size");
+    if ( Nh == 0 ) {
+        for (int i=0; i<Ns*N*Ne; i++)
+            A[i] *= H[0];
+        return;
+    }
+    TYPE *tmp = new TYPE[N+2*Nh];
+    for (int j=0; j<Ne; j++) {
+        for (int i=0; i<Ns; i++) {
+            copy_array( N, Ns, Nh, &A[i+j*Ns*N], boundary, X, tmp );
+            for (int k=0; k<N; k++) {
+                TYPE tmp2 = 0;
+                for (int m=0; m<=2*Nh; m++)
+                    tmp2 += H[m] * tmp[k+m];
+                A[i+k*Ns+j*Ns*N] = tmp2;
+            }
+        }
+    }
+    delete[] tmp;
+}
+template<class TYPE>
+static void filter_direction( int Ns, int N, int Ne, int Nh,
+    std::function<TYPE(const Array<TYPE>&)> H, imfilter::BC boundary, TYPE X, TYPE *A )
+{
+    if ( Nh < 0 )
+        IMFILTER_ERROR("Invalid filter size");
+    TYPE *tmp = new TYPE[N+2*Nh];
+    Array<TYPE> tmp2(2*Nh+1);
+    for (int j=0; j<Ne; j++) {
+        for (int i=0; i<Ns; i++) {
+            copy_array( N, Ns, Nh, &A[i+j*Ns*N], boundary, X, tmp );
+            for (int k=0; k<N; k++) {
+                for (int m=0; m<=2*Nh; m++)
+                    tmp2(m) = tmp[k+m];
+                A[i+k*Ns+j*Ns*N] = H(tmp2);
+            }
+        }
+    }
+    delete[] tmp;
+}
+template<class TYPE>
+static void filter_direction( int Ns, int N, int Ne, int Nh,
+    std::function<TYPE(int, const TYPE*)> H, imfilter::BC boundary, TYPE X, TYPE *A )
+{
+    if ( Nh < 0 )
+        IMFILTER_ERROR("Invalid filter size");
+    TYPE *tmp = new TYPE[N+2*Nh];
+    int Nh2 = 2*Nh+1;
+    for (int j=0; j<Ne; j++) {
+        for (int i=0; i<Ns; i++) {
+            copy_array( N, Ns, Nh, &A[i+j*Ns*N], boundary, X, tmp );
+            for (int k=0; k<N; k++)
+                A[i+k*Ns+j*Ns*N] = H(Nh2,&tmp[k]);
+        }
+    }
+    delete[] tmp;
+}
+
+
+
+/********************************************************
+*  Create a filter                                      *
+********************************************************/
+template<class TYPE>
+Array<TYPE> imfilter::create_filter( const std::vector<int>& N0, const std::string &type, const void *args )
+{
+    std::vector<size_t> N2(N0.size());
+    for (size_t i=0; i<N2.size(); i++)
+        N2[i] = 2*N0[i]+1;
+    Array<TYPE> h(N2);
+    h.fill(0);
+    if ( type == "average" ) {
+        // average
+        h.fill( 1.0 / static_cast<TYPE>( h.length() ) );
+    } else if ( type == "gaussian" ) {
+        // gaussian
+        if ( N0.size() > 3 )
+            IMFILTER_ERROR( "Not implimented for dimensions > 3" );
+        TYPE std[3] = { 0.5, 0.5, 0.5 };
+        if ( args != NULL ) {
+            const TYPE *args2 = reinterpret_cast<const TYPE*>( args );
+            for ( size_t d = 0; d < N0.size(); d++ )
+                std[d]  = args2[d];
+        }
+        auto N = N0;
+        N.resize(3,0);
+        for ( int k = -N[2]; k <= N[2]; k++ ) {
+            for ( int j = -N[1]; j <= N[1]; j++ ) {
+                for ( int i = -N[0]; i <= N[0]; i++ ) {
+                    h(i+N[0],j+N[1],k+N[2]) = 
+                        exp( -i * i / ( 2 * std[0] * std[0] ) ) *
+                        exp( -j * j / ( 2 * std[1] * std[1] ) ) *
+                        exp( -k * k / ( 2 * std[2] * std[2] ) );
+                }
+            }
+        }
+        h.scale( 1.0/h.sum() );
+    } else {
+        IMFILTER_ERROR( "Unknown filter" );
+    }
+    return h;
+}
+
+
+// Perform 2-D filtering
+template<class TYPE>
+void imfilter_2D( int Nx, int Ny, const TYPE *A, int Nhx, int Nhy, const TYPE *H,
+    imfilter::BC BCx, imfilter::BC BCy, const TYPE X, TYPE *B )
+{
+    IMFILTER_ASSERT( A != B );
+    PROFILE_START( "imfilter_2D" );
+    memset( B, 0, Nx * Ny * sizeof( TYPE ) );
+    for ( int j1 = 0; j1 < Ny; j1++ ) {
+        for ( int i1 = 0; i1 < Nx; i1++ ) {
+            TYPE tmp = 0;
+            if ( i1 >= Nhx && i1 < Nx - Nhx && j1 >= Nhy && j1 < Ny - Nhy ) {
+                int ijkh = 0;
+                for ( int j2 = j1 - Nhy; j2 <= j1 + Nhy; j2++ ) {
+                    for ( int i2 = i1 - Nhx; i2 <= i1 + Nhx; i2++, ijkh++ )
+                        tmp += H[ijkh] * A[i2 + j2 * Nx];
+                }
+            } else {
+                int ijkh = 0;
+                for ( int jh = -Nhy; jh <= Nhy; jh++ ) {
+                    int j2 = imfilter_index( j1+jh, Ny, BCy );
+                    for ( int ih = -Nhx; ih <= Nhx; ih++ ) {
+                        int i2     = imfilter_index( i1+ih, Nx, BCx );
+                        bool fixed = i2 == -1 || j2 == -1;
+                        TYPE A2  = fixed ? X : A[i2 + j2 * Nx];
+                        tmp += H[ijkh] * A2;
+                        ijkh++;
+                    }
+                }
+            }
+            B[i1 + j1 * Nx] = tmp;
+        }
+    }
+    PROFILE_STOP( "imfilter_2D" );
+}
+
+
+// Perform 3-D filtering
+template<class TYPE>
+void imfilter_3D( int Nx, int Ny, int Nz, const TYPE *A, int Nhx, int Nhy, int Nhz,
+    const TYPE *H, imfilter::BC BCx, imfilter::BC BCy, imfilter::BC BCz,
+    const TYPE X, TYPE *B )
+{
+    IMFILTER_ASSERT( A != B );
+    PROFILE_START( "imfilter_3D" );
+    memset( B, 0, Nx * Ny * Nz * sizeof( TYPE ) );
+    for ( int k1 = 0; k1 < Nz; k1++ ) {
+        for ( int j1 = 0; j1 < Ny; j1++ ) {
+            for ( int i1 = 0; i1 < Nx; i1++ ) {
+                TYPE tmp = 0;
+                int ijkh   = 0;
+                for ( int kh = -Nhz; kh <= Nhz; kh++ ) {
+                    int k2 = imfilter_index( k1+kh, Nz, BCz );
+                    for ( int jh = -Nhy; jh <= Nhy; jh++ ) {
+                        int j2 = imfilter_index( j1+jh, Ny, BCy );
+                        for ( int ih = -Nhx; ih <= Nhx; ih++ ) {
+                            int i2     = imfilter_index( i1+ih, Nx, BCx );
+                            bool fixed = i2 == -1 || j2 == -1 || k2 == -1;
+                            TYPE A2  = fixed ? X : A[i2 + j2 * Nx + k2 * Nx * Ny];
+                            tmp += H[ijkh] * A2;
+                            ijkh++;
+                        }
+                    }
+                }
+                B[i1 + j1 * Nx + k1 * Nx * Ny] = tmp;
+            }
+        }
+    }
+    PROFILE_STOP( "imfilter_3D" );
+}
+
+
+/********************************************************
+*  Perform N-D filtering                                *
+********************************************************/
+template<class TYPE>
+Array<TYPE> imfilter::imfilter( const Array<TYPE>& A, 
+    const Array<TYPE>& H, const std::vector<imfilter::BC>& BC, const TYPE X )
+{
+    IMFILTER_ASSERT( A.ndim() == H.ndim() );
+    IMFILTER_ASSERT( A.ndim() == BC.size() );
+    std::vector<size_t> Nh = H.size();
+    for (int d=0; d<A.ndim(); d++) {
+        Nh[d] = (H.size(d)-1)/2;
+        IMFILTER_INSIST(2*Nh[d]+1==H.size(d),"Filter must be of size 2*N+1");
+    }
+    auto B = A;
+    if ( A.ndim() == 1 ) {
+        PROFILE_START( "imfilter_1D" );
+        filter_direction( 1, A.size(0), 1, Nh[0], H.data(), BC[0], X, B.data() );
+        PROFILE_STOP( "imfilter_1D" );
+    } else if ( A.ndim() == 2 ) {
+        imfilter_2D( A.size(0), A.size(1), A.data(), Nh[0], Nh[1], H.data(), BC[0], BC[1], X, B.data() );
+    } else if ( A.ndim() == 3 ) {
+        imfilter_3D( A.size(0), A.size(1), A.size(2), A.data(),
+            Nh[0], Nh[1], Nh[2], H.data(), BC[0], BC[1], BC[2], X, B.data() );
+    } else {
+        IMFILTER_ERROR( "Arbitrary dimension not yet supported" );
+    }
+    return B;
+}
+template<class TYPE>
+Array<TYPE> imfilter::imfilter( const Array<TYPE>& A, const std::vector<int>& Nh0,
+    std::function<TYPE(const Array<TYPE>&)> H,
+    const std::vector<imfilter::BC>& BC0, const TYPE X )
+{
+    PROFILE_START( "imfilter (lambda)" );
+    IMFILTER_ASSERT( A.ndim() == Nh0.size() );
+    IMFILTER_ASSERT( A.ndim() == BC0.size() );
+    std::vector<size_t> Nh2( A.size() );
+    for (int d=0; d<A.ndim(); d++)
+        Nh2[d] = 2*Nh0[d]+1;
+    auto B = A;
+    Array<TYPE> data(Nh2);
+    IMFILTER_INSIST(A.ndim()<=3,"Not programmed for more than 3 dimensions yet");
+    auto N = A.size();
+    auto Nh = Nh0;
+    auto BC = BC0;
+    N.resize(3,1);
+    Nh.resize(3,0);
+    BC.resize(3,imfilter::BC::fixed);
+    for ( int k1 = 0; k1 < N[2]; k1++ ) {
+        for ( int j1 = 0; j1 < N[1]; j1++ ) {
+            for ( int i1 = 0; i1 < N[0]; i1++ ) {
+                for ( int kh = -Nh[2]; kh <= Nh[2]; kh++ ) {
+                    int k2 = imfilter_index( k1+kh, N[2], BC[2] );
+                    for ( int jh = -Nh[1]; jh <= Nh[1]; jh++ ) {
+                        int j2 = imfilter_index( j1+jh, N[1], BC[1] );
+                        for ( int ih = -Nh[0]; ih <= Nh[0]; ih++ ) {
+                            int i2 = imfilter_index( i1+ih, N[0], BC[0] );
+                            bool fixed = i2 == -1 || j2 == -1 || k2 == -1;
+                            data(ih+Nh[0],jh+Nh[1],kh+Nh[2]) = fixed ? X : A(i2,j2,k2);
+                        }
+                    }
+                }
+                B(i1,j1,k1) = H( data );
+            }
+        }
+    }
+    PROFILE_STOP( "imfilter (lambda)" );
+    return B;
+}
+
+
+/********************************************************
+* imfilter with separable filter functions              *
+********************************************************/
+template<class TYPE>
+Array<TYPE> imfilter::imfilter_separable( const Array<TYPE>& A,
+    const std::vector<Array<TYPE>>& H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X )
+{
+    PROFILE_START( "imfilter_separable" );
+    IMFILTER_ASSERT( A.ndim() == (int) H.size() );
+    IMFILTER_ASSERT( A.ndim() == (int) boundary.size() );
+    std::vector<size_t> Nh( H.size() );
+    for (int d=0; d<A.ndim(); d++) {
+        IMFILTER_ASSERT(H[d].ndim()==1);
+        Nh[d] = (H[d].length()-1)/2;
+        IMFILTER_INSIST(2*Nh[d]+1==H[d].length(),"Filter must be of size 2*N+1");
+    }
+    auto B = A;
+    for ( int d = 0; d < A.ndim(); d++ ) {
+        int N = A.size(d);
+        int Ns = 1;
+        int Ne = 1;
+        for ( int d2 = 0; d2 < d; d2++ )
+            Ns *= A.size(d2);
+        for ( int d2 = d+1; d2 < A.ndim(); d2++ )
+            Ne *= A.size(d2);
+        filter_direction( Ns, N, Ne, Nh[d], H[d].data(), boundary[d], X, B.data() );
+    }
+    PROFILE_STOP( "imfilter_separable" );
+    return B;
+}
+template<class TYPE>
+Array<TYPE> imfilter::imfilter_separable( const Array<TYPE>& A, const std::vector<int>& Nh,
+    std::vector<std::function<TYPE(const Array<TYPE>&)>> H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X )
+{
+    PROFILE_START( "imfilter_separable (lambda)" );
+    IMFILTER_ASSERT( A.ndim() == (int) boundary.size() );
+    auto B = A;
+    for ( int d = 0; d < A.ndim(); d++ ) {
+        int N = A.size(d);
+        int Ns = 1;
+        int Ne = 1;
+        for ( int d2 = 0; d2 < d; d2++ )
+            Ns *= A.size(d2);
+        for ( int d2 = d+1; d2 < A.ndim(); d2++ )
+            Ne *= A.size(d2);
+        filter_direction( Ns, N, Ne, Nh[d], H[d], boundary[d], X, B.data() );
+    }
+    PROFILE_STOP( "imfilter_separable (lambda)" );
+    return B;
+}
+template<class TYPE>
+Array<TYPE> imfilter::imfilter_separable( const Array<TYPE>& A, const std::vector<int>& Nh,
+    std::vector<std::function<TYPE(int, const TYPE*)>> H,
+    const std::vector<imfilter::BC>& boundary, const TYPE X )
+{
+    PROFILE_START( "imfilter_separable (function)" );
+    IMFILTER_ASSERT( A.ndim() == (int) boundary.size() );
+    auto B = A;
+    for ( int d = 0; d < A.ndim(); d++ ) {
+        int N = A.size(d);
+        int Ns = 1;
+        int Ne = 1;
+        for ( int d2 = 0; d2 < d; d2++ )
+            Ns *= A.size(d2);
+        for ( int d2 = d+1; d2 < A.ndim(); d2++ )
+            Ne *= A.size(d2);
+        filter_direction( Ns, N, Ne, Nh[d], H[d], boundary[d], X, B.data() );
+    }
+    PROFILE_STOP( "imfilter_separable (function)" );
+    return B;
+}
+
+

analysis/runAnalysis.cpp

@@ -0,0 +1,598 @@
+// Run the analysis, blob identification, and write restart files
+#include "analysis/runAnalysis.h"
+#include "analysis/analysis.h"
+#include "common/Array.h"
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "common/ScaLBL.h"
+#include "IO/MeshDatabase.h"
+#include "threadpool/thread_pool.h"
+
+#include "ProfilerApp.h"
+
+
+
+AnalysisType& operator |=(AnalysisType &lhs, AnalysisType rhs)  
+{
+    lhs = static_cast<AnalysisType>(
+        static_cast<std::underlying_type<AnalysisType>::type>(lhs) |
+        static_cast<std::underlying_type<AnalysisType>::type>(rhs)
+    );
+    return lhs;
+}
+bool matches( AnalysisType x, AnalysisType y )
+{
+    return static_cast<std::underlying_type<AnalysisType>::type>(x) &
+        static_cast<std::underlying_type<AnalysisType>::type>(y) != 0;
+}
+
+
+template<class TYPE>
+void DeleteArray( const TYPE *p )
+{
+    delete [] p;
+}
+
+
+// Helper class to write the restart file from a seperate thread
+class WriteRestartWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+    WriteRestartWorkItem( const char* filename_, const DoubleArray& phase_, const DoubleArray& dist_, int N_ ):
+        filename(filename_), phase(phase_), dist(dist_), N(N_) {}
+    virtual void run() {
+        PROFILE_START("Save Checkpoint",1);
+        char *IDS;
+        IDS = new char [N];
+        char local_id=0;
+        for (int idx=0; idx<N; idx++){
+        	if (dist(idx) < 0.f ) local_id = 0; 
+        	else if (phase(idx) > 0.f) local_id = 1;
+        	else local_id=2;
+        	IDS[idx] = local_id;
+        }
+		FILE *RESTART = fopen(filename,"wb");
+		fwrite(IDS,1,N,RESTART);
+		//    fwrite(Distance.get(),8,Distance.length(),ID);
+		fclose(RESTART);
+		PROFILE_STOP("Save Checkpoint",1);
+	
+    };
+private:
+    WriteRestartWorkItem();
+    const char* filename;
+    // std::shared_ptr<double> cPhi, cDist;
+    const DoubleArray& phase;
+    const DoubleArray& dist;
+    const int N;
+};
+
+
+// Helper class to compute the blob ids
+static const std::string id_map_filename = "lbpm_id_map.txt";
+class BlobIdentificationWorkItem1: public ThreadPool::WorkItemRet<void>
+{
+public:
+    BlobIdentificationWorkItem1( int timestep_, int Nx_, int Ny_, int Nz_, const RankInfoStruct& rank_info_, 
+        std::shared_ptr<const DoubleArray> phase_, const DoubleArray& dist_,
+        BlobIDstruct last_id_, BlobIDstruct new_index_, BlobIDstruct new_id_, BlobIDList new_list_, runAnalysis::commWrapper&& comm_ ):
+        timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_),
+        phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_), comm(std::move(comm_))
+        {
+        }
+    ~BlobIdentificationWorkItem1() { }
+    virtual void run() {
+        // Compute the global blob id and compare to the previous version
+        PROFILE_START("Identify blobs",1);
+        double vF = 0.0;
+        double vS = -1.0; // one voxel buffer region around solid
+        IntArray& ids = new_index->second;
+        new_index->first = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,*phase,dist,vF,vS,ids,comm.comm);
+        PROFILE_STOP("Identify blobs",1);
+    }
+private:
+    BlobIdentificationWorkItem1();
+    int timestep;
+    int Nx, Ny, Nz;
+    const RankInfoStruct& rank_info;
+    std::shared_ptr<const DoubleArray> phase;
+    const DoubleArray& dist;
+    BlobIDstruct last_id, new_index, new_id;
+    BlobIDList new_list;
+    runAnalysis::commWrapper comm;
+};
+class BlobIdentificationWorkItem2: public ThreadPool::WorkItemRet<void>
+{
+public:
+    BlobIdentificationWorkItem2( int timestep_, int Nx_, int Ny_, int Nz_, const RankInfoStruct& rank_info_, 
+        std::shared_ptr<const DoubleArray> phase_, const DoubleArray& dist_,
+        BlobIDstruct last_id_, BlobIDstruct new_index_, BlobIDstruct new_id_, BlobIDList new_list_ , runAnalysis::commWrapper&& comm_ ):
+        timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_),
+        phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_), comm(std::move(comm_))
+        {
+        }
+    ~BlobIdentificationWorkItem2() { }
+    virtual void run() {
+        // Compute the global blob id and compare to the previous version
+        PROFILE_START("Identify blobs maps",1);
+        const IntArray& ids = new_index->second;
+        static int max_id = -1;
+        new_id->first = new_index->first;
+        new_id->second = new_index->second;
+        if ( last_id.get()!=NULL ) {
+            // Compute the timestep-timestep map
+            const IntArray& old_ids = last_id->second;
+            ID_map_struct map = computeIDMap(Nx,Ny,Nz,old_ids,ids,comm.comm);
+            // Renumber the current timestep's ids
+            getNewIDs(map,max_id,*new_list);
+            renumberIDs(*new_list,new_id->second);
+            writeIDMap(map,timestep,id_map_filename);
+        } else {
+            max_id = -1;
+            ID_map_struct map(new_id->first);
+            getNewIDs(map,max_id,*new_list);
+            writeIDMap(map,timestep,id_map_filename);
+        }
+        PROFILE_STOP("Identify blobs maps",1);
+    }
+private:
+    BlobIdentificationWorkItem2();
+    int timestep;
+    int Nx, Ny, Nz;
+    const RankInfoStruct& rank_info;
+    std::shared_ptr<const DoubleArray> phase;
+    const DoubleArray& dist;
+    BlobIDstruct last_id, new_index, new_id;
+    BlobIDList new_list;
+    runAnalysis::commWrapper comm;
+};
+
+
+// Helper class to write the vis file from a thread
+class WriteVisWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+    WriteVisWorkItem( int timestep_, std::vector<IO::MeshDataStruct>& visData_,
+        TwoPhase& Avgerages_, fillHalo<double>& fillData_, runAnalysis::commWrapper&& comm_ ):
+        timestep(timestep_), visData(visData_), Averages(Avgerages_), fillData(fillData_), comm(std::move(comm_))
+        {
+        }
+    ~WriteVisWorkItem() { }
+    virtual void run() {
+        PROFILE_START("Save Vis",1);
+        ASSERT(visData[0].vars[0]->name=="phase");
+        ASSERT(visData[0].vars[1]->name=="Pressure");
+        ASSERT(visData[0].vars[2]->name=="SignDist");
+        ASSERT(visData[0].vars[3]->name=="BlobID");
+        Array<double>& PhaseData = visData[0].vars[0]->data;
+        Array<double>& PressData = visData[0].vars[1]->data;
+        Array<double>& SignData  = visData[0].vars[2]->data;
+        Array<double>& BlobData  = visData[0].vars[3]->data;
+        fillData.copy(Averages.SDn,PhaseData);
+        fillData.copy(Averages.Press,PressData);
+        fillData.copy(Averages.SDs,SignData);
+        fillData.copy(Averages.Label_NWP,BlobData);
+        IO::writeData( timestep, visData, comm.comm );
+        PROFILE_STOP("Save Vis",1);
+    };
+private:
+    WriteVisWorkItem();
+    int timestep;
+    std::vector<IO::MeshDataStruct>& visData;
+    TwoPhase& Averages;
+    fillHalo<double>& fillData;
+    runAnalysis::commWrapper comm;
+};
+
+
+// Helper class to run the analysis from within a thread
+// Note: Averages will be modified after the constructor is called
+class AnalysisWorkItem: public ThreadPool::WorkItemRet<void>
+{
+public:
+    AnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_, 
+        BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
+        type(type_), timestep(timestep_), Averages(Averages_), 
+        blob_ids(ids), id_list(id_list_), beta(beta_) { }
+    ~AnalysisWorkItem() { }
+    virtual void run() {
+        Averages.NumberComponents_NWP = blob_ids->first;
+        Averages.Label_NWP = blob_ids->second;
+        Averages.Label_NWP_map = *id_list;
+        Averages.NumberComponents_WP = 1;
+        Averages.Label_WP.fill(0.0);
+        if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
+            // Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
+        }
+        if ( matches(type,AnalysisType::ComputeAverages) ) {
+            PROFILE_START("Compute dist",1);
+            Averages.Initialize();
+            Averages.ComputeDelPhi();
+            Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.SDn);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tminus,Averages.Phase_tminus);
+            Averages.ColorToSignedDistance(beta,Averages.Phase_tplus,Averages.Phase_tplus);
+            Averages.UpdateMeshValues();
+            Averages.ComputeLocal();
+            Averages.Reduce();
+            Averages.PrintAll(timestep);
+            Averages.Initialize();
+            Averages.ComponentAverages();
+            Averages.SortBlobs();
+            Averages.PrintComponents(timestep);
+            PROFILE_STOP("Compute dist",1);
+        }
+    }
+private:
+    AnalysisWorkItem();
+    AnalysisType type;
+    int timestep;
+    TwoPhase& Averages;
+    BlobIDstruct blob_ids;
+    BlobIDList id_list;
+    double beta;
+};
+
+
+/******************************************************************
+ *  MPI comm wrapper for use with analysis                         *
+ ******************************************************************/
+runAnalysis::commWrapper::commWrapper( int tag_, MPI_Comm comm_, runAnalysis* analysis_ ):
+    tag(tag_),
+    comm(comm_),
+    analysis(analysis_)
+{
+}
+runAnalysis::commWrapper::commWrapper( commWrapper &&rhs ):
+    tag(rhs.tag),
+    comm(rhs.comm),
+    analysis(rhs.analysis)
+{
+    rhs.tag = -1;
+}
+runAnalysis::commWrapper::~commWrapper()
+{
+    if ( tag == -1 )
+        return;
+    MPI_Barrier( comm );
+    analysis->d_comm_used[tag] = false;
+}
+runAnalysis::commWrapper runAnalysis::getComm( )
+{
+    // Get a tag from root
+    int tag = -1;
+    if ( d_rank == 0 ) {
+        for (int i=0; i<1024; i++) {
+            if ( !d_comm_used[i] ) {
+                tag = i;
+                break;
+            }
+        }
+        if ( tag == -1 )
+            ERROR("Unable to get comm");
+    }
+    MPI_Bcast( &tag, 1, MPI_INT, 0, d_comm );
+    d_comm_used[tag] = true;
+    if ( d_comms[tag] == MPI_COMM_NULL )
+        MPI_Comm_dup( MPI_COMM_WORLD, &d_comms[tag] );
+    return commWrapper(tag,d_comms[tag],this);
+}
+
+
+/******************************************************************
+ *  Constructor/Destructors                                        *
+ ******************************************************************/
+runAnalysis::runAnalysis( int restart_interval, int analysis_interval,
+    int blobid_interval, const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& Dm,
+    int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
+    IntArray Map, const std::string& LocalRestartFile ):
+    d_Np( Np ),
+    d_restart_interval( restart_interval ),
+    d_analysis_interval( analysis_interval ),
+    d_blobid_interval( blobid_interval ),
+    d_beta( beta ),
+    d_ScaLBL_Comm( ScaLBL_Comm ),
+    d_rank_info( rank_info ),
+    d_Map( Map ),
+    d_fillData(Dm.Comm,Dm.rank_info,Nx-2,Ny-2,Nz-2,1,1,1,0,1),
+    d_restartFile( LocalRestartFile )
+{
+    d_N[0] = Nx;
+    d_N[1] = Ny;
+    d_N[2] = Nz;
+    d_rank = MPI_WORLD_RANK();
+	writeIDMap(ID_map_struct(),0,id_map_filename);
+	// Create the MeshDataStruct
+	d_meshData.resize(1);
+	d_meshData[0].meshName = "domain";
+	d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( Dm.rank_info,Nx-2,Ny-2,Nz-2,Lx,Ly,Lz );
+	auto PhaseVar = std::make_shared<IO::Variable>();
+	auto PressVar = std::make_shared<IO::Variable>();
+	auto SignDistVar = std::make_shared<IO::Variable>();
+	auto BlobIDVar = std::make_shared<IO::Variable>();
+	PhaseVar->name = "phase";
+	PhaseVar->type = IO::VariableType::VolumeVariable;
+	PhaseVar->dim = 1;
+	PhaseVar->data.resize(Nx-2,Ny-2,Nz-2);
+	d_meshData[0].vars.push_back(PhaseVar);
+	PressVar->name = "Pressure";
+	PressVar->type = IO::VariableType::VolumeVariable;
+	PressVar->dim = 1;
+	PressVar->data.resize(Nx-2,Ny-2,Nz-2);
+	d_meshData[0].vars.push_back(PressVar);
+	SignDistVar->name = "SignDist";
+	SignDistVar->type = IO::VariableType::VolumeVariable;
+	SignDistVar->dim = 1;
+	SignDistVar->data.resize(Nx-2,Ny-2,Nz-2);
+	d_meshData[0].vars.push_back(SignDistVar);
+	BlobIDVar->name = "BlobID";
+	BlobIDVar->type = IO::VariableType::VolumeVariable;
+	BlobIDVar->dim = 1;
+	BlobIDVar->data.resize(Nx-2,Ny-2,Nz-2);
+	d_meshData[0].vars.push_back(BlobIDVar);
+    // Initialize the comms
+    MPI_Comm_dup(MPI_COMM_WORLD,&d_comm);
+    for (int i=0; i<1024; i++) {
+        d_comms[i] = MPI_COMM_NULL;
+        d_comm_used[i] = false;
+    }
+}
+runAnalysis::~runAnalysis( )
+{
+    // Finish processing analysis
+    finish();
+    // Clear internal data
+    MPI_Comm_free( &d_comm );
+    for (int i=0; i<1024; i++) {
+        if ( d_comms[i] != MPI_COMM_NULL )
+            MPI_Comm_free(&d_comms[i]);
+    }
+}
+void runAnalysis::finish( )
+{
+    PROFILE_START("finish");
+    // Wait for the work items to finish
+	d_tpool.wait_pool_finished();
+    // Clear the wait ids
+    d_wait_blobID.reset();
+    d_wait_analysis.reset();
+    d_wait_vis.reset();
+    d_wait_restart.reset();
+    // Syncronize
+    MPI_Barrier( d_comm );
+    PROFILE_STOP("finish");
+}
+
+
+/******************************************************************
+ *  Set the thread affinities                                      *
+ ******************************************************************/
+void print( const std::vector<int>& ids )
+{
+    if ( ids.empty() )
+        return;
+    printf("%i",ids[0]);
+    for (size_t i=1; i<ids.size(); i++)
+        printf(", %i",ids[i]);
+    printf("\n");
+}
+void runAnalysis::createThreads( const std::string& method, int N_threads )
+{
+    // Check if we are not using analysis threads
+	if ( method == "none" )
+        return;
+    // Check if we have thread support
+    int thread_support;
+    MPI_Query_thread( &thread_support );
+	if ( thread_support < MPI_THREAD_MULTIPLE ) {
+		std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
+        return;
+    }
+    // Create the threads
+    const auto cores = d_tpool.getProcessAffinity();
+    if ( cores.empty() ) {
+        // We were not able to get the cores for the process
+        d_tpool.setNumThreads( N_threads );
+    } else if ( method == "default" ) {
+        // Create the given number of threads, but let the OS manage affinities
+        d_tpool.setNumThreads( N_threads );
+    } else if ( method == "independent" ) {
+        int N = cores.size() - 1;
+        d_tpool.setNumThreads( N );
+        d_tpool.setThreadAffinity( { cores[0] } );
+        for ( int i=0; i<N; i++)
+            d_tpool.setThreadAffinity( i, { cores[i+1] } );
+    }
+    // Print the current affinities
+    if ( d_rank == 0 ) {
+        printf("Affinities - rank 0:\n");
+        printf("Main: ");
+        print(d_tpool.getProcessAffinity());
+        for (int i=0; i<d_tpool.getNumThreads(); i++) {
+            printf("Thread %i: ",i+1);
+            print(d_tpool.getThreadAffinity(i));
+        }
+    }
+}
+
+
+/******************************************************************
+ *  Check which analysis we want to perform                        *
+ ******************************************************************/
+AnalysisType runAnalysis::computeAnalysisType( int timestep )
+{
+    AnalysisType type = AnalysisType::AnalyzeNone;
+    if ( timestep%d_analysis_interval + 8 == d_analysis_interval ) {
+        // Copy the phase indicator field for the earlier timestep
+        type |= AnalysisType::CopyPhaseIndicator;
+    }
+    if ( timestep%d_blobid_interval == 0 ) {
+        // Identify blobs and update global ids in time
+        type |= AnalysisType::IdentifyBlobs;
+    }
+    /*#ifdef USE_CUDA
+        if ( tpool.getQueueSize()<=3 && tpool.getNumThreads()>0 && timestep%50==0 ) {
+            // Keep a few blob identifications queued up to keep the processors busy,
+            // allowing us to track the blobs as fast as possible
+            // Add more detailed estimates of the update frequency required to track blobs
+            type |= AnalysisType::IdentifyBlobs;
+        }
+    #endif */
+    if ( timestep%d_analysis_interval + 4 == 0 ) {
+        // Copy the averages to the CPU (and identify blobs)
+        type |= AnalysisType::CopySimState;
+        type |= AnalysisType::IdentifyBlobs;
+    }
+    if ( timestep%d_analysis_interval == 0 ) {
+        // Run the analysis
+        type |= AnalysisType::ComputeAverages;
+    }
+    if (timestep%d_restart_interval == 0) {
+        // Write the restart file
+        type |= AnalysisType::CreateRestart;
+    }
+    if (timestep%d_restart_interval == 0) {
+        // Write the visualization data
+        type |= AnalysisType::WriteVis;
+        type |= AnalysisType::CopySimState;
+        type |= AnalysisType::IdentifyBlobs;
+    }
+    return type;
+}
+
+
+
+/******************************************************************
+ *  Run the analysis                                               *
+ ******************************************************************/
+void runAnalysis::run( int timestep, TwoPhase& Averages, const double *Phi,
+    double *Pressure, double *Velocity, double *fq, double *Den )
+{
+    int N = d_N[0]*d_N[1]*d_N[2];
+
+    // Check which analysis steps we need to perform
+    auto type = computeAnalysisType( timestep );
+    if ( type == AnalysisType::AnalyzeNone )
+        return;
+
+    // Check how may queued items we have
+    if ( d_tpool.N_queued() > 20 ) {
+        std::cerr << "Analysis queue is getting behind, waiting ...\n";
+        finish();
+    }
+
+    PROFILE_START("run");
+
+    // Copy the appropriate variables to the host (so we can spawn new threads)
+    ScaLBL_DeviceBarrier();
+    PROFILE_START("Copy data to host",1);
+    std::shared_ptr<DoubleArray> phase;
+    if ( matches(type,AnalysisType::CopyPhaseIndicator) ||
+         matches(type,AnalysisType::ComputeAverages) ||
+         matches(type,AnalysisType::CopySimState) || 
+         matches(type,AnalysisType::IdentifyBlobs) )
+    {
+        phase = std::shared_ptr<DoubleArray>(new DoubleArray(d_N[0],d_N[1],d_N[2]));
+        ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double));
+    }
+    if ( matches(type,AnalysisType::CopyPhaseIndicator) ) {
+        memcpy(Averages.Phase_tplus.data(),phase->data(),N*sizeof(double));
+        //Averages.ColorToSignedDistance(d_beta,Averages.Phase,Averages.Phase_tplus);
+    }
+    if ( matches(type,AnalysisType::ComputeAverages) ) {
+        memcpy(Averages.Phase_tminus.data(),phase->data(),N*sizeof(double));
+        //Averages.ColorToSignedDistance(d_beta,Averages.Phase,Averages.Phase_tminus);
+    }
+    if ( matches(type,AnalysisType::CopySimState) ) {
+        // Copy the members of Averages to the cpu (phase was copied above)
+        PROFILE_START("Copy-Pressure",1);
+        ScaLBL_D3Q19_Pressure(fq,Pressure,d_Np);
+        ScaLBL_D3Q19_Momentum(fq,Velocity,d_Np);
+        ScaLBL_DeviceBarrier();
+        PROFILE_STOP("Copy-Pressure",1);
+        PROFILE_START("Copy-Wait",1);
+        PROFILE_STOP("Copy-Wait",1);
+        PROFILE_START("Copy-State",1);
+        memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double));
+        d_ScaLBL_Comm.RegularLayout(d_Map,Pressure,Averages.Press);
+        d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[0],Averages.Vel_x);
+        d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[d_Np],Averages.Vel_y);
+        d_ScaLBL_Comm.RegularLayout(d_Map,&Velocity[2*d_Np],Averages.Vel_z);
+        PROFILE_STOP("Copy-State",1);
+    }
+    std::shared_ptr<double> cDen, cfq;
+    if ( matches(type,AnalysisType::CreateRestart) ) {
+        // Copy restart data to the CPU
+        //cDen = std::shared_ptr<double>(new double[2*d_Np],DeleteArray<double>);
+        //cfq = std::shared_ptr<double>(new double[19*d_Np],DeleteArray<double>);
+        //ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double));
+        //ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double));
+        //cPhi = std::shared_ptr<double>(new double[N],DeleteArray<double>);
+        //cDist = std::shared_ptr<double>(new double[N],DeleteArray<double>);
+        //ScaLBL_CopyToHost(cfq.get(),fq,19*d_Np*sizeof(double));
+        //ScaLBL_CopyToHost(cDen.get(),Den,2*d_Np*sizeof(double));
+    }
+    PROFILE_STOP("Copy data to host",1);
+
+    // Spawn threads to do blob identification work
+    if ( matches(type,AnalysisType::IdentifyBlobs) ) {
+        BlobIDstruct new_index(new std::pair<int,IntArray>(0,IntArray()));
+        BlobIDstruct new_ids(new std::pair<int,IntArray>(0,IntArray()));
+        BlobIDList new_list(new std::vector<BlobIDType>());
+        auto work1 = new BlobIdentificationWorkItem1(timestep,d_N[0],d_N[1],d_N[2],d_rank_info,
+            phase,Averages.SDs,d_last_ids,new_index,new_ids,new_list,getComm());
+        auto work2 = new BlobIdentificationWorkItem2(timestep,d_N[0],d_N[1],d_N[2],d_rank_info,
+            phase,Averages.SDs,d_last_ids,new_index,new_ids,new_list,getComm());
+        work1->add_dependency(d_wait_blobID);
+        work2->add_dependency(d_tpool.add_work(work1));
+        d_wait_blobID = d_tpool.add_work(work2);
+        d_last_index = new_index;
+        d_last_ids = new_ids;
+        d_last_id_map = new_list;
+    }
+
+    // Spawn threads to do the analysis work
+    if ( matches(type,AnalysisType::ComputeAverages) ) {
+        auto work = new AnalysisWorkItem(type,timestep,Averages,d_last_index,d_last_id_map,d_beta);
+        work->add_dependency(d_wait_blobID);
+        work->add_dependency(d_wait_analysis);
+        work->add_dependency(d_wait_vis);     // Make sure we are done using analysis before modifying
+        d_wait_analysis = d_tpool.add_work(work);
+    }
+
+    // Spawn a thread to write the restart file
+    //    if ( matches(type,AnalysisType::CreateRestart) ) {
+    if (timestep%d_restart_interval==0){
+        /* if (pBC) {
+            err = fabs(sat_w - sat_w_previous);
+            sat_w_previous = sat_w;
+            if (rank==0){
+               printf("Timestep %i: change in saturation since last checkpoint is %f \n",timestep,err);
+           }
+        } */
+        // Retain the timestep associated with the restart files
+        if (d_rank==0) {
+            FILE *Rst = fopen("Restart.txt","w");
+            fprintf(Rst,"%i\n",timestep+4);
+            fclose(Rst);
+        }
+        // Write the restart file (using a seperate thread)
+        auto work = new WriteRestartWorkItem(d_restartFile.c_str(),*phase,Averages.SDs,N);
+        work->add_dependency(d_wait_restart);
+        d_wait_restart = d_tpool.add_work(work);
+    }
+
+    // Save the results for visualization
+    //    if ( matches(type,AnalysisType::CreateRestart) ) {
+    if (timestep%d_restart_interval==0){
+        // Write the vis files
+        auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, d_fillData, getComm() );
+        work->add_dependency(d_wait_blobID);
+        work->add_dependency(d_wait_analysis);
+        work->add_dependency(d_wait_vis);
+        d_wait_vis = d_tpool.add_work(work);
+    }
+    PROFILE_STOP("run");
+}
+
+
+

analysis/runAnalysis.h

@@ -0,0 +1,116 @@
+#ifndef RunAnalysis_H_INC
+#define RunAnalysis_H_INC
+
+#include "analysis/analysis.h"
+#include "analysis/TwoPhase.h"
+#include "common/Communication.h"
+#include "common/ScaLBL.h"
+#include "threadpool/thread_pool.h"
+
+
+typedef std::shared_ptr<std::pair<int,IntArray>> BlobIDstruct;
+typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
+
+
+// Types of analysis
+enum class AnalysisType : uint64_t { AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02, 
+    CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20 };
+
+
+//! Class to run the analysis in multiple threads
+class runAnalysis
+{
+public:
+
+    //! Constructor
+    runAnalysis( int restart_interval, int analysis_interval, int blobid_interval,
+        const RankInfoStruct& rank_info, const ScaLBL_Communicator &ScaLBL_Comm, const Domain& dm,
+        int Np, int Nx, int Ny, int Nz, double Lx, double Ly, double Lz, bool pBC, double beta, double err,
+        IntArray Map, const std::string& LocalRestartFile );
+
+    //! Destructor
+    ~runAnalysis();
+
+    //! Run the next analysis
+    void run( int timestep, TwoPhase& Averages, const double *Phi,
+        double *Pressure, double *Velocity, double *fq, double *Den );
+
+    //! Finish all active analysis
+    void finish();
+
+    /*!
+     *  \brief    Set the affinities
+     *  \details  This function will create the analysis threads and set the affinity
+     *      of this thread and all analysis threads.  If MPI_THREAD_MULTIPLE is not
+     *      enabled, the analysis threads will be disabled and the analysis will run in the current thread.
+     * @param[in] method    Method used to control the affinities:
+     *                      none - Don't use threads (runs all analysis in the current thread)
+     *                      default - Create the specified number of threads, but don't load balance
+     *                      independent - Create the necessary number of threads to fill all cpus,
+     *                                and set the affinities based on the current process such
+     *                                that all threads run on independent cores
+     * @param[in] N_threads Number of threads, only used by some of the methods
+     */
+    void createThreads( const std::string& method = "default", int N_threads = 4 );
+
+
+private:
+
+    runAnalysis();
+
+    // Determine the analysis to perform
+    AnalysisType computeAnalysisType( int timestep );
+
+public:
+
+    class commWrapper
+    {
+      public:
+        MPI_Comm comm;
+        int tag;
+        runAnalysis *analysis;
+        commWrapper( int tag, MPI_Comm comm, runAnalysis *analysis );
+        commWrapper( ) = delete;
+        commWrapper( const commWrapper &rhs ) = delete;
+        commWrapper& operator=( const commWrapper &rhs ) = delete;
+        commWrapper( commWrapper &&rhs );
+        ~commWrapper();
+    };
+
+    // Get a comm (not thread safe)
+    commWrapper getComm( );
+
+private:
+
+    int d_N[3];
+    int d_Np;
+    int d_rank;
+    int d_restart_interval, d_analysis_interval, d_blobid_interval;
+    double d_beta;
+    ThreadPool d_tpool;
+    ScaLBL_Communicator d_ScaLBL_Comm;
+    RankInfoStruct d_rank_info;
+    IntArray d_Map;
+    BlobIDstruct d_last_ids;
+    BlobIDstruct d_last_index;
+    BlobIDList d_last_id_map;
+    std::vector<IO::MeshDataStruct> d_meshData;
+    fillHalo<double> d_fillData;
+    std::string d_restartFile;
+    MPI_Comm d_comm;
+    MPI_Comm d_comms[1024];
+    volatile bool d_comm_used[1024];
+
+    // Ids of work items to use for dependencies
+    ThreadPool::thread_id_t d_wait_blobID;
+    ThreadPool::thread_id_t d_wait_analysis;
+    ThreadPool::thread_id_t d_wait_vis;
+    ThreadPool::thread_id_t d_wait_restart;
+
+    // Friends
+    friend commWrapper::~commWrapper();
+
+};
+
+#endif
+

analysis/uCT.cpp

@@ -0,0 +1,394 @@
+#include "analysis/uCT.h"
+#include "analysis/analysis.h"
+#include "analysis/eikonal.h"
+#include "analysis/filters.h"
+#include "analysis/imfilter.h"
+
+
+template<class T>
+inline int sign( T x )
+{
+    if ( x==0 )
+        return 0;
+    return x>0 ? 1:-1;
+}
+
+
+inline float trilinear( float dx, float dy, float dz, float f1, float f2,
+    float f3, float f4, float f5, float f6, float f7, float f8 )
+{
+    double f, dx2, dy2, dz2, h0, h1;
+    dx2 = 1.0 - dx;
+    dy2 = 1.0 - dy;
+    dz2 = 1.0 - dz;
+    h0  = ( dx * f2 + dx2 * f1 ) * dy2 + ( dx * f4 + dx2 * f3 ) * dy;
+    h1  = ( dx * f6 + dx2 * f5 ) * dy2 + ( dx * f8 + dx2 * f7 ) * dy;
+    f   = h0 * dz2 + h1 * dz;
+    return ( f );
+}
+
+
+void InterpolateMesh( const Array<float> &Coarse, Array<float> &Fine )
+{
+	PROFILE_START("InterpolateMesh");
+
+	// Interpolate values from a Coarse mesh to a fine one
+	// This routine assumes cell-centered meshes with 1 ghost cell
+
+	// Fine mesh
+	int Nx = int(Fine.size(0))-2;
+	int Ny = int(Fine.size(1))-2;
+	int Nz = int(Fine.size(2))-2;
+
+	// Coarse mesh
+	int nx = int(Coarse.size(0))-2;
+	int ny = int(Coarse.size(1))-2;
+	int nz = int(Coarse.size(2))-2;
+
+	// compute the stride
+	int hx = Nx/nx;
+	int hy = Ny/ny;
+	int hz = Nz/nz;
+    ASSERT(nx*hx==Nx);
+    ASSERT(ny*hy==Ny);
+    ASSERT(nz*hz==Nz);
+
+	// value to map distance between meshes (since distance is in voxels)
+	//  usually hx=hy=hz (or something very close)
+	//  the mapping is not exact
+	//  however, it's assumed the coarse solution will be refined
+	//  a good guess is the goal here!
+	float mapvalue = sqrt(hx*hx+hy*hy+hz*hz);
+
+	// Interpolate to the fine mesh
+	for (int k=-1; k<Nz+1; k++){
+        int k0 = floor((k-0.5*hz)/hz);
+        int k1 = k0+1;
+        int k2 = k0+2;
+        float dz = ( (k+0.5) - (k0+0.5)*hz ) / hz;
+        ASSERT(k0>=-1&&k0<nz+1&&dz>=0&&dz<=1);
+		for (int j=-1; j<Ny+1; j++){
+            int j0 = floor((j-0.5*hy)/hy);
+            int j1 = j0+1;
+            int j2 = j0+2;
+            float dy = ( (j+0.5) - (j0+0.5)*hy ) / hy;
+            ASSERT(j0>=-1&&j0<ny+1&&dy>=0&&dy<=1);
+			for (int i=-1; i<Nx+1; i++){
+                int i0 = floor((i-0.5*hx)/hx);
+                int i1 = i0+1;
+                int i2 = i0+2;
+                float dx = ( (i+0.5) - (i0+0.5)*hx ) / hx;
+                ASSERT(i0>=-1&&i0<nx+1&&dx>=0&&dx<=1);
+                float val = trilinear( dx, dy, dz,
+                    Coarse(i1,j1,k1), Coarse(i2,j1,k1), Coarse(i1,j2,k1), Coarse(i2,j2,k1),
+                    Coarse(i1,j1,k2), Coarse(i2,j1,k2), Coarse(i1,j2,k2), Coarse(i2,j2,k2) );
+                Fine(i+1,j+1,k+1) = mapvalue*val;
+			}
+		}
+	}
+	PROFILE_STOP("InterpolateMesh");
+}
+
+
+// Smooth the data using the distance
+void smooth( const Array<float>& VOL, const Array<float>& Dist, float sigma, Array<float>& MultiScaleSmooth, fillHalo<float>& fillFloat )
+{
+    for (size_t i=0; i<VOL.length(); i++) {
+		// use exponential weight based on the distance
+		float dst = Dist(i);
+		float tmp = exp(-(dst*dst)/(sigma*sigma));
+		float value = dst>0 ? -1:1;
+		MultiScaleSmooth(i) = tmp*VOL(i) + (1-tmp)*value;
+	}
+	fillFloat.fill(MultiScaleSmooth);
+}
+
+
+// Segment the data
+void segment( const Array<float>& data, Array<char>& ID, float tol )
+{
+    ASSERT(data.size()==ID.size());
+    for (size_t i=0; i<data.length(); i++) {
+        if ( data(i) > tol )
+            ID(i) = 0;
+        else
+            ID(i) = 1;
+    }
+}
+
+
+// Remove disconnected phases
+void removeDisconnected( Array<char>& ID, const Domain& Dm )
+{
+    // Run blob identification to remove disconnected volumes
+    BlobIDArray GlobalBlobID;
+    DoubleArray SignDist(ID.size());
+    DoubleArray Phase(ID.size());
+    for (size_t i=0; i<ID.length(); i++) {
+        SignDist(i) = (2*ID(i)-1);
+        Phase(i) = 1;
+    }
+    ComputeGlobalBlobIDs( ID.size(0)-2, ID.size(1)-2, ID.size(2)-2,
+        Dm.rank_info, Phase, SignDist, 0, 0, GlobalBlobID, Dm.Comm );
+    for (size_t i=0; i<ID.length(); i++) {
+        if ( GlobalBlobID(i) > 0 )
+            ID(i) = 0;
+        ID(i) = GlobalBlobID(i);
+    }
+}
+
+
+// Solve a level (without any coarse level information)
+void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
+    Array<float>& Dist, Array<float>& MultiScaleSmooth, Array<float>& NonLocalMean, 
+    fillHalo<float>& fillFloat, const Domain& Dm, int nprocx )
+{
+    PROFILE_SCOPED(timer,"solve");
+    // Compute the median filter on the sparse array
+    Med3D( VOL, Mean );
+    fillFloat.fill( Mean );
+    segment( Mean, ID, 0.01 );
+    // Compute the distance using the segmented volume
+	Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx );
+	fillFloat.fill(Dist);
+    smooth( VOL, Dist, 2.0, MultiScaleSmooth, fillFloat );
+    // Compute non-local mean
+	int depth = 5;
+	float sigsq=0.1;
+	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+	fillFloat.fill(NonLocalMean);
+}
+
+
+// Refine a solution from a coarse grid to a fine grid
+void refine( const Array<float>& Dist_coarse, 
+    const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
+    Array<float>& Dist, Array<float>& MultiScaleSmooth, Array<float>& NonLocalMean, 
+    fillHalo<float>& fillFloat, const Domain& Dm, int nprocx, int level )
+{
+    PROFILE_SCOPED(timer,"refine");
+    int ratio[3] = { int(Dist.size(0)/Dist_coarse.size(0)),
+                     int(Dist.size(1)/Dist_coarse.size(1)),
+                     int(Dist.size(2)/Dist_coarse.size(2)) };
+    // Interpolate the distance from the coarse to fine grid
+	InterpolateMesh( Dist_coarse, Dist );
+    // Compute the median filter on the array and segment
+    Med3D( VOL, Mean );
+    fillFloat.fill( Mean );
+    segment( Mean, ID, 0.01 );
+    // If the ID has the wrong distance, set the distance to 0 and run a simple filter to set neighbors to 0
+    for (size_t i=0; i<ID.length(); i++) {
+        char id = Dist(i)>0 ? 1:0;
+        if ( id != ID(i) )
+            Dist(i) = 0;
+    }
+    fillFloat.fill( Dist );
+    std::function<float(int,const float*)> filter_1D = []( int N, const float* data )
+    {
+        bool zero = data[0]==0 || data[2]==0;
+        return zero ? data[1]*1e-12 : data[1];
+    };
+    std::vector<imfilter::BC> BC(3,imfilter::BC::replicate);
+    std::vector<std::function<float(int,const float*)>> filter_set(3,filter_1D);
+    Dist = imfilter::imfilter_separable<float>( Dist, {1,1,1}, filter_set, BC );
+    fillFloat.fill( Dist );
+    // Smooth the volume data
+    float lambda = 2*sqrt(double(ratio[0]*ratio[0]+ratio[1]*ratio[1]+ratio[2]*ratio[2]));
+    smooth( VOL, Dist, lambda, MultiScaleSmooth, fillFloat );
+    // Compute non-local mean
+	int depth = 3;
+	float sigsq = 0.1;
+	int nlm_count = NLM3D( MultiScaleSmooth, Mean, Dist, NonLocalMean, depth, sigsq);
+	fillFloat.fill(NonLocalMean);
+    segment( NonLocalMean, ID, 0.001 );
+    for (size_t i=0; i<ID.length(); i++) {
+        char id = Dist(i)>0 ? 1:0;
+        if ( id!=ID(i) || fabs(Dist(i))<1 )
+    	    Dist(i) = 2.0*ID(i)-1.0;
+    }
+    // Remove disconnected domains
+    //removeDisconnected( ID, Dm );
+    // Compute the distance using the segmented volume
+    if ( level > 0 ) {
+    	//Eikonal3D( Dist, ID, Dm, ID.size(0)*nprocx );
+        //CalcDist3D( Dist, ID, Dm );
+        CalcDistMultiLevel( Dist, ID, Dm );
+	    fillFloat.fill(Dist);
+    }
+}
+
+
+// Remove regions that are likely noise by shrinking the volumes by dx,
+// removing all values that are more than dx+delta from the surface, and then
+// growing by dx+delta and intersecting with the original data
+void filter_final( Array<char>& ID, Array<float>& Dist,
+    fillHalo<float>& fillFloat, const Domain& Dm,
+    Array<float>& Mean, Array<float>& Dist1, Array<float>& Dist2 )
+{
+    PROFILE_SCOPED(timer,"filter_final");
+	int rank;
+	MPI_Comm_rank(Dm.Comm,&rank);
+    int Nx = Dm.Nx-2;
+    int Ny = Dm.Ny-2;
+    int Nz = Dm.Nz-2;
+    // Calculate the distance
+    CalcDistMultiLevel( Dist, ID, Dm );
+    fillFloat.fill(Dist);
+    // Compute the range to shrink the volume based on the L2 norm of the distance
+    Array<float> Dist0(Nx,Ny,Nz);
+    fillFloat.copy(Dist,Dist0);
+    float tmp = 0;
+    for (size_t i=0; i<Dist0.length(); i++)
+        tmp += Dist0(i)*Dist0(i);
+    tmp = sqrt( sumReduce(Dm.Comm,tmp) / sumReduce(Dm.Comm,(float)Dist0.length()) );
+    const float dx1 = 0.3*tmp;
+    const float dx2 = 1.05*dx1;
+    if (rank==0)
+        printf("   %0.1f %0.1f %0.1f\n",tmp,dx1,dx2);
+    // Update the IDs/Distance removing regions that are < dx of the range
+    Dist1 = Dist;
+    Dist2 = Dist;
+    Array<char> ID1 = ID;
+    Array<char> ID2 = ID;
+    for (size_t i=0; i<ID.length(); i++) {
+        ID1(i) = Dist(i)<-dx1 ? 1:0;
+        ID2(i) = Dist(i)> dx1 ? 1:0;
+    }
+    //Array<float> Dist1 = Dist;
+    //Array<float> Dist2 = Dist;
+    CalcDistMultiLevel( Dist1, ID1, Dm );
+    CalcDistMultiLevel( Dist2, ID2, Dm );
+    fillFloat.fill(Dist1);
+    fillFloat.fill(Dist2);
+    // Keep those regions that are within dx2 of the new volumes
+    Mean = Dist;
+    for (size_t i=0; i<ID.length(); i++) {
+        if ( Dist1(i)+dx2>0 && ID(i)<=0 ) {
+            Mean(i) = -1;
+        } else if ( Dist2(i)+dx2>0 && ID(i)>0 ) {
+            Mean(i) = 1;
+        } else {
+            Mean(i) = Dist(i)>0 ? 0.5:-0.5;
+        }
+    }
+    // Find regions of uncertainty that are entirely contained within another region
+    fillHalo<double> fillDouble(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
+    fillHalo<BlobIDType> fillInt(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
+    BlobIDArray GlobalBlobID;
+    DoubleArray SignDist(ID.size());
+    for (size_t i=0; i<ID.length(); i++)
+        SignDist(i) = fabs(Mean(i))==1 ? -1:1;
+    fillDouble.fill(SignDist);
+    DoubleArray Phase(ID.size());
+    Phase.fill(1);
+    ComputeGlobalBlobIDs( Nx, Ny, Nz, Dm.rank_info, Phase, SignDist, 0, 0, GlobalBlobID, Dm.Comm );
+    fillInt.fill(GlobalBlobID);
+    int N_blobs = maxReduce(Dm.Comm,GlobalBlobID.max()+1);
+    std::vector<float> mean(N_blobs,0);
+    std::vector<int> count(N_blobs,0);
+    for (int k=1; k<=Nz; k++) {
+        for (int j=1; j<=Ny; j++) {
+            for (int i=1; i<=Nx; i++) {
+                int id = GlobalBlobID(i,j,k);
+                if ( id >= 0 ) {
+                    if ( GlobalBlobID(i-1,j,k)<0 ) {
+                        mean[id] += Mean(i-1,j,k);
+                        count[id]++;
+                    }
+                    if ( GlobalBlobID(i+1,j,k)<0 ) {
+                        mean[id] += Mean(i+1,j,k);
+                        count[id]++;
+                    }
+                    if ( GlobalBlobID(i,j-1,k)<0 ) {
+                        mean[id] += Mean(i,j-1,k);
+                        count[id]++;
+                    }
+                    if ( GlobalBlobID(i,j+1,k)<0 ) {
+                        mean[id] += Mean(i,j+1,k);
+                        count[id]++;
+                    }
+                    if ( GlobalBlobID(i,j,k-1)<0 ) {
+                        mean[id] += Mean(i,j,k-1);
+                        count[id]++;
+                    }
+                    if ( GlobalBlobID(i,j,k+1)<0 ) {
+                        mean[id] += Mean(i,j,k+1);
+                        count[id]++;
+                    }
+                }
+            }
+        }
+    }
+    mean = sumReduce(Dm.Comm,mean);
+    count = sumReduce(Dm.Comm,count);
+    for (size_t i=0; i<mean.size(); i++)
+        mean[i] /= count[i];
+    /*if (rank==0) {
+        for (size_t i=0; i<mean.size(); i++)
+            printf("%i %0.4f\n",i,mean[i]);
+    }*/
+    for (size_t i=0; i<Mean.length(); i++) {
+        int id = GlobalBlobID(i);
+        if ( id >= 0 ) {
+            if ( fabs(mean[id]) > 0.95 ) {
+                // Isolated domain surrounded by one domain
+                GlobalBlobID(i) = -2;
+                Mean(i) = sign(mean[id]);
+            } else {
+                // Boarder volume, set to liquid
+                Mean(i) = 1;
+            }
+        }
+    }
+    // Perform the final segmentation and update the distance
+    fillFloat.fill(Mean);
+    segment( Mean, ID, 0.01 );
+    CalcDistMultiLevel( Dist, ID, Dm );
+    fillFloat.fill(Dist);
+}
+
+
+
+// Filter the original data
+void filter_src( const Domain& Dm, Array<float>& src )
+{
+	PROFILE_START("Filter source data");
+    int Nx = Dm.Nx-2;
+    int Ny = Dm.Ny-2;
+    int Nz = Dm.Nz-2;
+	fillHalo<float> fillFloat(Dm.Comm,Dm.rank_info,Nx,Ny,Nz,1,1,1,0,1);
+    // Perform a hot-spot filter on the data
+    std::vector<imfilter::BC> BC = { imfilter::BC::replicate, imfilter::BC::replicate, imfilter::BC::replicate };
+    std::function<float(const Array<float>&)> filter_3D = []( const Array<float>& data )
+    {
+        float min1 = std::min(data(0,1,1),data(2,1,1));
+        float min2 = std::min(data(1,0,1),data(1,2,1));
+        float min3 = std::min(data(1,1,0),data(1,1,2));
+        float max1 = std::max(data(0,1,1),data(2,1,1));
+        float max2 = std::max(data(1,0,1),data(1,2,1));
+        float max3 = std::max(data(1,1,0),data(1,1,2));
+        float min = std::min(min1,std::min(min2,min3));
+        float max = std::max(max1,std::max(max2,max3));
+        return std::max(std::min(data(1,1,1),max),min);
+    };
+    std::function<float(const Array<float>&)> filter_1D = []( const Array<float>& data )
+    {
+        float min = std::min(data(0),data(2));
+        float max = std::max(data(0),data(2));
+        return std::max(std::min(data(1),max),min);
+    };
+    //LOCVOL[0] = imfilter::imfilter<float>( LOCVOL[0], {1,1,1}, filter_3D, BC );
+    std::vector<std::function<float(const Array<float>&)>> filter_set(3,filter_1D);
+    src = imfilter::imfilter_separable<float>( src, {1,1,1}, filter_set, BC );
+    fillFloat.fill( src );
+    // Perform a gaussian filter on the data
+    int Nh[3] = { 2, 2, 2 };
+    float sigma[3] = { 1.0, 1.0, 1.0 };
+    std::vector<Array<float>> H(3);
+    H[0] = imfilter::create_filter<float>( { Nh[0] }, "gaussian", &sigma[0] );
+    H[1] = imfilter::create_filter<float>( { Nh[1] }, "gaussian", &sigma[1] );
+    H[2] = imfilter::create_filter<float>( { Nh[2] }, "gaussian", &sigma[2] );
+    src = imfilter::imfilter_separable( src, H, BC );
+    fillFloat.fill( src );
+	PROFILE_STOP("Filter source data");
+}

analysis/uCT.h

@@ -0,0 +1,56 @@
+#ifndef uCT_H_INC
+#define uCT_H_INC
+
+#include "common/Array.h"
+#include "common/Domain.h"
+#include "common/Communication.h"
+
+
+
+/*!
+ * @brief  Interpolate between meshes
+ * @details  This routine interpolates from a coarse to a fine mesh
+ * @param[in] Coarse    Coarse mesh solution
+ * @param[out] Fine     Fine mesh solution
+ */
+void InterpolateMesh( const Array<float> &Coarse, Array<float> &Fine );
+
+
+// Smooth the data using the distance
+void smooth( const Array<float>& VOL, const Array<float>& Dist, float sigma, Array<float>& MultiScaleSmooth, fillHalo<float>& fillFloat );
+
+
+// Segment the data
+void segment( const Array<float>& data, Array<char>& ID, float tol );
+
+
+// Remove disconnected phases
+void removeDisconnected( Array<char>& ID, const Domain& Dm );
+
+
+// Solve a level (without any coarse level information)
+void solve( const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
+    Array<float>& Dist, Array<float>& MultiScaleSmooth, Array<float>& NonLocalMean, 
+    fillHalo<float>& fillFloat, const Domain& Dm, int nprocx );
+
+
+// Refine a solution from a coarse grid to a fine grid
+void refine( const Array<float>& Dist_coarse, 
+    const Array<float>& VOL, Array<float>& Mean, Array<char>& ID,
+    Array<float>& Dist, Array<float>& MultiScaleSmooth, Array<float>& NonLocalMean, 
+    fillHalo<float>& fillFloat, const Domain& Dm, int nprocx, int level );
+
+
+// Remove regions that are likely noise by shrinking the volumes by dx,
+// removing all values that are more than dx+delta from the surface, and then
+// growing by dx+delta and intersecting with the original data
+void filter_final( Array<char>& ID, Array<float>& Dist,
+    fillHalo<float>& fillFloat, const Domain& Dm,
+    Array<float>& Mean, Array<float>& Dist1, Array<float>& Dist2 );
+
+
+// Filter the original data
+void filter_src( const Domain& Dm, Array<float>& src );
+
+
+#endif

cmake/CompareOutput.cmake

@@ -0,0 +1,50 @@
+# This script compares the output of TEST against GOLD, 
+#    ensuring that all lines within GOLD are int TEST.
+# Note that TEST may have additional lines that are not checked
+CMAKE_POLICY(SET CMP0007 OLD)
+
+FILE(READ "${TEST}" output )
+FILE(READ "${GOLD}" sol )
+
+macro(LIST_REPLACE LIST INDEX NEWVALUE)
+    list(INSERT ${LIST} ${INDEX} ${NEWVALUE})
+    MATH(EXPR __INDEX "${INDEX} + 1")
+    list(REMOVE_AT ${LIST} ${__INDEX})
+endmacro(LIST_REPLACE)
+
+# Convert file contents into a CMake list (where each element in the list is one line of the file)
+STRING(REGEX REPLACE ";" "\\\\;" data "${output}")
+STRING(REGEX REPLACE ";" "\\\\;" sol  "${sol}")
+STRING(REGEX REPLACE "\n" ";" data "${data}")
+STRING(REGEX REPLACE "\n" ";" sol "${sol}")
+LIST( LENGTH data N_data )
+LIST( LENGTH sol N_sol )
+MATH( EXPR N_data "${N_data}-1" )
+MATH( EXPR N_sol "${N_sol}-1" )
+FOREACH( index RANGE ${N_data} )
+    LIST(GET data ${index} tmp )
+    STRING(REGEX REPLACE "(\n|\r)" "" tmp "${tmp}")
+    STRING(STRIP "${tmp}" tmp )
+    LIST_REPLACE( data ${index} "${tmp}")
+ENDFOREACH()
+FOREACH( index RANGE ${N_sol} )
+    LIST( GET sol ${index} tmp )
+    STRING(REGEX REPLACE "(\n|\r)" "" tmp "${tmp}")
+    STRING(STRIP "${tmp}" tmp )
+    LIST_REPLACE( sol ${index} "${tmp}")
+ENDFOREACH()
+
+# Check that each line of sol is present in data (and delete it)
+FOREACH( tmp ${sol} )
+    LIST(FIND data "${tmp}" result )
+    IF ( ${result} EQUAL -1 )
+        MESSAGE("Test output:\n${output}\n\n")
+        MESSAGE(FATAL_ERROR "Did not find '${tmp}' in test output\n" )
+    ELSE()
+        LIST(REMOVE_AT data ${result} )
+    ENDIF()
+ENDFOREACH()
+
+# Finished
+MESSAGE( "All lines in ${GOLD} were found in ${TEST}")
+

cmake/FindMPI.cmake

@@ -0,0 +1,362 @@
+# - Message Passing Interface (MPI) module.
+#
+# The Message Passing Interface (MPI) is a library used to write
+# high-performance parallel applications that use message passing, and
+# is typically deployed on a cluster. MPI is a standard interface
+# (defined by the MPI forum) for which many implementations are
+# available. All of these implementations have somewhat different
+# compilation approaches (different include paths, libraries to link
+# against, etc.), and this module tries to smooth out those differences.
+#
+# This module will set the following variables:
+#   MPI_FOUND                  TRUE if we have found MPI
+#   MPI_COMPILE_FLAGS          Compilation flags for MPI programs
+#   MPI_INCLUDE_PATH           Include path(s) for MPI header
+#   MPI_LINK_FLAGS             Linking flags for MPI programs
+#   MPI_LIBRARY                First MPI library to link against (cached)
+#   MPI_EXTRA_LIBRARY          Extra MPI libraries to link against (cached)
+#   MPI_LIBRARIES              All libraries to link MPI programs against
+#   MPIEXEC                    Executable for running MPI programs
+#   MPIEXEC_NUMPROC_FLAG       Flag to pass to MPIEXEC before giving it the
+#                              number of processors to run on
+#   MPIEXEC_PREFLAGS           Flags to pass to MPIEXEC directly before the
+#                              executable to run.
+#   MPIEXEC_POSTFLAGS          Flags to pass to MPIEXEC after all other flags.
+#
+# This module will attempt to auto-detect these settings, first by
+# looking for a MPI compiler, which many MPI implementations provide
+# as a pass-through to the native compiler to simplify the compilation
+# of MPI programs. The MPI compiler is stored in the cache variable
+# MPI_COMPILER, and will attempt to look for commonly-named drivers
+# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
+# recognized, it will be used to set all of the module variables. To
+# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
+# the CMake cache.
+#
+# If no compiler driver is found or the compiler driver is not
+# recognized, this module will then search for common include paths
+# and library names to try to detect MPI.
+#
+# If CMake initially finds a different MPI than was intended, and you
+# want to use the MPI compiler auto-detection for a different MPI
+# implementation, set MPI_COMPILER to the MPI compiler driver you want
+# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
+# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
+# will run again with the newly-specified MPI_COMPILER.
+#
+# When using MPIEXEC to execute MPI applications, you should typically
+# use all of the MPIEXEC flags as follows:
+#   ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
+#     ${MPIEXEC_POSTFLAGS} ARGS
+# where PROCS is the number of processors on which to execute the program,
+# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
+# MPI program.
+
+#=============================================================================
+# Copyright 2001-2009 Kitware, Inc.
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+#  License text for the above reference.)
+
+# This module is maintained by David Partyka <dave.partyka@kitware.com>.
+
+# A set of directories to search through in addition to the standard system paths
+# that find_program will search through.
+# Microsoft HPC SDK is automatically added to the system path
+# Argonne National Labs MPICH2 sets a registry key that we can use.
+
+set(_MPI_PACKAGE_DIR
+  mpi
+  mpich
+  openmpi
+  lib/mpi
+  lib/mpich
+  lib/openmpi
+  "MPICH/SDK"
+  "Microsoft Compute Cluster Pack"
+  "Microsoft HPC Pack 2008 R2"
+  )
+
+set(_MPI_PREFIX_PATH)
+if(WIN32)
+  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
+  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
+endif()
+
+foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
+  foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
+    if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
+      list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
+    endif()
+  endforeach(MpiPackageDir)
+endforeach(SystemPrefixDir)
+
+# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
+find_program(MPIEXEC
+  NAMES mpiexec mpirun lamexec
+  PATHS ${_MPI_PREFIX_PATH}
+  PATH_SUFFIXES bin
+  DOC "Executable for running MPI programs."
+  )
+
+# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
+# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
+get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
+get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
+
+# If there is an mpi compiler find it and interogate (farther below) it for the include
+# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
+find_program(MPI_COMPILER
+  NAMES mpic++ mpicxx mpiCC mpicc
+  HINTS "${_MPI_BASE_DIR}"
+  PATH_SUFFIXES bin
+  DOC "MPI compiler. Used only to detect MPI compilation flags.")
+mark_as_advanced(MPI_COMPILER)
+
+set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
+set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
+set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
+set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
+mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
+  MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
+  # the cache, and we don't want to override those settings.
+elseif (MPI_COMPILER)
+  # Check whether the -showme:compile option works. This indicates
+  # that we have either Open MPI or a newer version of LAM-MPI, and
+  # implies that -showme:link will also work.
+  # Note that Windows distros do not have an mpi compiler to interogate.
+  exec_program(${MPI_COMPILER}
+    ARGS -showme:compile
+    OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+    RETURN_VALUE MPI_COMPILER_RETURN)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # If we appear to have -showme:compile, then we should also have
+    # -showme:link. Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -showme:link
+      OUTPUT_VARIABLE MPI_LINK_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+
+    # Note that we probably have -showme:incdirs and -showme:libdirs
+    # as well.
+    set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # Do nothing: we have our command lines now
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    # Older versions of LAM-MPI have "-showme". Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -showme
+      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # Do nothing: we have our command lines now
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    # MPICH uses "-show". Try it.
+    exec_program(${MPI_COMPILER}
+      ARGS -show
+      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
+      RETURN_VALUE MPI_COMPILER_RETURN)
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+
+  if (MPI_COMPILER_RETURN EQUAL 0)
+    # We have our command lines, but we might need to copy
+    # MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
+    if (NOT MPI_LINK_CMDLINE)
+      SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
+    endif (NOT MPI_LINK_CMDLINE)
+  else (MPI_COMPILER_RETURN EQUAL 0)
+    message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
+  endif (MPI_COMPILER_RETURN EQUAL 0)
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
+  # the cache, and we don't want to override those settings.
+elseif (MPI_COMPILE_CMDLINE)
+  # Extract compile flags from the compile command line.
+  string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
+  set(MPI_COMPILE_FLAGS_WORK)
+  foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
+    if (MPI_COMPILE_FLAGS_WORK)
+      set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
+    else(MPI_COMPILE_FLAGS_WORK)
+      set(MPI_COMPILE_FLAGS_WORK ${FLAG})
+    endif(MPI_COMPILE_FLAGS_WORK)
+  endforeach(FLAG)
+
+  # Extract include paths from compile command line
+  string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
+  set(MPI_INCLUDE_PATH_WORK)
+  foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
+    string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
+    string(REGEX REPLACE "//" "/" IPATH ${IPATH})
+    list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
+  endforeach(IPATH)
+
+  if (NOT MPI_INCLUDE_PATH_WORK)
+    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+      # The compile command line didn't have any include paths on it,
+      # but we may have -showme:incdirs. Use it.
+      exec_program(${MPI_COMPILER}
+        ARGS -showme:incdirs
+        OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
+        RETURN_VALUE MPI_COMPILER_RETURN)
+      separate_arguments(MPI_INCLUDE_PATH_WORK)
+    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+  endif (NOT MPI_INCLUDE_PATH_WORK)
+
+  if (NOT MPI_INCLUDE_PATH_WORK)
+    # If all else fails, just search for mpi.h in the normal include
+    # paths.
+    find_path(MPI_INCLUDE_PATH mpi.h
+  HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+  PATH_SUFFIXES include
+    )
+    set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
+  endif (NOT MPI_INCLUDE_PATH_WORK)
+
+  # Extract linker paths from the link command line
+  string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
+  set(MPI_LINK_PATH)
+  foreach(LPATH ${MPI_ALL_LINK_PATHS})
+    string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
+    string(REGEX REPLACE "//" "/" LPATH ${LPATH})
+    list(APPEND MPI_LINK_PATH ${LPATH})
+  endforeach(LPATH)
+
+  if (NOT MPI_LINK_PATH)
+    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+      # The compile command line didn't have any linking paths on it,
+      # but we may have -showme:libdirs. Use it.
+      exec_program(${MPI_COMPILER}
+        ARGS -showme:libdirs
+        OUTPUT_VARIABLE MPI_LINK_PATH
+        RETURN_VALUE MPI_COMPILER_RETURN)
+      separate_arguments(MPI_LINK_PATH)
+    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
+  endif (NOT MPI_LINK_PATH)
+
+  # Extract linker flags from the link command line
+  string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
+  set(MPI_LINK_FLAGS_WORK)
+  foreach(FLAG ${MPI_ALL_LINK_FLAGS})
+    if (MPI_LINK_FLAGS_WORK)
+      set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
+    else(MPI_LINK_FLAGS_WORK)
+      set(MPI_LINK_FLAGS_WORK ${FLAG})
+    endif(MPI_LINK_FLAGS_WORK)
+  endforeach(FLAG)
+  if ( MPI_LINK_FLAGS_WORK )
+    string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
+  endif ()
+
+  # Extract the set of libraries to link against from the link command
+  # line
+  string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
+
+  # Determine full path names for all of the libraries that one needs
+  # to link against in an MPI program
+  set(MPI_LIBRARIES)
+  foreach(LIB ${MPI_LIBNAMES})
+    string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
+    set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
+    find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
+    if (MPI_LIB)
+      list(APPEND MPI_LIBRARIES ${MPI_LIB})
+    elseif (NOT MPI_FIND_QUIETLY)
+      message(WARNING "Unable to find MPI library ${LIB}")
+    endif ()
+  endforeach(LIB)
+  set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
+
+  # Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
+  # MPI_EXTRA_LIBRARY.
+  list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
+  list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
+  if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+    list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
+    set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
+  else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+    set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
+  endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
+  if (MPI_NUMLIBS GREATER 1)
+    set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
+    list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
+    set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
+  else (MPI_NUMLIBS GREATER 1)
+    set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
+  endif (MPI_NUMLIBS GREATER 1)
+
+  # Set up all of the appropriate cache entries
+  set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
+  set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
+  set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
+else (MPI_COMPILE_CMDLINE)
+# No MPI compiler to interogate so attempt to find everything with find functions.
+  find_path(MPI_INCLUDE_PATH mpi.h
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES include Inc
+    )
+
+  # Decide between 32-bit and 64-bit libraries for Microsoft's MPI
+  if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
+    set(MS_MPI_ARCH_DIR amd64)
+  else()
+    set(MS_MPI_ARCH_DIR i386)
+  endif()
+
+  find_library(MPI_LIBRARY
+    NAMES mpi mpich msmpi
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
+    )
+
+  find_library(MPI_EXTRA_LIBRARY
+    NAMES mpi++
+    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
+    PATH_SUFFIXES lib
+    DOC "Extra MPI libraries to link against.")
+
+  set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
+  set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+# Set up extra variables to conform to
+if (MPI_EXTRA_LIBRARY)
+  set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
+else (MPI_EXTRA_LIBRARY)
+  set(MPI_LIBRARIES ${MPI_LIBRARY})
+endif (MPI_EXTRA_LIBRARY)
+
+if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  set(MPI_FOUND TRUE)
+else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+  set(MPI_FOUND FALSE)
+endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
+
+#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
+# handle the QUIETLY and REQUIRED arguments
+#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
+
+mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
+  MPI_EXTRA_LIBRARY)
+
+# unset to cleanup namespace
+unset(_MPI_PACKAGE_DIR)
+unset(_MPI_PREFIX_PATH)
+unset(_MPI_BASE_DIR)

cmake/FindVisIt.cmake

@@ -0,0 +1,61 @@
+# Tool for building visit plugins
+#
+# The script will prompt the user to specify VISIT_ROOT_DIR if the prefix
+# cannot be determined by the location of xml2cmake in the system path.
+# Users can set the environment variable VISIT_BIN_PATH before running cmake
+# (e.g. VISIT_BIN_PATH=/usr/local/bin instead of VISIT_ROOT_DIR)
+
+
+
+# Find the xml2cmake executable and set VISIT_XML_CMAKE
+SET( VISIT_XML_CMAKE )
+FIND_PROGRAM( VISIT_XML_CMAKE
+    NAMES xml2cmake
+    PATHS 
+        "${VISIT_ROOT_DIR}"
+        "${VISIT_BIN_PATH}"
+        "$ENV{VISIT_ROOT_DIR}"
+        "$ENV{VISIT_BIN_PATH}"
+        PATH_SUFFIXES bin bin64
+        NO_DEFAULT_PATH
+)
+IF( NOT VISIT_XML_CMAKE )
+    MESSAGE( FATAL_ERROR "xml2cmake not found in:\n"
+        "${VISIT_ROOT_DIR}/bin\n"
+        "${VISIT_BIN_PATH}\n"
+        "$ENV{VISIT_ROOT_DIR}/bin\n"
+        "$ENV{VISIT_BIN_PATH}\n"
+    )
+ELSE()
+    MESSAGE( "VISIT_XML_CMAKE=${VISIT_XML_CMAKE}" )
+ENDIF()
+
+
+# Install plugin 
+MACRO( VISIT_PLUGIN SRC_DIR TARGET )
+    CONFIGURE_FILE( "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_DIR}/${TARGET}.xml" "${CMAKE_CURRENT_BINARY_DIR}/${SRC_DIR}/${TARGET}.xml" )
+    FILE( GLOB ConfigFiles RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_DIR}" 
+        "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_DIR}/*.C" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_DIR}/*.h" )
+    ADD_CUSTOM_TARGET(copy-${SRC_DIR})
+    FOREACH( ConfigFile ${ConfigFiles} )
+        ADD_CUSTOM_COMMAND(TARGET copy-${SRC_DIR} PRE_BUILD 
+            COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_DIR}/${ConfigFile}" 
+                "${CMAKE_CURRENT_BINARY_DIR}/${SRC_DIR}/${ConfigFile}"
+        )
+    ENDFOREACH()
+    ADD_CUSTOM_TARGET( 
+        ${SRC_DIR}
+        COMMAND ${VISIT_XML_CMAKE} -clobber ${TARGET}.xml
+        COMMAND ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+            -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+            -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+            -DVISIT_NOLINK_MPI_WITH_LIBRARIES=TRUE
+            .
+        COMMAND make
+        WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${SRC_DIR}"
+        SOURCES ${SRC_DIR}
+        DEPENDS lbpm-wia copy-${SRC_DIR}
+    )
+ENDMACRO()
+
+

cmake/Find_TIMER.cmake

@@ -0,0 +1,99 @@
+# Configure the Profiler App for the current project
+# The 'CONFIGURE_TIMER' macro searches for the timer library if included,
+# or creates a dummy null timer if it is not used:
+#    CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR )
+# This function assumes that USE_TIMER is set to indicate if the timer should be used
+# If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer
+# If USE_TIMER is not set we will create a summy timer that does nothing.
+# The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default.
+# The input argument NULL_TIMER_DIR specifies the location to install the dummy timer.  
+#    If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used.
+# This function will set the following variables and add the appropriate paths to the include list
+#    TIMER_INCLUDE  - Path to the timer headers
+#    TIMER_CXXFLAGS - C++ flags for the timer library
+#    TIMER_LDFLAGS  - Linker flags to link the timer library
+#    TIMER_LDLIBS   - Linker libraries to link the timer library
+FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
+    # Determine if we want to use the timer utility
+    CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} )
+    SET( TIMER_INCLUDE )
+    SET( TIMER_CXXFLAGS )
+    SET( TIMER_LDFLAGS )
+    SET( TIMER_LDLIBS )
+    IF ( USE_TIMER )
+        # Check if we specified the timer directory
+        EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E remove -f "${NULL_TIMER_DIR}/ProfilerApp.h" )
+        IF ( NOT TIMER_DIRECTORY AND TIMER_INSTALL_DIR )
+            SET( TIMER_DIRECTORY ${TIMER_INSTALL_DIR} )
+        ENDIF()
+        IF ( TIMER_DIRECTORY )
+            VERIFY_PATH( ${TIMER_DIRECTORY} )
+            VERIFY_PATH( ${TIMER_DIRECTORY}/include )
+            VERIFY_PATH( ${TIMER_DIRECTORY}/lib )
+            FIND_LIBRARY( TIMER_LIBS  NAMES timerutility  PATHS ${TIMER_DIRECTORY}/lib  NO_DEFAULT_PATH )
+            SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include )
+            SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" )
+            SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib )
+            SET( TIMER_LDLIBS -ltimerutility )
+        ELSE()
+            MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported.  Use -D TIMER_DIRECTORY=" )
+        ENDIF()
+        SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
+        INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
+        ADD_DEFINITIONS( -DUSE_TIMER )
+        MESSAGE( "Using timer utility" )
+        MESSAGE( "  TIMER_LIBRARIES = ${TIMER_LIBS}" )
+    ELSE()
+        IF ( "${NULL_TIMER_DIR}" STREQUAL "" )
+            SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" )
+        ENDIF()
+        FILE(WRITE  "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...)       do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...)        do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...)      do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP2(...)       do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_SCOPED(...)      do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_SYNCHRONIZE()    do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_SAVE(...)        do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STORE_TRACE(X)   do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE(...)      do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE()        do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_TRACE()   do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE()  do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY()  do {} while(0)\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" )
+        SET( TIMER_INCLUDE  "${NULL_TIMER_DIR}" )
+        INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
+        MESSAGE( "Disabling timer utility" )
+    ENDIF()
+    SET( TIMER_INCLUDE  "${TIMER_INCLUDE}"  PARENT_SCOPE )
+    SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE )
+    SET( TIMER_LDFLAGS  "${TIMER_LDFLAGS}"  PARENT_SCOPE )
+    SET( TIMER_LDLIBS   "${TIMER_LDLIBS}"   PARENT_SCOPE )
+    SET( USE_TIMER      "${USE_TIMER}"      PARENT_SCOPE )
+ENDFUNCTION()
+
+# Check that a path is valid
+FUNCTION( VERIFY_PATH PATH_NAME )
+    IF ("${PATH_NAME}" STREQUAL "")
+        MESSAGE( FATAL_ERROR "Path is not set: ${PATH_NAME}" )
+    ENDIF()
+    IF ( NOT EXISTS ${PATH_NAME} )
+        MESSAGE( FATAL_ERROR "Path does not exist: ${PATH_NAME}" )
+    ENDIF()
+ENDFUNCTION()
+
+# Macro to check if a flag is enabled
+MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT )
+    IF( NOT DEFINED ${FLAG} )
+        SET( ${FLAG} ${DEFAULT} )
+    ELSEIF( ${FLAG}  STREQUAL "" )
+        SET( ${FLAG} ${DEFAULT} )
+    ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
+        SET( ${FLAG} 0 )
+    ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
+        SET( ${FLAG} 1 )
+    ELSE()
+        MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
+    ENDIF ()
+ENDMACRO()
+

cmake/LBPM-macros.cmake

@@ -0,0 +1,42 @@
+# Copy an example folder
+MACRO( INSTALL_EXAMPLE EXAMPLE )
+    INSTALL( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${EXAMPLE}" DESTINATION "${LBPM_INSTALL_DIR}/example" )
+ENDMACRO()
+
+
+# Create an example test
+CONFIGURE_FILE( "${${PROJ}_SOURCE_DIR}/cmake/CompareOutput.cmake" "${${PROJ}_BUILD_DIR}/CompareOutput.cmake" COPYONLY )
+MACRO( TEST_EXAMPLE EXAMPLE EXEFILE PROCS ${ARGN} )
+    SET( EXAMPLE_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXAMPLE}" )
+    # Copy the example directory
+    ADD_CUSTOM_TARGET(
+        ${EXAMPLE} ALL
+        ${CMAKE_COMMAND} -E copy_directory "${CMAKE_CURRENT_SOURCE_DIR}/${EXAMPLE}" "${EXAMPLE_DIR}"
+        DEPENDS ${EXEFILE}
+    )
+    # Create a wrapper script to run the test and copy the output to ${EXAMPLE}.out
+    SET( FILENAME "${EXAMPLE_DIR}/run-${EXAMPLE}" )
+    FILE(WRITE  "${FILENAME}" "# This is a automatically generated file to run example--${EXAMPLE}\n" )
+    FILE(APPEND "${FILENAME}" "${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${PROCS} \"${LBPM_INSTALL_DIR}/bin/${EXEFILE}\" ${ARGN} 2>&1 | tee ${EXAMPLE}.out\n\n" )
+    # Create the test to run the example
+    SET( TESTNAME example--${EXAMPLE} )
+    EXECUTE_PROCESS(COMMAND chmod 755 "${FILENAME}")
+    ADD_TEST( 
+        NAME ${TESTNAME}
+        WORKING_DIRECTORY "${EXAMPLE_DIR}"
+        COMMAND "${FILENAME}"
+    )
+    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS ${PROCS} )
+    SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES RESOURCE_LOCK ${EXEFILE} )
+    # Create a test that checks the output against the data in EXAMPLE/OutputAns.txt
+    IF ( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${EXAMPLE}/ExampleOutput.txt" )
+        ADD_TEST( 
+            NAME ${TESTNAME}-output
+            WORKING_DIRECTORY "${EXAMPLE_DIR}"
+            COMMAND ${CMAKE_COMMAND} -DTEST=${EXAMPLE}.out -DGOLD=ExampleOutput.txt -P "${${PROJ}_BUILD_DIR}/CompareOutput.cmake"
+        )
+        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES FAIL_REGULAR_EXPRESSION "${TEST_FAIL_REGULAR_EXPRESSION}" PROCESSORS 1 )
+        SET_TESTS_PROPERTIES( ${TESTNAME} PROPERTIES DEPENDS ${TESTNAME} )
+    ENDIF()
+ENDMACRO()
+

cmake/SharedPtr.cmake

@@ -0,0 +1,170 @@
+# Create a shared_ptr.h file in the include directory that contains 
+#    a shared_ptr class (hopefully typedef to a compiler basic)
+# Arguements:
+#    INSTALL_DIR - Directory to install shared_ptr.h
+#    NAMESPACE - Namespace to contain the shared_ptr class (may be empty)
+INCLUDE( CheckCXXSourceCompiles )
+FUNCTION( CONFIGURE_SHARED_PTR INSTALL_DIR NAMESPACE )
+    SET( CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS} )
+    CHECK_CXX_SOURCE_COMPILES(
+	    "   #include <memory>
+            namespace ${NAMESPACE} { using std::shared_ptr; }
+	        int main() {
+	            ${NAMESPACE}::shared_ptr<int> ptr;
+	            return 0;
+	        }
+	    "
+	    MEMORY_SHARED_PTR )
+    CHECK_CXX_SOURCE_COMPILES(
+	    "   #include <memory>
+            namespace ${NAMESPACE} { using std::tr1::shared_ptr; }
+	        int main() {
+	            ${NAMESPACE}::shared_ptr<int> ptr;
+	            return 0;
+	        }
+	    "
+	    MEMORY_TR1_SHARED_PTR )
+    CHECK_CXX_SOURCE_COMPILES(
+	    "   #include <tr1/memory>
+            namespace  ${NAMESPACE} { using std::tr1::shared_ptr; }
+	        int main() {
+	            ${NAMESPACE}::shared_ptr<int> ptr;
+	            return 0;
+	        }
+	    "
+	    TR1_MEMORY_TR1_SHARED_PTR )
+    GET_DIRECTORY_PROPERTY( dirs INCLUDE_DIRECTORIES )
+    SET( CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS}" )
+    SET( CMAKE_REQUIRED_INCLUDES ${dirs} "${BOOST_INCLUDE}" )
+    CHECK_CXX_SOURCE_COMPILES(
+	    "   #include \"boost/shared_ptr.hpp\"
+            namespace  ${NAMESPACE} { using boost::shared_ptr; }
+	        int main() {
+	            ${NAMESPACE}::shared_ptr<int> ptr;
+	            return 0;
+	        }
+	    "
+	    BOOST_SHARED_PTR )
+    WRITE_DUMMY_SHARED_PTR( "${NAMESPACE}" "${CMAKE_CURRENT_BINARY_DIR}/tmp/dummy_shared_ptr.h" )
+    CHECK_CXX_SOURCE_COMPILES(
+	    "   #include <iostream>
+	        #include \"${CMAKE_CURRENT_BINARY_DIR}/tmp/dummy_shared_ptr.h\"
+	        int main() {
+	            ${NAMESPACE}::shared_ptr<int> ptr;
+	            return 0;
+	        }
+	    "
+	    DUMMY_SHARED_PTR )
+    IF ( NOT NAMESPACE )
+        SET( NAMESPACE " " )
+    ENDIF()
+    IF ( BOOST_SHARED_PTR )
+        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/shared_ptr.hpp\"\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/weak_ptr.hpp\"\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include \"boost/enable_shared_from_this.hpp\"\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::shared_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::dynamic_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::const_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::weak_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using boost::enable_shared_from_this; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
+    ELSEIF ( MEMORY_SHARED_PTR )
+        IF ( ${NAMESPACE} STREQUAL "std" )
+            FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
+        ELSE()
+            FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::shared_ptr; \n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::dynamic_pointer_cast; \n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::const_pointer_cast; \n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::weak_ptr; \n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::enable_shared_from_this; \n")
+            FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
+        ENDIF()
+    ELSEIF ( MEMORY_TR1_SHARED_PTR )
+        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <memory>\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::shared_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::dynamic_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::const_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::weak_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::enable_shared_from_this; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
+    ELSEIF ( TR1_MEMORY_TR1_SHARED_PTR )
+        FILE(WRITE  "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "#include <tr1/memory>\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "namespace ${NAMESPACE} {\n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::shared_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::dynamic_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::const_pointer_cast; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::weak_ptr; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "    using std::tr1::enable_shared_from_this; \n")
+        FILE(APPEND "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "}\n")
+    ELSEIF ( DUMMY_SHARED_PTR ) 
+        MESSAGE("Warning: No valid shared_ptr found, using dummy shared_ptr" )
+        WRITE_DUMMY_SHARED_PTR( "${NAMESPACE}" "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" )
+    ELSE()
+        MESSAGE(FATAL_ERROR "No shared_ptr availible")
+    ENDIF()
+    EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+        "${CMAKE_CURRENT_BINARY_DIR}/tmp/shared_ptr.h" "${INSTALL_DIR}/shared_ptr.h" )
+ENDFUNCTION()
+
+
+FUNCTION( WRITE_DUMMY_SHARED_PTR NAMESPACE FILENAME )
+    FILE(WRITE  "${FILENAME}" "#ifndef DUMMY_SHARED_PTR_INC\n")
+    FILE(APPEND "${FILENAME}" "#define DUMMY_SHARED_PTR_INC\n")
+    FILE(APPEND "${FILENAME}" "namespace dummy {\n\n")
+    FILE(APPEND "${FILENAME}" "template<class T> void DefaultDeleter(T* p) {delete p;}\n\n")
+    FILE(APPEND "${FILENAME}" "template<class T> class shared_ptr {\n")
+    FILE(APPEND "${FILENAME}" "public:\n")
+    FILE(APPEND "${FILENAME}" "    typedef void (*D)(T*);\n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr( ): obj(NULL), deleter(DefaultDeleter<T>), count(NULL) {}\n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr( T *ptr, void (*D)(T*)=DefaultDeleter<T>):\n")
+    FILE(APPEND "${FILENAME}" "        obj(ptr), deleter(D), count(NULL) { if (ptr) { count = new int; (*count)=1; } } \n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr( const shared_ptr<T>& rhs ):  \n")
+    FILE(APPEND "${FILENAME}" "        obj(rhs.get()), deleter(reinterpret_cast<D>(rhs.deleter)), count(rhs.count) { if ( count!=NULL ) { ++(*count); } } \n")
+    FILE(APPEND "${FILENAME}" "    template<class U> shared_ptr( const shared_ptr<U>& rhs ):  \n")
+    FILE(APPEND "${FILENAME}" "        obj(rhs.get()), deleter(reinterpret_cast<D>(rhs.deleter)), count(rhs.count) { if ( count!=NULL ) { ++(*count); } } \n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr& operator=( const shared_ptr<T>& rhs )\n")
+    FILE(APPEND "${FILENAME}" "        { if (this==&rhs) { return *this;} reset(); obj=rhs.obj; deleter=reinterpret_cast<D>(rhs.deleter); count=rhs.count; ++(*count); return *this; } \n")
+    FILE(APPEND "${FILENAME}" "    ~shared_ptr( ) { reset(); }\n")
+    FILE(APPEND "${FILENAME}" "    void reset( T *ptr ) { reset(); obj=ptr; count=new int; (*count)=1; }\n")
+    FILE(APPEND "${FILENAME}" "    void reset( void ) { \n")
+    FILE(APPEND "${FILENAME}" "        if ( count!=NULL) { int tmp=--(*count); if ( tmp==0 ) { deleter(obj); delete count; } } \n")
+    FILE(APPEND "${FILENAME}" "        obj=NULL; count=NULL; \n")
+    FILE(APPEND "${FILENAME}" "    }\n")
+    FILE(APPEND "${FILENAME}" "    T* get( ) const { return obj; } \n")
+    FILE(APPEND "${FILENAME}" "    T* operator->( ) const { return obj; } \n")
+    FILE(APPEND "${FILENAME}" "    const T& operator*( ) const { return *obj; } \n")
+    FILE(APPEND "${FILENAME}" "    bool operator==( const T * rhs ) const { return obj==rhs; } \n")
+    FILE(APPEND "${FILENAME}" "    bool operator!=( const T * rhs ) const { return obj!=rhs; } \n")
+    FILE(APPEND "${FILENAME}" "protected:\n")
+    FILE(APPEND "${FILENAME}" "    T *obj;\n")
+    FILE(APPEND "${FILENAME}" "    void (*deleter)(T*);\n")
+    FILE(APPEND "${FILENAME}" "    volatile int *count;\n")
+    FILE(APPEND "${FILENAME}" "template<class T1, class U> friend shared_ptr<T1> dynamic_pointer_cast( shared_ptr<U> const & );\n")
+    FILE(APPEND "${FILENAME}" "template<class T1, class U> friend shared_ptr<T1> const_pointer_cast( shared_ptr<U> const & );\n")
+    FILE(APPEND "${FILENAME}" "template<class Y> friend class shared_ptr;\n")
+    FILE(APPEND "${FILENAME}" "};\n\n")
+    FILE(APPEND "${FILENAME}" "template<class T, class U> shared_ptr<T> dynamic_pointer_cast( shared_ptr<U> const & rhs ) {\n")
+    FILE(APPEND "${FILENAME}" "    T* obj = dynamic_cast<T*>(rhs.obj);\n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr<T> ptr;\n")
+    FILE(APPEND "${FILENAME}" "    if ( obj!=NULL ) { ptr.obj = obj; ptr.count=rhs.count; ++(*ptr.count); }\n")
+    FILE(APPEND "${FILENAME}" "    return ptr;\n}\n")
+    FILE(APPEND "${FILENAME}" "template<class T, class U> shared_ptr<T> const_pointer_cast( shared_ptr<U> const & rhs ) {\n")
+    FILE(APPEND "${FILENAME}" "    T* obj = const_cast<T*>(rhs.obj);\n")
+    FILE(APPEND "${FILENAME}" "    shared_ptr<T> ptr;\n")
+    FILE(APPEND "${FILENAME}" "    if ( obj!=NULL ) { ptr.obj = obj; ptr.count=rhs.count; ++(*ptr.count); }\n")
+    FILE(APPEND "${FILENAME}" "    return ptr;\n}\n")
+    FILE(APPEND "${FILENAME}" "\n} // namespace dummy\n")
+    FILE(APPEND "${FILENAME}" "\n\n")
+    FILE(APPEND "${FILENAME}" "namespace ${NAMESPACE} {\n")
+    FILE(APPEND "${FILENAME}" "    using dummy::shared_ptr; \n")
+    FILE(APPEND "${FILENAME}" "    using dummy::dynamic_pointer_cast; \n")
+    FILE(APPEND "${FILENAME}" "    using dummy::const_pointer_cast; \n")
+    FILE(APPEND "${FILENAME}" "}\n\n")
+    FILE(APPEND "${FILENAME}" "#endif\n")
+ENDFUNCTION()
+
+

cmake/UseDoxygen.cmake

@@ -0,0 +1,119 @@
+# - Run Doxygen
+#
+# Adds a doxygen target that runs doxygen to generate the html
+# and optionally the LaTeX API documentation.
+# The doxygen target is added to the doc target as a dependency.
+# i.e.: the API documentation is built with:
+#  make doc
+#
+# USAGE: GLOBAL INSTALL
+#
+# Install it with:
+#  cmake ./ && sudo make install
+# Add the following to the CMakeLists.txt of your project:
+#  include(UseDoxygen OPTIONAL)
+# Optionally copy Doxyfile.in in the directory of CMakeLists.txt and edit it.
+#
+# USAGE: INCLUDE IN PROJECT
+#
+#  set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+#  include(UseDoxygen)
+# Add the Doxyfile.in and UseDoxygen.cmake files to the projects source directory.
+#
+#
+# Variables you may define are:
+#  DOXYFILE_SOURCE_DIR - Path where the Doxygen input files are.
+#  	Defaults to the current source and binary directory.
+#  DOXYFILE_OUTPUT_DIR - Path where the Doxygen output is stored. Defaults to "doc".
+#
+#  DOXYFILE_LATEX - Set to "NO" if you do not want the LaTeX documentation
+#  	to be built.
+#  DOXYFILE_LATEX_DIR - Directory relative to DOXYFILE_OUTPUT_DIR where
+#  	the Doxygen LaTeX output is stored. Defaults to "latex".
+#
+#  DOXYFILE_HTML_DIR - Directory relative to DOXYFILE_OUTPUT_DIR where
+#  	the Doxygen html output is stored. Defaults to "html".
+#
+
+#
+#  Copyright (c) 2009, 2010 Tobias Rautenkranz <tobias@rautenkranz.ch>
+#
+#  Redistribution and use is allowed according to the terms of the New
+#  BSD license.
+#  For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+#
+
+macro(usedoxygen_set_default name value)
+	if(NOT DEFINED "${name}")
+		set("${name}" "${value}")
+	endif()
+endmacro()
+
+find_package(Doxygen)
+
+if(DOXYGEN_FOUND)
+	find_file(DOXYFILE_IN "Doxyfile.in"
+			PATHS "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_ROOT}/Modules/"
+			NO_DEFAULT_PATH)
+	set(DOXYFILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")
+	include(FindPackageHandleStandardArgs)
+	find_package_handle_standard_args(DOXYFILE_IN DEFAULT_MSG "DOXYFILE_IN")
+endif()
+
+if(DOXYGEN_FOUND AND DOXYFILE_IN_FOUND)
+	usedoxygen_set_default(DOXYFILE_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/doc")
+	usedoxygen_set_default(DOXYFILE_HTML_DIR "html")
+	usedoxygen_set_default(DOXYFILE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}\"
+		\"${CMAKE_CURRENT_BINARY_DIR}")
+
+	set_property(DIRECTORY APPEND PROPERTY
+		ADDITIONAL_MAKE_CLEAN_FILES
+		"${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_HTML_DIR}")
+
+	add_custom_target(doxygen
+		COMMAND ${DOXYGEN_EXECUTABLE}
+			${DOXYFILE} 
+		COMMENT "Writing documentation to ${DOXYFILE_OUTPUT_DIR}..."
+		WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+	## LaTeX
+	set(DOXYFILE_PDFLATEX "NO")
+	set(DOXYFILE_DOT "NO")
+
+	find_package(LATEX)
+	find_program(MAKE_PROGRAM make)
+	if(LATEX_COMPILER AND MAKEINDEX_COMPILER AND MAKE_PROGRAM AND
+			(NOT DEFINED DOXYFILE_LATEX OR DOXYFILE_LATEX STREQUAL "YES"))
+		set(DOXYFILE_LATEX "YES")
+		usedoxygen_set_default(DOXYFILE_LATEX_DIR "latex")
+
+		set_property(DIRECTORY APPEND PROPERTY
+				ADDITIONAL_MAKE_CLEAN_FILES
+				"${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}")
+
+		if(PDFLATEX_COMPILER)
+			set(DOXYFILE_PDFLATEX "YES")
+		endif()
+		if(DOXYGEN_DOT_EXECUTABLE)
+			set(DOXYFILE_DOT "YES")
+		endif()
+
+		add_custom_command(TARGET doxygen
+			POST_BUILD
+			COMMAND ${MAKE_PROGRAM}
+			COMMENT	"Running LaTeX for Doxygen documentation in ${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}..."
+			WORKING_DIRECTORY "${DOXYFILE_OUTPUT_DIR}/${DOXYFILE_LATEX_DIR}")
+	else()
+		set(DOXYGEN_LATEX "NO")
+	endif()
+
+
+	configure_file(${DOXYFILE_IN} Doxyfile IMMEDIATE @ONLY)
+
+	get_target_property(DOC_TARGET doc TYPE)
+	if(NOT DOC_TARGET)
+		add_custom_target(doc)
+	endif()
+
+	add_dependencies(doc doxygen)
+endif()

cmake/ctest_script.cmake

@@ -0,0 +1,272 @@
+# ctest script for building, running, and submitting the test results 
+# Usage:  ctest -s script,build
+#   build = debug / optimized / weekly / valgrind / valgrind-matlab
+# Note: this test will use use the number of processors defined in the variable N_PROCS,
+#   the enviornmental variable N_PROCS, or the number of processors availible (if not specified)
+
+# Set platform specific variables
+SITE_NAME( HOSTNAME )
+STRING( REGEX REPLACE  "-(ext|login)(..|.)"  ""  HOSTNAME  "${HOSTNAME}" )
+
+SET( CC                 $ENV{CC}                )
+SET( CXX                $ENV{CXX}               )
+SET( CFLAGS             $ENV{CFLAGS}            )
+SET( CXXFLAGS           $ENV{CXXFLAGS}          )
+SET( MPIEXEC            $ENV{MPIEXEC}           )
+SET( USE_TIMER          "$ENV{USE_TIMER}"       )
+SET( TIMER_DIRECTORY    "$ENV{TIMER_DIRECTORY}" )
+SET( RATES_DIRECTORY    "$ENV{RATES_DIRECTORY}" )
+SET( USE_ACML           $ENV{USE_ACML}          )
+SET( ACML_DIRECTORY     $ENV{ACML_DIRECTORY}    )
+SET( USE_MKL            $ENV{USE_MKL}           )
+SET( MKL_DIRECTORY      $ENV{MKL_DIRECTORY}     )
+SET( BLAS_DIRECTORY     $ENV{BLAS_DIRECTORY}    )
+SET( BLAS_LIB           $ENV{BLAS_LIB}          )
+SET( LAPACK_DIRECTORY   $ENV{LAPACK_DIRECTORY}  )
+SET( LAPACK_LIB         $ENV{LAPACK_LIB}        )
+SET( USE_MATLAB         $ENV{USE_MATLAB}        )
+SET( MATLAB_DIRECTORY   $ENV{MATLAB_DIRECTORY}  )
+SET( COVERAGE_COMMAND   $ENV{COVERAGE_COMMAND}  )
+SET( VALGRIND_COMMAND   $ENV{VALGRIND_COMMAND}  )
+SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} )
+SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} )
+SET( LDLIBS             $ENV{LDLIBS}            )
+SET( LDFLAGS            $ENV{LDFLAGS}           )
+SET( MPI_COMPILER       $ENV{MPI_COMPILER}      )
+SET( MPI_DIRECTORY      $ENV{MPI_DIRECTORY}     )
+SET( MPI_INCLUDE        $ENV{MPI_INCLUDE}       )
+SET( MPI_LINK_FLAGS     $ENV{MPI_LINK_FLAGS}    )
+SET( MPI_LIBRARIES      $ENV{MPI_LIBRARIES}     )
+SET( MPIEXEC            $ENV{MPIEXEC}           )
+SET( BUILD_SERIAL       $ENV{BUILD_SERIAL}      )
+SET( CUDA_FLAGS         $ENV{CUDA_FLAGS}        )
+SET( CUDA_HOST_COMPILER $ENV{CUDA_HOST_COMPILER} )
+SET( SKIP_TESTS         $ENV{SKIP_TESTS}        )
+SET( BUILDNAME_POSTFIX  "$ENV{BUILDNAME_POSTFIX}" )
+SET( LIB_TYPE           "$ENV{LIB_TYPE}"        )
+SET( USE_MPI            TRUE                    )
+IF ( DEFINED ENV{USE_MPI} )
+    SET( USE_MPI        $ENV{USE_MPI}           )
+ENDIF()
+SET( USE_VISIT          $ENV{USE_VISIT}         )
+SET( VISIT_ROOT_DIR     $ENV{VISIT_ROOT_DIR}    )
+
+
+# Get the source directory based on the current directory
+IF ( NOT LBPM_SOURCE_DIR )
+    SET( LBPM_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." )
+ENDIF()
+IF ( NOT CMAKE_MAKE_PROGRAM )
+    SET( CMAKE_MAKE_PROGRAM make )
+ENDIF()
+
+
+# Check that we specified the build type to run
+SET( USE_VALGRIND FALSE )
+SET( RUN_WEEKLY FALSE )
+SET( USE_CUDA FALSE )
+SET( ENABLE_GCOV "false" )
+SET( CTEST_COVERAGE_COMMAND ${COVERAGE_COMMAND} )
+IF( NOT CTEST_SCRIPT_ARG )
+    MESSAGE(FATAL_ERROR "No build specified: ctest -S /path/to/script,build (debug/optimized/valgrind")
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "debug" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-debug" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( ENABLE_GCOV "true" )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "debug-cuda" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-debug-cuda" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( ENABLE_GCOV "true" )
+    SET( USE_CUDA TRUE )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "optimized") OR (${CTEST_SCRIPT_ARG} STREQUAL "opt") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-opt" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "optimized-cuda") OR (${CTEST_SCRIPT_ARG} STREQUAL "opt-cuda") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-opt-cuda" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+    SET( USE_CUDA TRUE )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "weekly") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-weekly" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+    SET( RUN_WEEKLY TRUE )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "weekly-cuda") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-weekly-cuda" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+    SET( RUN_WEEKLY TRUE )
+    SET( USE_CUDA TRUE )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "valgrind" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-valgrind" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( USE_VALGRIND TRUE )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "valgrind-cuda" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-valgrind-cuda" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( USE_VALGRIND TRUE )
+    SET( USE_CUDA TRUE )
+ELSE()
+    MESSAGE(FATAL_ERROR "Invalid build (${CTEST_SCRIPT_ARG}): ctest -S /path/to/script,build (debug/opt/valgrind")
+ENDIF()
+IF ( USE_VISIT )
+    STRING( REGEX REPLACE  "LBPM-WIA-"  "LBPM-WIA-visit-"  CTEST_BUILD_NAME  "${CTEST_BUILD_NAME}" )
+ENDIF()
+IF ( BUILDNAME_POSTFIX )
+    SET( CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-${BUILDNAME_POSTFIX}" )
+ENDIF()
+IF ( NOT CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+ENDIF()
+
+
+# Set the number of processors
+IF( NOT DEFINED N_PROCS )
+    SET( N_PROCS $ENV{N_PROCS} )
+ENDIF()
+IF( NOT DEFINED N_PROCS )
+    SET(N_PROCS 1)
+    # Linux:
+    SET(cpuinfo_file "/proc/cpuinfo")
+    IF(EXISTS "${cpuinfo_file}")
+        FILE(STRINGS "${cpuinfo_file}" procs REGEX "^processor.: [0-9]+$")
+        list(LENGTH procs N_PROCS)
+    ENDIF()
+    # Mac:
+    IF(APPLE)
+        find_program(cmd_sys_pro "sysctl")
+        if(cmd_sys_pro)
+            execute_process(COMMAND ${cmd_sys_pro} hw.physicalcpu OUTPUT_VARIABLE info)
+            STRING(REGEX REPLACE "^.*hw.physicalcpu: ([0-9]+).*$" "\\1" N_PROCS "${info}")
+        ENDIF()
+    ENDIF()
+    # Windows:
+    IF(WIN32)
+        SET(N_PROCS "$ENV{NUMBER_OF_PROCESSORS}")
+    ENDIF()
+ENDIF()
+
+
+# Set basic variables
+SET( CTEST_PROJECT_NAME "LBPM-WIA" )
+SET( CTEST_SOURCE_DIRECTORY "${LBPM_SOURCE_DIR}" )
+SET( CTEST_BINARY_DIRECTORY "." )
+SET( CTEST_DASHBOARD "Nightly" )
+SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 500 )
+SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 500 )
+SET( CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 10000 )
+SET( CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 10000 )
+SET( NIGHTLY_START_TIME "18:00:00 EST" )
+SET( CTEST_NIGHTLY_START_TIME "22:00:00 EST" )
+SET( CTEST_COMMAND "\"${CTEST_EXECUTABLE_NAME}\" -D ${CTEST_DASHBOARD}" )
+IF ( BUILD_SERIAL )
+    SET( CTEST_BUILD_COMMAND "${CMAKE_MAKE_PROGRAM} -i install" )
+ELSE()
+    SET( CTEST_BUILD_COMMAND "${CMAKE_MAKE_PROGRAM} -i -j ${N_PROCS} install" )
+ENDIF()
+IF ( USE_VISIT )
+    SET( CTEST_BUILD_COMMAND "${CTEST_BUILD_COMMAND} visit" )
+ENDIF()
+SET( CTEST_CUSTOM_WARNING_EXCEPTION "has no symbols" )
+
+
+# Set timeouts: 30 minutes for debug, 15 for opt, and 60 minutes for valgrind/weekly
+IF ( USE_VALGRIND )
+    SET( CTEST_TEST_TIMEOUT 3600 )
+ELSEIF ( RUN_WEEKLY )
+    SET( CTEST_TEST_TIMEOUT 3600 )
+ELSEIF( ${CMAKE_BUILD_TYPE} STREQUAL "Debug" )
+    SET( CTEST_TEST_TIMEOUT 1800 )
+ELSE()
+    SET( CTEST_TEST_TIMEOUT 900 )
+ENDIF()
+
+
+# Set valgrind options
+#SET (VALGRIND_COMMAND_OPTIONS "--tool=memcheck --leak-check=yes --track-fds=yes --num-callers=50 --show-reachable=yes --trace-children=yes --track-origins=yes --malloc-fill=0xff --free-fill=0xfe --suppressions=${LBPM_SOURCE_DIR}/ValgrindSuppresionFile" )
+SET( VALGRIND_COMMAND_OPTIONS  "--tool=memcheck --leak-check=yes --num-callers=50 --show-reachable=yes --trace-children=yes --suppressions=${LBPM_SOURCE_DIR}/ValgrindSuppresionFile" )
+IF ( USE_VALGRIND )
+    SET( MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
+    SET( MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECK_COMMAND_OPTIONS ${VALGRIND_COMMAND_OPTIONS} )
+    SET( CTEST_MEMORYCHECKCOMMAND_OPTIONS  ${VALGRIND_COMMAND_OPTIONS} )
+ENDIF()
+
+
+# Clear the binary directory and create an initial cache
+EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E remove -f CMakeCache.txt )
+EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E remove_directory CMakeFiles )
+FILE(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" "CTEST_TEST_CTEST:BOOL=1")
+
+
+# Set the configure options
+SET( CTEST_OPTIONS )
+SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_CXX_COMPILER:PATH=${CXX}" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLAGS='${CXXFLAGS}'" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
+IF ( USE_MPI )
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}")
+    IF ( NOT USE_VALGRIND )
+        SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true")
+    ENDIF()
+ELSE()
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI:BOOL=false")
+ENDIF()
+IF ( USE_TIMER )
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_TIMER:BOOL=true;-DTIMER_DIRECTORY='${TIMER_DIRECTORY}'" )
+ELSE()
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_TIMER:BOOL=false" )
+ENDIF()
+IF ( USE_VISIT )
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_VISIT:BOOL=true;-DVISIT_ROOT_DIR='${VISIT_ROOT_DIR}'" )
+ENDIF()
+IF ( USE_CUDA )
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCMAKE_CUDA_FLAGS='${CUDA_FLAGS}';-DCMAKE_CUDA_HOST_COMPILER=${CUDA_HOST_COMPILER};-DLIB_TYPE=${LIB_TYPE}" )
+ELSE()
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=false" )
+ENDIF()
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDLIBS:STRING=\"${LDLIBS}\"" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
+
+
+# Configure and run the tests
+SET( CTEST_SITE ${HOSTNAME} )
+CTEST_START("${CTEST_DASHBOARD}")
+CTEST_UPDATE()
+CTEST_CONFIGURE(
+    BUILD   ${CTEST_BINARY_DIRECTORY}
+    SOURCE  ${CTEST_SOURCE_DIRECTORY}
+    OPTIONS "${CTEST_OPTIONS}"
+)
+CTEST_BUILD()
+IF ( SKIP_TESTS )
+    # Do not run tests
+    SET( CTEST_COVERAGE_COMMAND )
+ELSEIF ( USE_VALGRIND_MATLAB )
+    CTEST_TEST( INCLUDE MATLAB  PARALLEL_LEVEL ${N_PROCS} )
+ELSEIF ( USE_VALGRIND )
+    # CTEST_MEMCHECK( EXCLUDE "(WEEKLY|procs|example--)"  PARALLEL_LEVEL ${N_PROCS} )
+    CTEST_MEMCHECK( EXCLUDE "(WEEKLY|example--)"  PARALLEL_LEVEL ${N_PROCS} )
+ELSEIF ( RUN_WEEKLY )
+    CTEST_TEST( INCLUDE "(WEEKLY|example--)"  PARALLEL_LEVEL ${N_PROCS} )
+ELSE()
+    CTEST_TEST( EXCLUDE "(WEEKLY|example--)"  PARALLEL_LEVEL ${N_PROCS} )
+ENDIF()
+IF( ENABLE_GCOV )
+    CTEST_COVERAGE()
+ENDIF()
+
+
+# Submit the results to oblivion
+SET( CTEST_DROP_METHOD "http" )
+SET( CTEST_DROP_SITE "mberrill.myqnapcloud.com" )
+SET( CTEST_DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
+SET( CTEST_DROP_SITE_CDASH TRUE )
+SET( DROP_SITE_CDASH TRUE )
+CTEST_SUBMIT()
+
+
+# Write a message to test for success in the ctest-builder
+MESSAGE( "ctest_script ran to completion" )
+

cmake/libraries.cmake

@@ -0,0 +1,314 @@
+INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Find_TIMER.cmake" )
+INCLUDE(CheckCCompilerFlag)
+INCLUDE(CheckCXXCompilerFlag)
+
+
+FUNCTION( CONFIGURE_LINE_COVERAGE )
+    SET( COVERAGE_FLAGS )
+    SET( COVERAGE_LIBS )
+    IF ( ENABLE_GCOV )
+        SET( COVERAGE_FLAGS -DUSE_GCOV )
+        SET( CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage )
+        CHECK_CXX_SOURCE_COMPILES( "int main() { return 0;}" profile-arcs )
+        IF ( profile-arcs )
+            SET( COVERAGE_FLAGS "${COVERAGE_FLAGS} -fprofile-arcs -ftest-coverage" )
+            SET( COVERAGE_LIBS ${COVERAGE_LIBS} -fprofile-arcs )
+        ENDIF()
+        SET( CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_FLAGS} -lgcov" )
+        CHECK_CXX_SOURCE_COMPILES( "int main() { return 0;}" lgcov )
+        IF ( lgcov )
+            SET( COVERAGE_LIBS -lgcov ${COVERAGE_LIBS} )
+        ENDIF()
+        MESSAGE("Enabling coverage:")
+        MESSAGE("   COVERAGE_FLAGS = ${COVERAGE_FLAGS}")
+        MESSAGE("   COVERAGE_LIBS = ${COVERAGE_LIBS}")
+        ADD_DEFINITIONS( ${COVERAGE_FLAGS} )
+        SET( COVERAGE_FLAGS ${COVERAGE_FLAGS} PARENT_SCOPE )
+        SET( COVERAGE_LIBS  ${COVERAGE_LIBS}  PARENT_SCOPE )
+    ENDIF()
+ENDFUNCTION()
+
+
+# Macro to configure MIC
+MACRO( CONFIGURE_MIC )
+    CHECK_ENABLE_FLAG( USE_MIC 0 )
+    IF ( USE_MIC )
+        ADD_DEFINITIONS( "-D USE_MIC" )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to find and configure the MPI libraries
+MACRO( CONFIGURE_MPI )
+    # Determine if we want to use MPI
+    CHECK_ENABLE_FLAG(USE_MPI 1 )
+    IF ( USE_MPI )
+        # Check if we specified the MPI directory
+        IF ( MPI_DIRECTORY )
+            # Check the provided MPI directory for include files
+            VERIFY_PATH( "${MPI_DIRECTORY}" )
+            IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" )
+                SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" )
+            ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" )
+                SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" )
+            ELSE()
+                MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" )
+            ENDIF ()
+            INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
+            SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
+            # Set MPI libraries
+            IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
+                FIND_LIBRARY( MSMPI_LIB     NAMES msmpi     PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
+                FIND_LIBRARY( MSMPI_LIB     NAMES msmpi     PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
+                FIND_LIBRARY( MSMPIFEC_LIB  NAMES msmpifec  PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
+                FIND_LIBRARY( MSMPIFEC_LIB  NAMES msmpifec  PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
+                FIND_LIBRARY( MSMPIFMC_LIB  NAMES msmpifmc  PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
+                FIND_LIBRARY( MSMPIFMC_LIB  NAMES msmpifmc  PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
+                SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} )
+            ENDIF()
+            # Set the mpi executable
+            IF ( MPIEXEC ) 
+                # User specified the MPI command directly, use as is
+            ELSEIF ( MPIEXEC_CMD )
+                # User specified the name of the MPI executable
+                SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
+                IF ( NOT EXISTS ${MPIEXEC} )
+                    MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
+                ENDIF ()
+            ELSE ()
+                # Search for the MPI executable in the current directory
+                FIND_PROGRAM ( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
+                IF ( NOT MPIEXEC )
+                    MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
+                ENDIF()
+            ENDIF ()
+            # Set MPI flags
+            IF ( NOT MPIEXEC_NUMPROC_FLAG )
+                SET( MPIEXEC_NUMPROC_FLAG "-np" )
+            ENDIF()
+        ELSEIF ( MPI_COMPILER )
+            # The mpi compiler should take care of everything
+            IF ( MPI_INCLUDE )
+                INCLUDE_DIRECTORIES( ${MPI_INCLUDE} )
+            ENDIF()
+        ELSE()
+            # Perform the default search for MPI
+            INCLUDE ( FindMPI )
+            IF ( NOT MPI_FOUND )
+                MESSAGE( "  MPI_INCLUDE = ${MPI_INCLUDE}" )
+                MESSAGE( "  MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
+                MESSAGE( "  MPI_LIBRARIES = ${MPI_LIBRARIES}" )
+                MESSAGE( FATAL_ERROR "Did not find MPI" )
+            ENDIF ()
+            INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" )
+            SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" )
+        ENDIF()
+        # Check if we need to use MPI for serial tests
+        CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
+        # Set defaults if they have not been set
+        IF ( NOT MPIEXEC )
+            SET( MPIEXEC mpirun )
+        ENDIF()
+        IF ( NOT MPIEXEC_NUMPROC_FLAG )
+            SET( MPIEXEC_NUMPROC_FLAG "-np" )
+        ENDIF()
+        # Set the definitions
+        ADD_DEFINITIONS( "-DUSE_MPI" )  
+        MESSAGE( "Using MPI" )
+        MESSAGE( "  MPIEXEC = ${MPIEXEC}" )
+        MESSAGE( "  MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
+        MESSAGE( "  MPI_INCLUDE = ${MPI_INCLUDE}" )
+        MESSAGE( "  MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
+        MESSAGE( "  MPI_LIBRARIES = ${MPI_LIBRARIES}" )
+    ELSE()
+        SET( USE_MPI_FOR_SERIAL_TESTS 0 )
+        SET( MPIEXEC "" )
+        SET( MPIEXEC_NUMPROC_FLAG "" )
+        SET( MPI_INCLUDE "" )
+        SET( MPI_LINK_FLAGS "" )
+        SET( MPI_LIBRARIES "" )
+        MESSAGE( "Not using MPI, all parallel tests will be disabled" )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to find and configure hdf5
+MACRO ( CONFIGURE_HDF5 )
+    CHECK_ENABLE_FLAG( USE_HDF5 0 )
+    IF ( USE_HDF5 )
+        # Check if we specified the silo directory
+        IF ( HDF5_DIRECTORY )
+            VERIFY_PATH ( ${HDF5_DIRECTORY} )
+            INCLUDE_DIRECTORIES ( ${HDF5_DIRECTORY}/include )
+            SET ( HDF5_INCLUDE ${HDF5_DIRECTORY}/include )
+            FIND_LIBRARY ( HDF5_LIB    NAMES hdf5    PATHS ${HDF5_DIRECTORY}/lib  NO_DEFAULT_PATH )
+            FIND_LIBRARY ( HDF5_HL_LIB NAMES hdf5_hl PATHS ${HDF5_DIRECTORY}/lib  NO_DEFAULT_PATH )
+        ELSE()
+            MESSAGE( FATAL_ERROR "Default search for hdf5 is not yet supported.  Use -D HDF5_DIRECTORY=" )
+        ENDIF()
+        SET ( HDF5_LIBS
+            ${HDF5_HL_LIB}
+            ${HDF5_LIB}
+        )
+        ADD_DEFINITIONS ( -DUSE_HDF5 )  
+        MESSAGE( "Using hdf5" )
+        MESSAGE( "   ${HDF5_LIB}" )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to find and configure netcdf
+MACRO( CONFIGURE_NETCDF )
+    CHECK_ENABLE_FLAG( USE_NETCDF 0 )
+    IF ( USE_NETCDF )
+        SET( USE_HDF5 1 )
+        CONFIGURE_HDF5()
+        IF ( NETCDF_DIRECTORY )
+            VERIFY_PATH ( ${NETCDF_DIRECTORY} )
+            INCLUDE_DIRECTORIES ( ${NETCDF_DIRECTORY}/include )
+            SET ( NETCDF_INCLUDE ${NETCDF_DIRECTORY}/include )
+            FIND_LIBRARY( NETCDF_NETCDF_LIB    NAMES netcdf    PATHS ${NETCDF_DIRECTORY}/lib  NO_DEFAULT_PATH )
+            FIND_LIBRARY( NETCDF_HDF5_LIB      NAMES hdf5      PATHS ${NETCDF_DIRECTORY}/lib  NO_DEFAULT_PATH )
+            FIND_LIBRARY( NETCDF_HL_LIB        NAMES hl        PATHS ${NETCDF_DIRECTORY}/lib  NO_DEFAULT_PATH )
+            IF ( NOT NETCDF_NETCDF_LIB )
+                MESSAGE( FATAL_ERROR "Did not find library for netcdf" )
+            ENDIF()
+            SET ( NETCDF_LIBS ${NETCDF_NETCDF_LIB} )
+            IF ( NETCDF_HDF5_LIB )
+                SET ( NETCDF_LIBS ${NETCDF_LIBS} ${NETCDF_HDF5_LIB} )
+            ENDIF()
+            IF ( NETCDF_HL_LIB )
+                SET ( NETCDF_LIBS ${NETCDF_LIBS} ${NETCDF_HL_LIB} )
+            ENDIF()
+        ELSE()
+            MESSAGE( FATAL_ERROR "Default search for netcdf is not yet supported.  Use -D NETCDF_DIRECTORY=" )
+        ENDIF()
+        SET( EXTERNAL_LIBS ${NETCDF_LIBS} ${HDF5_LIBS} ${EXTERNAL_LIBS} )
+        ADD_DEFINITIONS ( -DUSE_NETCDF )
+        MESSAGE( "Using netcdf" )
+        MESSAGE( "   ${NETCDF_LIBS}" )
+    ENDIF()
+ENDMACRO()
+
+
+# Macro to find and configure the silo libraries
+MACRO ( CONFIGURE_SILO )
+    # Determine if we want to use silo
+    CHECK_ENABLE_FLAG( USE_EXT_SILO 0 )
+    IF ( USE_SILO )
+        SET( USE_HDF5 1 )
+        CONFIGURE_HDF5()
+        # Check if we specified the silo directory
+        IF ( SILO_DIRECTORY )
+            VERIFY_PATH ( ${SILO_DIRECTORY} )
+            INCLUDE_DIRECTORIES ( ${SILO_DIRECTORY}/include )
+            SET ( SILO_INCLUDE ${SILO_DIRECTORY}/include )
+            FIND_LIBRARY ( SILO_LIB  NAMES siloh5  PATHS ${SILO_DIRECTORY}/lib  NO_DEFAULT_PATH )
+        ELSE()
+            MESSAGE( "Default search for silo is not yet supported")
+            MESSAGE( "Use -D SILO_DIRECTORY=" FATAL_ERROR)
+        ENDIF()
+        SET ( SILO_LIBS
+            ${SILO_LIB}
+        )
+        SET( EXTERNAL_LIBS ${SILO_LIBS} ${HDF5_LIBS} ${EXTERNAL_LIBS} )
+        ADD_DEFINITIONS ( -DUSE_SILO )  
+        MESSAGE( "Using silo" )
+        MESSAGE( "   ${SILO_LIB}" )
+    ENDIF ()
+ENDMACRO()
+
+
+# Macro to configure system-specific libraries and flags
+MACRO( CONFIGURE_SYSTEM )
+    # First check/set the compile mode
+    IF( NOT CMAKE_BUILD_TYPE )
+        MESSAGE(FATAL_ERROR "CMAKE_BUILD_TYPE is not set")
+    ENDIF()
+    # Disable gxx debug flags if we are building the visit plugin
+    # This is necessary to prvent segfaults caused by inconsistent object sizes
+    #    caused by std::vector<std::string> in the avtMeshMetaData class
+    IF ( USE_VISIT )
+        SET( DISABLE_GXX_DEBUG 1 )
+    ENDIF()
+    # Remove extra library links
+    # Get the compiler
+    SET_COMPILER_FLAGS()
+    CHECK_ENABLE_FLAG( USE_STATIC 0 )
+    # Add system dependent flags
+    MESSAGE("System is: ${CMAKE_SYSTEM_NAME}")
+    IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
+        # Windows specific system libraries
+        SET( SYSTEM_PATHS "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib/x64" 
+                          "C:/Program Files (x86)/Microsoft Visual Studio 8/VC/PlatformSDK/Lib/AMD64" 
+                          "C:/Program Files (x86)/Microsoft Visual Studio 12.0/Common7/Packages/Debugger/X64" )
+        FIND_LIBRARY( PSAPI_LIB    NAMES Psapi    PATHS ${SYSTEM_PATHS}  NO_DEFAULT_PATH )
+        FIND_LIBRARY( DBGHELP_LIB  NAMES DbgHelp  PATHS ${SYSTEM_PATHS}  NO_DEFAULT_PATH )
+        FIND_LIBRARY( DBGHELP_LIB  NAMES DbgHelp )
+        IF ( PSAPI_LIB ) 
+            ADD_DEFINITIONS( -D PSAPI )
+            SET( SYSTEM_LIBS ${PSAPI_LIB} )
+        ENDIF()
+        IF ( DBGHELP_LIB ) 
+            ADD_DEFINITIONS( -D DBGHELP )
+            SET( SYSTEM_LIBS ${DBGHELP_LIB} )
+        ELSE()
+            MESSAGE( WARNING "Did not find DbgHelp, stack trace will not be availible" )
+        ENDIF()
+        MESSAGE("System libs: ${SYSTEM_LIBS}")
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" )
+        # Linux specific system libraries
+        SET( SYSTEM_LIBS -lz -lpthread -ldl )
+        IF ( NOT USE_STATIC )
+            # Try to add rdynamic so we have names in backtrace
+            SET( CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_FLAGS} -rdynamic" )
+            CHECK_CXX_SOURCE_COMPILES( "int main() { return 0;}" rdynamic )
+            IF ( rdynamic )
+                SET( SYSTEM_LDFLAGS ${SYSTEM_LDFLAGS} -rdynamic )
+            ENDIF()
+        ENDIF()
+        # Try to add -fPIC
+        SET( CMAKE_REQUIRED_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_FLAGS} -fPIC" )
+        CHECK_CXX_SOURCE_COMPILES( "int main() { return 0;}" fPIC )
+        IF ( fPIC )
+            SET( SYSTEM_LDFLAGS ${SYSTEM_LDFLAGS} -fPIC )
+            SET( SYSTEM_LDFLAGS ${SYSTEM_LDFLAGS} -fPIC )
+        ENDIF()
+        IF ( USING_GCC )
+            SET( SYSTEM_LIBS ${SYSTEM_LIBS} -lgfortran )
+            SET(CFLAGS_EXTRA   " ${CFLAGS_EXTRA} -fPIC" )
+            SET(CXXFLAGS_EXTRA " ${CXXFLAGS_EXTRA} -fPIC" )
+        ENDIF()
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" )
+        # Max specific system libraries
+        SET( SYSTEM_LIBS -lz -lpthread -ldl )
+    ELSEIF( ${CMAKE_SYSTEM_NAME} STREQUAL "Generic" )
+        # Generic system libraries
+    ELSE()
+        MESSAGE( FATAL_ERROR "OS not detected" )
+    ENDIF()
+    # Add the static flag if necessary
+    IF ( USE_STATIC )
+        SET_STATIC_FLAGS()
+    ENDIF()
+    # Print some flags
+    MESSAGE( "LDLIBS = ${LDLIBS}" )
+ENDMACRO ()
+
+
+# Macro to configure LBPM specific options
+MACRO ( CONFIGURE_LBPM )
+    # Set the maximum number of processors for the tests
+    IF ( NOT TEST_MAX_PROCS )
+        SET( TEST_MAX_PROCS 32 )
+    ENDIF()
+    # Add the correct paths to rpath in case we build shared libraries
+    IF ( USE_STATIC )
+        SET( CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE )
+        SET( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE )
+    ELSE()
+        SET( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
+        SET( CMAKE_BUILD_WITH_INSTALL_RPATH TRUE )
+        SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}" "${LBPM_INSTALL_DIR}/lib" )
+    ENDIF()
+ENDMACRO ()

common/Array.h

@@ -0,0 +1,962 @@
+#ifndef included_ArrayClass
+#define included_ArrayClass
+
+#include <array>
+#include <cstring>
+#include <functional>
+#include <initializer_list>
+#include <iostream>
+#include <memory>
+#include <vector>
+
+#include "Utilities.h"
+
+
+#if defined( __CUDA_ARCH__ )
+#include <cuda.h>
+#define HOST_DEVICE __host__ __device__
+#else
+#define HOST_DEVICE
+#endif
+#if defined( USING_GCC ) || defined( USING_CLANG )
+#define ATTRIBUTE_INLINE __attribute__( ( always_inline ) )
+#else
+#define ATTRIBUTE_INLINE
+#endif
+
+
+#if ( defined( DEBUG ) || defined( _DEBUG ) ) && !defined( NDEBUG )
+#define CHECK_ARRAY_LENGTH( i )                                      \
+    do {                                                             \
+        if ( i >= d_length )                                         \
+            throw std::length_error( "Index exceeds array bounds" ); \
+    } while ( 0 )
+#else
+#define CHECK_ARRAY_LENGTH( i ) \
+    do {                        \
+    } while ( 0 )
+#endif
+
+
+// Forward decleration
+class FunctionTable;
+
+
+//! Simple range class
+template<class TYPE = size_t>
+class Range final
+{
+public:
+    //! Empty constructor
+    Range() : i( 0 ), j( -1 ), k( 1 ) {}
+
+    /*!
+     * Create a range i:k:j (or i:j)
+     * @param i_            Starting value
+     * @param j_            Ending value
+     * @param k_            Increment value
+     */
+    Range( TYPE i_, TYPE j_, TYPE k_ = 1 ) : i( i_ ), j( j_ ), k( k_ ) {}
+
+    TYPE i, j, k;
+};
+
+
+//! Simple class to store the array dimensions
+class ArraySize final
+{
+public:
+    //! Empty constructor
+    inline ArraySize();
+
+    /*!
+     * Create the vector size
+     * @param N1            Number of elements in the first dimension
+     */
+    inline ArraySize( size_t N1 );
+
+    /*!
+     * Create the vector size
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     */
+    inline ArraySize( size_t N1, size_t N2 );
+
+    /*!
+     * Create the vector size
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     * @param N3            Number of elements in the third dimension
+     */
+    inline ArraySize( size_t N1, size_t N2, size_t N3 );
+
+    /*!
+     * Create the vector size
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     * @param N3            Number of elements in the third dimension
+     * @param N4            Number of elements in the fourth dimension
+     */
+    inline ArraySize( size_t N1, size_t N2, size_t N3, size_t N4 );
+
+    /*!
+     * Create the vector size
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     * @param N3            Number of elements in the third dimension
+     * @param N4            Number of elements in the fourth dimension
+     * @param N5            Number of elements in the fifth dimension
+     */
+    inline ArraySize( size_t N1, size_t N2, size_t N3, size_t N4, size_t N5 );
+
+    /*!
+     * Create from initializer list
+     * @param N             Size of the array
+     */
+    inline ArraySize( std::initializer_list<size_t> N );
+
+    /*!
+     * Create from raw pointer
+     * @param ndim          Number of dimensions
+     * @param ndim          Dimensions
+     */
+    inline ArraySize( size_t ndim, const size_t *dims );
+
+    /*!
+     * Create from std::vector
+     * @param N             Size of the array
+     */
+    inline ArraySize( const std::vector<size_t> &N );
+
+    /*!
+     * Copy constructor
+     * @param rhs           Array to copy
+     */
+    inline ArraySize( const ArraySize &rhs );
+
+    /*!
+     * Move constructor
+     * @param rhs           Array to copy
+     */
+    inline ArraySize( ArraySize &&rhs );
+
+    /*!
+     * Assignment operator
+     * @param rhs           Array to copy
+     */
+    inline ArraySize &operator=( const ArraySize &rhs );
+
+    /*!
+     * Move assignment operator
+     * @param rhs           Array to copy
+     */
+    inline ArraySize &operator=( ArraySize &&rhs );
+
+    /*!
+     * Access the ith dimension
+     * @param i             Index to access
+     */
+    inline size_t operator[]( size_t i ) const { return d_N[i]; }
+
+    //! Sum the elements
+    inline uint8_t ndim() const ATTRIBUTE_INLINE { return d_ndim; }
+
+    //! Sum the elements
+    inline size_t size() const ATTRIBUTE_INLINE { return d_ndim; }
+
+    //! Sum the elements
+    inline size_t length() const ATTRIBUTE_INLINE { return d_length; }
+
+    //! Sum the elements
+    inline void resize( uint8_t dim, size_t N );
+
+    //! Returns an iterator to the beginning
+    inline const size_t *begin() const ATTRIBUTE_INLINE { return d_N; }
+
+    //! Returns an iterator to the end
+    inline const size_t *end() const ATTRIBUTE_INLINE { return d_N + d_ndim; }
+
+    // Check if two matrices are equal
+    inline bool operator==( const ArraySize &rhs ) const ATTRIBUTE_INLINE
+    {
+        return d_ndim == rhs.d_ndim && memcmp( d_N, rhs.d_N, sizeof( d_N ) ) == 0;
+    }
+
+    //! Check if two matrices are not equal
+    inline bool operator!=( const ArraySize &rhs ) const ATTRIBUTE_INLINE
+    {
+        return d_ndim != rhs.d_ndim || memcmp( d_N, rhs.d_N, sizeof( d_N ) ) != 0;
+    }
+
+    //! Maximum supported dimension
+    constexpr static uint8_t maxDim() ATTRIBUTE_INLINE { return 5u; }
+
+    //! Get the index
+    inline size_t index( size_t i ) const ATTRIBUTE_INLINE
+    {
+        CHECK_ARRAY_LENGTH( i );
+        return i;
+    }
+
+    //! Get the index
+    inline size_t index( size_t i1, size_t i2 ) const ATTRIBUTE_INLINE
+    {
+        size_t index = i1 + i2 * d_N[0];
+        CHECK_ARRAY_LENGTH( index );
+        return index;
+    }
+
+    //! Get the index
+    inline size_t index( size_t i1, size_t i2, size_t i3 ) const ATTRIBUTE_INLINE
+    {
+        size_t index = i1 + d_N[0] * ( i2 + d_N[1] * i3 );
+        CHECK_ARRAY_LENGTH( index );
+        return index;
+    }
+
+    //! Get the index
+    inline size_t index( size_t i1, size_t i2, size_t i3, size_t i4 ) const ATTRIBUTE_INLINE
+    {
+        size_t index = i1 + d_N[0] * ( i2 + d_N[1] * ( i3 + d_N[2] * i4 ) );
+        CHECK_ARRAY_LENGTH( index );
+        return index;
+    }
+
+    //! Get the index
+    inline size_t index(
+        size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const ATTRIBUTE_INLINE
+    {
+        size_t index = i1 + d_N[0] * ( i2 + d_N[1] * ( i3 + d_N[2] * ( i4 + d_N[3] * i5 ) ) );
+        CHECK_ARRAY_LENGTH( index );
+        return index;
+    }
+
+private:
+    uint8_t d_ndim;
+    size_t d_length;
+    size_t d_N[5];
+};
+
+
+/*!
+ * Class Array is a multi-dimensional array class written by Mark Berrill
+ */
+template<class TYPE, class FUN = FunctionTable>
+class Array final
+{
+public: // Constructors / assignment operators
+    /*!
+     * Create a new empty Array
+     */
+    Array();
+
+    /*!
+     * Create an Array with the given size
+     * @param N             Size of the array
+     */
+    explicit Array( const ArraySize &N );
+
+    /*!
+     * Create a new 1D Array with the given number of elements
+     * @param N             Number of elements in the array
+     */
+    explicit Array( size_t N );
+
+    /*!
+     * Create a new 2D Array with the given number of rows and columns
+     * @param N_rows        Number of rows
+     * @param N_columns     Number of columns
+     */
+    explicit Array( size_t N_rows, size_t N_columns );
+
+    /*!
+     * Create a new 3D Array with the given number of rows and columns
+     * @param N1            Number of rows
+     * @param N2            Number of columns
+     * @param N3            Number of elements in the third dimension
+     */
+    explicit Array( size_t N1, size_t N2, size_t N3 );
+
+    /*!
+     * Create a new 4D Array with the given number of rows and columns
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     * @param N3            Number of elements in the third dimension
+     * @param N4            Number of elements in the fourth dimension
+     */
+    explicit Array( size_t N1, size_t N2, size_t N3, size_t N4 );
+
+    /*!
+     * Create a new 4D Array with the given number of rows and columns
+     * @param N1            Number of elements in the first dimension
+     * @param N2            Number of elements in the second dimension
+     * @param N3            Number of elements in the third dimension
+     * @param N4            Number of elements in the fourth dimension
+     * @param N5            Number of elements in the fifth dimension
+     */
+    explicit Array( size_t N1, size_t N2, size_t N3, size_t N4, size_t N5 );
+
+    /*!
+     * Create a multi-dimensional Array with the given number of elements
+     * @param N             Number of elements in each dimension
+     * @param data          Optional raw array to copy the src data
+     */
+    explicit Array( const std::vector<size_t> &N, const TYPE *data = NULL );
+
+    /*!
+     * Create a 1D Array with the range
+     * @param range         Range of the data
+     */
+    explicit Array( const Range<TYPE> &range );
+
+    /*!
+     * Create a 1D Array with the given initializer list
+     * @param data          Input data
+     */
+    Array( std::initializer_list<TYPE> data );
+
+
+    /*!
+     * Copy constructor
+     * @param rhs           Array to copy
+     */
+    Array( const Array &rhs );
+
+    /*!
+     * Move constructor
+     * @param rhs           Array to copy
+     */
+    Array( Array &&rhs );
+
+    /*!
+     * Assignment operator
+     * @param rhs           Array to copy
+     */
+    Array &operator=( const Array &rhs );
+
+    /*!
+     * Move assignment operator
+     * @param rhs           Array to copy
+     */
+    Array &operator=( Array &&rhs );
+
+    /*!
+     * Assignment operator
+     * @param rhs           std::vector to copy
+     */
+    Array &operator=( const std::vector<TYPE> &rhs );
+
+public: // Views/copies/subset
+    /*!
+     * Create a 1D Array view to a raw block of data
+     * @param N             Number of elements in the array
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<Array> view( size_t N, std::shared_ptr<TYPE> const &data );
+
+    /*!
+     * Create a new 2D Array with the given number of rows and columns
+     * @param N_rows        Number of rows
+     * @param N_columns     Number of columns
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<Array> view(
+        size_t N_rows, size_t N_columns, std::shared_ptr<TYPE> const &data );
+
+    /*!
+     * Create a new 3D Array view to a raw block of data
+     * @param N1            Number of rows
+     * @param N2            Number of columns
+     * @param N3            Number of elements in the third dimension
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<Array> view(
+        size_t N1, size_t N2, size_t N3, std::shared_ptr<TYPE> const &data );
+
+    /*!
+     * Create a multi-dimensional Array view to a raw block of data
+     * @param N             Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<Array> view( const ArraySize &N, std::shared_ptr<TYPE> const &data );
+
+
+    /*!
+     * Create a 1D Array view to a raw block of data
+     * @param N             Number of elements in the array
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<const Array> constView(
+        size_t N, std::shared_ptr<const TYPE> const &data );
+
+    /*!
+     * Create a new 2D Array with the given number of rows and columns
+     * @param N_rows        Number of rows
+     * @param N_columns     Number of columns
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<const Array> constView(
+        size_t N_rows, size_t N_columns, std::shared_ptr<const TYPE> const &data );
+
+    /*!
+     * Create a new 3D Array view to a raw block of data
+     * @param N1            Number of rows
+     * @param N2            Number of columns
+     * @param N3            Number of elements in the third dimension
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<const Array> constView(
+        size_t N1, size_t N2, size_t N3, std::shared_ptr<const TYPE> const &data );
+
+    /*!
+     * Create a multi-dimensional Array view to a raw block of data
+     * @param N             Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    static std::shared_ptr<const Array> constView(
+        const ArraySize &N, std::shared_ptr<const TYPE> const &data );
+
+
+    /*!
+     * Make this object a view of the src
+     * @param src           Source vector to take the view of
+     */
+    void view2( Array &src );
+
+    /*!
+     * Make this object a view of the data
+     * @param N             Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    void view2( const ArraySize &N, std::shared_ptr<TYPE> const &data );
+
+    /*!
+     * Make this object a view of the raw data (expert use only).
+     * Use view2( N, std::shared_ptr(data,[](TYPE*){}) ) instead.
+     *   Note: this interface is not recommended as it does not protect from
+     *   the src data being deleted while still being used by the Array.
+     *   Additionally for maximum performance it does not set the internal shared_ptr
+     *   so functions like getPtr and resize will not work correctly.
+     * @param ndim          Number of dimensions
+     * @param dims          Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    void viewRaw( int ndim, const size_t *dims, TYPE *data );
+
+    /*!
+     * Make this object a view of the raw data (expert use only).
+     * Use view2( N, std::shared_ptr(data,[](TYPE*){}) ) instead.
+     *   Note: this interface is not recommended as it does not protect from
+     *   the src data being deleted while still being used by the Array.
+     *   Additionally for maximum performance it does not set the internal shared_ptr
+     *   so functions like getPtr and resize will not work correctly.
+     * @param N             Number of elements in each dimension
+     * @param data          Pointer to the data
+     */
+    void viewRaw( const ArraySize &N, TYPE *data );
+
+    /*!
+     * Convert an array of one type to another.  This may or may not allocate new memory.
+     * @param array         Input array
+     */
+    template<class TYPE2>
+    static std::shared_ptr<Array<TYPE2>> convert( std::shared_ptr<Array<TYPE, FUN>> array );
+
+
+    /*!
+     * Convert an array of one type to another.  This may or may not allocate new memory.
+     * @param array         Input array
+     */
+    template<class TYPE2>
+    static std::shared_ptr<const Array<TYPE2>> convert(
+        std::shared_ptr<const Array<TYPE, FUN>> array );
+
+
+    /*!
+     * Copy and convert data from another array to this array
+     * @param array         Source array
+     */
+    template<class TYPE2>
+    void copy( const Array<TYPE2> &array );
+
+    /*!
+     * Copy and convert data from a raw vector to this array.
+     *    Note: The current array must be allocated to the proper size first.
+     * @param array         Source array
+     */
+    template<class TYPE2>
+    void copy( const TYPE2 *array );
+
+    /*!
+     * Copy and convert data from this array to a raw vector.
+     * @param array         Source array
+     */
+    template<class TYPE2>
+    void copyTo( TYPE2 *array ) const;
+
+    /*!
+     * Copy and convert data from this array to a raw vector.
+     * @param array         Source array
+     */
+    template<class TYPE2>
+    Array<TYPE2, FUN> cloneTo() const;
+
+
+    /*!
+     * Fill the array with the given value
+     * @param value         Value to fill
+     */
+    void fill( const TYPE &value );
+
+    /*!
+     * Scale the array by the given value
+     * @param scale         Value to scale by
+     */
+    void scale( const TYPE &scale );
+
+    /*!
+     * Set the values of this array to pow(base, exp)
+     * @param base        Base array
+     * @param exp         Exponent value
+     */
+    void pow( const Array<TYPE, FUN> &base, const TYPE &exp );
+
+    //! Destructor
+    ~Array();
+
+
+    //! Clear the data in the array
+    void clear();
+
+
+    //! Return the size of the Array
+    inline int ndim() const { return d_size.ndim(); }
+
+
+    //! Return the size of the Array
+    inline ArraySize &size() { return d_size; }
+
+
+    //! Return the size of the Array
+    inline ArraySize size() const { return d_size; }
+
+
+    //! Return the size of the Array
+    inline size_t size( int d ) const { return d_size[d]; }
+
+
+    //! Return the size of the Array
+    inline size_t length() const { return d_size.length(); }
+
+
+    //! Return true if the Array is empty
+    inline bool empty() const { return d_size.length() == 0; }
+
+
+    /*!
+     * Resize the Array
+     * @param N             NUmber of elements
+     */
+    void resize( size_t N );
+
+    /*!
+     * Resize the Array
+     * @param N_rows        Number of rows
+     * @param N_columns     Number of columns
+     */
+    void resize( size_t N_rows, size_t N_columns );
+
+    /*!
+     * Resize the Array
+     * @param N1            Number of rows
+     * @param N2            Number of columns
+     * @param N3            Number of elements in the third dimension
+     */
+    void resize( size_t N1, size_t N2, size_t N3 );
+
+    /*!
+     * Resize the Array
+     * @param N             Number of elements in each dimension
+     */
+    void resize( const ArraySize &N );
+
+
+    /*!
+     * Resize the given dimension of the array
+     * @param dim           The dimension to resize
+     * @param N             Number of elements for the given dimension
+     * @param value         Value to initialize new elements
+     */
+    void resizeDim( int dim, size_t N, const TYPE &value );
+
+
+    /*!
+     * Reshape the Array (total size of array will not change)
+     * @param N             Number of elements in each dimension
+     */
+    void reshape( const ArraySize &N );
+
+
+    /*!
+     * Subset the Array (total size of array will not change)
+     * @param index         Index to subset (imin,imax,jmin,jmax,kmin,kmax,...)
+     */
+    template<class TYPE2 = TYPE>
+    Array<TYPE2, FUN> subset( const std::vector<size_t> &index ) const;
+
+
+    /*!
+     * Subset the Array (total size of array will not change)
+     * @param index         Index to subset (ix:kx:jx,iy:ky:jy,...)
+     */
+    template<class TYPE2 = TYPE>
+    Array<TYPE2, FUN> subset( const std::vector<Range<size_t>> &index ) const;
+
+
+    /*!
+     * Copy data from an array into a subset of this array
+     * @param index         Index of the subset (imin,imax,jmin,jmax,kmin,kmax,...)
+     * @param subset        The subset array to copy from
+     */
+    template<class TYPE2>
+    void copySubset( const std::vector<size_t> &index, const Array<TYPE2, FUN> &subset );
+
+    /*!
+     * Copy data from an array into a subset of this array
+     * @param index         Index of the subset
+     * @param subset        The subset array to copy from
+     */
+    template<class TYPE2>
+    void copySubset( const std::vector<Range<size_t>> &index, const Array<TYPE2, FUN> &subset );
+
+    /*!
+     * Add data from an array into a subset of this array
+     * @param index         Index of the subset (imin,imax,jmin,jmax,kmin,kmax,...)
+     * @param subset        The subset array to add from
+     */
+    void addSubset( const std::vector<size_t> &index, const Array<TYPE, FUN> &subset );
+
+    /*!
+     * Add data from an array into a subset of this array
+     * @param index         Index of the subset
+     * @param subset        The subset array to add from
+     */
+    void addSubset( const std::vector<Range<size_t>> &index, const Array<TYPE, FUN> &subset );
+
+
+public: // Accessors
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     */
+    HOST_DEVICE inline TYPE &operator()( size_t i ) ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     */
+    HOST_DEVICE inline const TYPE &operator()( size_t i ) const ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     * @param j             The column index
+     */
+    HOST_DEVICE inline TYPE &operator()( size_t i, size_t j ) ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i, j )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     * @param j             The column index
+     */
+    HOST_DEVICE inline const TYPE &operator()( size_t i, size_t j ) const ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i, j )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     * @param j             The column index
+     * @param k             The third index
+     */
+    HOST_DEVICE inline TYPE &operator()( size_t i, size_t j, size_t k ) ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i, j, k )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i             The row index
+     * @param j             The column index
+     * @param k             The third index
+     */
+    HOST_DEVICE inline const TYPE &operator()( size_t i, size_t j, size_t k ) const ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i, j, k )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i1            The first index
+     * @param i2            The second index
+     * @param i3            The third index
+     * @param i4            The fourth index
+     */
+    HOST_DEVICE inline TYPE &operator()(
+        size_t i1, size_t i2, size_t i3, size_t i4 ) ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i1, i2, i3, i4 )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i1            The first index
+     * @param i2            The second index
+     * @param i3            The third index
+     * @param i4            The fourth index
+     */
+    HOST_DEVICE inline const TYPE &operator()(
+        size_t i1, size_t i2, size_t i3, size_t i4 ) const ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i1, i2, i3, i4 )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i1            The first index
+     * @param i2            The second index
+     * @param i3            The third index
+     * @param i4            The fourth index
+     * @param i5            The fifth index
+     */
+    HOST_DEVICE inline TYPE &operator()(
+        size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i1, i2, i3, i4, i5 )];
+    }
+
+    /*!
+     * Access the desired element
+     * @param i1            The first index
+     * @param i2            The second index
+     * @param i3            The third index
+     * @param i4            The fourth index
+     * @param i5            The fifth index
+     */
+    HOST_DEVICE inline const TYPE &operator()(
+        size_t i1, size_t i2, size_t i3, size_t i4, size_t i5 ) const ATTRIBUTE_INLINE
+    {
+        return d_data[d_size.index( i1, i2, i3, i4, i5 )];
+    }
+
+    /*!
+     * Access the desired element as a raw pointer
+     * @param i             The global index
+     */
+    HOST_DEVICE inline TYPE *ptr( size_t i ) ATTRIBUTE_INLINE
+    {
+        return i >= d_size.length() ? nullptr : &d_data[i];
+    }
+
+    /*!
+     * Access the desired element as a raw pointer
+     * @param i             The global index
+     */
+    HOST_DEVICE inline const TYPE *ptr( size_t i ) const ATTRIBUTE_INLINE
+    {
+        return i >= d_size.length() ? nullptr : &d_data[i];
+    }
+
+    //! Get iterator to beginning of data
+    inline TYPE *begin() ATTRIBUTE_INLINE { return d_data; }
+
+    //! Get iterator to beginning of data
+    inline const TYPE *begin() const ATTRIBUTE_INLINE { return d_data; }
+
+    //! Get iterator to beginning of data
+    inline TYPE *end() ATTRIBUTE_INLINE { return d_data + d_size.length(); }
+
+    //! Get iterator to beginning of data
+    inline const TYPE *end() const ATTRIBUTE_INLINE { return d_data + d_size.length(); }
+
+    //! Return the pointer to the raw data
+    inline std::shared_ptr<TYPE> getPtr() ATTRIBUTE_INLINE { return d_ptr; }
+
+    //! Return the pointer to the raw data
+    inline std::shared_ptr<const TYPE> getPtr() const ATTRIBUTE_INLINE { return d_ptr; }
+
+    //! Return the pointer to the raw data
+    HOST_DEVICE inline TYPE *data() ATTRIBUTE_INLINE { return d_data; }
+
+    //! Return the pointer to the raw data
+    HOST_DEVICE inline const TYPE *data() const ATTRIBUTE_INLINE { return d_data; }
+
+
+public: // Operator overloading
+    //! Check if two matrices are equal
+    // Equality means the dimensions and data have to be identical
+    bool operator==( const Array &rhs ) const;
+
+    //! Check if two matrices are not equal
+    inline bool operator!=( const Array &rhs ) const { return !this->operator==( rhs ); }
+
+    //! Add another array
+    Array &operator+=( const Array &rhs );
+
+    //! Subtract another array
+    Array &operator-=( const Array &rhs );
+
+    //! Add a scalar
+    Array &operator+=( const TYPE &rhs );
+
+    //! Subtract a scalar
+    Array &operator-=( const TYPE &rhs );
+
+
+public: // Math operations
+    //! Concatenates the arrays along the dimension dim.
+    static Array cat( const std::vector<Array> &x, int dim = 0 );
+
+    //! Concatenates a given array with the current array
+    void cat( const Array &x, int dim = 0 );
+
+    //! Initialize the array with random values (defined from the function table)
+    void rand();
+
+    //! Return true if NaNs are present
+    inline bool NaNs() const;
+
+    //! Return the smallest value
+    inline TYPE min() const;
+
+    //! Return the largest value
+    inline TYPE max() const;
+
+    //! Return the sum of all elements
+    inline TYPE sum() const;
+
+    //! Return the mean of all elements
+    inline TYPE mean() const;
+
+    //! Return the min of all elements in a given direction
+    Array<TYPE, FUN> min( int dir ) const;
+
+    //! Return the max of all elements in a given direction
+    Array<TYPE, FUN> max( int dir ) const;
+
+    //! Return the sum of all elements in a given direction
+    Array<TYPE, FUN> sum( int dir ) const;
+
+    //! Return the smallest value
+    inline TYPE min( const std::vector<size_t> &index ) const;
+
+    //! Return the largest value
+    inline TYPE max( const std::vector<size_t> &index ) const;
+
+    //! Return the sum of all elements
+    inline TYPE sum( const std::vector<size_t> &index ) const;
+
+    //! Return the mean of all elements
+    inline TYPE mean( const std::vector<size_t> &index ) const;
+
+    //! Return the smallest value
+    inline TYPE min( const std::vector<Range<size_t>> &index ) const;
+
+    //! Return the largest value
+    inline TYPE max( const std::vector<Range<size_t>> &index ) const;
+
+    //! Return the sum of all elements
+    inline TYPE sum( const std::vector<Range<size_t>> &index ) const;
+
+    //! Return the mean of all elements
+    inline TYPE mean( const std::vector<Range<size_t>> &index ) const;
+
+    //! Find all elements that match the operator
+    std::vector<size_t> find(
+        const TYPE &value, std::function<bool( const TYPE &, const TYPE & )> compare ) const;
+
+
+    //! Print an array
+    void print(
+        std::ostream &os, const std::string &name = "A", const std::string &prefix = "" ) const;
+
+    //! Multiply two arrays
+    static Array multiply( const Array &a, const Array &b );
+
+    //! Transpose an array
+    Array<TYPE, FUN> reverseDim() const;
+
+    //! Replicate an array a given number of times in each direction
+    Array<TYPE, FUN> repmat( const std::vector<size_t> &N ) const;
+
+    //! Coarsen an array using the given filter
+    Array<TYPE, FUN> coarsen( const Array<TYPE, FUN> &filter ) const;
+
+    //! Coarsen an array using the given filter
+    Array<TYPE, FUN> coarsen( const std::vector<size_t> &ratio,
+        std::function<TYPE( const Array<TYPE, FUN> & )> filter ) const;
+
+    /*!
+     * Perform a element-wise operation y = f(x)
+     * @param[in] fun           The function operation
+     * @param[in] x             The input array
+     */
+    static Array transform( std::function<TYPE( const TYPE & )> fun, const Array &x );
+
+    /*!
+     * Perform a element-wise operation z = f(x,y)
+     * @param[in] fun           The function operation
+     * @param[in] x             The first array
+     * @param[in] y             The second array
+     */
+    static Array transform(
+        std::function<TYPE( const TYPE &, const TYPE & )> fun, const Array &x, const Array &y );
+
+    /*!
+     * axpby operation: this = alpha*x + beta*this
+     * @param[in] alpha         alpha
+     * @param[in] x             x
+     * @param[in] beta          beta
+     */
+    void axpby( const TYPE &alpha, const Array<TYPE, FUN> &x, const TYPE &beta );
+
+private:
+    ArraySize d_size;            // Size of each dimension
+    TYPE *d_data;                // Raw pointer to data in array
+    std::shared_ptr<TYPE> d_ptr; // Shared pointer to data in array
+    void allocate( const ArraySize &N );
+
+public:
+    template<class TYPE2, class FUN2>
+    inline bool sizeMatch( const Array<TYPE2, FUN2> &rhs ) const
+    {
+        return d_size == rhs.d_size;
+    }
+
+private:
+    inline void checkSubsetIndex( const std::vector<Range<size_t>> &range ) const;
+    inline std::vector<Range<size_t>> convert( const std::vector<size_t> &index ) const;
+    static inline void getSubsetArrays( const std::vector<Range<size_t>> &range,
+        std::array<size_t, 5> &first, std::array<size_t, 5> &last, std::array<size_t, 5> &inc,
+        std::array<size_t, 5> &N );
+};
+
+
+typedef Array<double> DoubleArray;
+typedef Array<float> FloatArray;
+typedef Array<int> IntArray;
+
+
+#include "common/Array.hpp"
+
+#endif

common/Communication.cpp

@@ -0,0 +1,74 @@
+#include "common/Communication.h"
+
+
+/********************************************************
+*  Structure to store the rank info                     *
+********************************************************/
+inline int getRankForBlock( int nprocx, int nprocy, int nprocz, int i, int j, int k )
+{
+	int i2 = (i+nprocx)%nprocx;
+	int j2 = (j+nprocy)%nprocy;
+	int k2 = (k+nprocz)%nprocz;
+	return i2 + j2*nprocx + k2*nprocx*nprocy;
+}
+RankInfoStruct::RankInfoStruct()
+{
+    memset(this,0,sizeof(RankInfoStruct));
+}
+RankInfoStruct::RankInfoStruct( int rank0, int nprocx, int nprocy, int nprocz )
+{
+    memset(this,0,sizeof(RankInfoStruct));
+    nx = nprocx;
+    ny = nprocy;
+    nz = nprocz;
+	ix = rank0%nprocx;
+	jy = (rank0/nprocx)%nprocy;
+	kz = rank0/(nprocx*nprocy);
+    for (int i=-1; i<=1; i++) {
+        for (int j=-1; j<=1; j++) {
+            for (int k=-1; k<=1; k++) {
+                rank[i+1][j+1][k+1] = getRankForBlock(nprocx,nprocy,nprocz,ix+i,jy+j,kz+k);
+            }
+        }
+    }
+    ASSERT(rank[1][1][1]==rank0);
+}
+
+
+/********************************************************
+*  Deprecated functions                                 *
+********************************************************/
+void InitializeRanks( const int rank, const int nprocx, const int nprocy, const int nprocz,
+	int& iproc, int& jproc, int& kproc, 
+	int& rank_x, int& rank_y, int& rank_z, 
+	int& rank_X, int& rank_Y, int& rank_Z,
+	int& rank_xy, int& rank_XY, int& rank_xY, int& rank_Xy,
+	int& rank_xz, int& rank_XZ, int& rank_xZ, int& rank_Xz,
+	int& rank_yz, int& rank_YZ, int& rank_yZ, int& rank_Yz )
+{
+    const RankInfoStruct data(rank,nprocx,nprocy,nprocz);
+	iproc = data.ix;
+    jproc = data.jy;
+    kproc = data.kz;
+	rank_X = data.rank[2][1][1];
+	rank_x = data.rank[0][1][1];
+	rank_Y = data.rank[1][2][1];
+	rank_y = data.rank[1][0][1];
+	rank_Z = data.rank[1][1][2];
+	rank_z = data.rank[1][1][0];
+	rank_XY = data.rank[2][2][1];
+	rank_xy = data.rank[0][0][1];
+	rank_Xy = data.rank[2][0][1];
+	rank_xY = data.rank[0][2][1];
+	rank_XZ = data.rank[2][1][2];
+	rank_xz = data.rank[0][1][0];
+	rank_Xz = data.rank[2][1][0];
+	rank_xZ = data.rank[0][1][2];
+	rank_YZ = data.rank[1][2][2];
+	rank_yz = data.rank[1][0][0];
+	rank_Yz = data.rank[1][2][0];
+	rank_yZ = data.rank[1][0][2];
+}
+
+
+

common/Communication.h

@@ -0,0 +1,373 @@
+#ifndef COMMUNICATION_H_INC
+#define COMMUNICATION_H_INC
+
+#include "common/MPI_Helpers.h"
+#include "common/Utilities.h"
+#include "common/Array.h"
+
+// ********** COMMUNICTION **************************************
+/*
+ //..............................................................
+ // Communication helper routines for MPI
+ //..............................................................
+ */
+
+using namespace std;
+
+
+/*!
+ * @brief  Rank info structure
+ * @details  Structure used to hold the ranks for the current process and it's neighbors
+ */
+struct RankInfoStruct {
+    int nx;                 //!<  The number of processors in the x direction
+    int ny;                 //!<  The number of processors in the y direction
+    int nz;                 //!<  The number of processors in the z direction
+    int ix;                 //!<  The index of the current process in the x direction
+    int jy;                 //!<  The index of the current process in the y direction
+    int kz;                 //!<  The index of the current process in the z direction
+    int rank[3][3][3];      //!<  The rank for the neighbor [i][j][k]
+    RankInfoStruct();
+    RankInfoStruct( int rank, int nprocx, int nprocy, int nprocz );
+};
+
+
+/*!
+ * @brief  Communicate halo
+ * @details  Fill the halo cells in an array from the neighboring processes
+ */
+template<class TYPE>
+class fillHalo
+{
+public:
+    /*!
+     * @brief  Default constructor
+     * @param[in] info          Rank and neighbor rank info
+     * @param[in] nx            Number of local cells in the x direction
+     * @param[in] ny            Number of local cells in the y direction
+     * @param[in] nz            Number of local cells in the z direction
+     * @param[in] ngx           Number of ghost cells in the x direction
+     * @param[in] ngy           Number of ghost cells in the y direction
+     * @param[in] ngz           Number of ghost cells in the z direction
+     * @param[in] tag           Initial tag to use for the communication (we will require tag:tag+26)
+     * @param[in] depth         Maximum depth to support
+     */
+    fillHalo( MPI_Comm comm, const RankInfoStruct& info, int nx, int ny, int nz, 
+        int ngx, int ngy, int ngz, int tag, int depth,
+        bool fill_face=true, bool fill_edge=true, bool fill_corner=true );
+
+    //!  Destructor
+    ~fillHalo( );
+
+    /*!
+     * @brief  Communicate the halos
+     * @param[in] array         The array on which we fill the halos
+     */
+    void fill( Array<TYPE>& array );
+
+    /*!
+     * @brief  Copy data from the src array to the dst array
+     * @param[in] src           The src array with or without halos
+     * @param[in] dst           The dst array with or without halos
+     */
+    template<class TYPE1, class TYPE2>
+    void copy( const Array<TYPE1>& src, Array<TYPE2>& dst );
+
+
+private:
+    RankInfoStruct info;
+    int nx, ny, nz, ngx, ngy, ngz, depth;
+    bool fill_pattern[3][3][3];
+    int tag[3][3][3];
+    int N_send_recv[3][3][3];
+    TYPE *mem;
+    TYPE *send[3][3][3], *recv[3][3][3];
+    MPI_Request send_req[3][3][3], recv_req[3][3][3];
+    MPI_Comm comm;
+    MPI_Datatype datatype;
+    fillHalo();                             // Private empty constructor
+    fillHalo(const fillHalo&);              // Private copy constructor
+    fillHalo& operator=(const fillHalo&);   // Private assignment operator
+    void pack( const Array<TYPE>& array, int i, int j, int k, TYPE *buffer );
+    void unpack( Array<TYPE>& array, int i, int j, int k, const TYPE *buffer );
+};
+
+
+//***************************************************************************************
+inline void PackMeshData(int *list, int count, double *sendbuf, double *data){
+	// Fill in the phase ID values from neighboring processors
+	// This packs up the values that need to be sent from one processor to another
+	int idx,n;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		sendbuf[idx] = data[n];
+	}
+}
+inline void UnpackMeshData(int *list, int count, double *recvbuf, double *data){
+	// Fill in the phase ID values from neighboring processors
+	// This unpacks the values once they have been recieved from neighbors
+	int idx,n;
+
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		data[n] = recvbuf[idx];
+	}
+}
+
+
+// Initialize the ranks (this is deprecated, see RankInfoStruct)
+void InitializeRanks( const int rank, const int nprocx, const int nprocy, const int nprocz,
+	int& iproc, int& jproc, int& kproc, 
+	int& rank_x, int& rank_y, int& rank_z, 
+	int& rank_X, int& rank_Y, int& rank_Z,
+	int& rank_xy, int& rank_XY, int& rank_xY, int& rank_Xy,
+	int& rank_xz, int& rank_XZ, int& rank_xZ, int& rank_Xz,
+	int& rank_yz, int& rank_YZ, int& rank_yZ, int& rank_Yz );
+
+
+//***************************************************************************************
+inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag, 
+		int rank_x, int rank_y, int rank_z, 
+		int rank_X, int rank_Y, int rank_Z,
+		int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
+		int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz,
+		int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz,
+		int sendCount_x, int sendCount_y, int sendCount_z, 
+		int sendCount_X, int sendCount_Y, int sendCount_Z,
+		int sendCount_xy, int sendCount_XY, int sendCount_xY, int sendCount_Xy,
+		int sendCount_xz, int sendCount_XZ, int sendCount_xZ, int sendCount_Xz,
+		int sendCount_yz, int sendCount_YZ, int sendCount_yZ, int sendCount_Yz,
+		int& recvCount_x, int& recvCount_y, int& recvCount_z, 
+		int& recvCount_X, int& recvCount_Y, int& recvCount_Z,
+		int& recvCount_xy, int& recvCount_XY, int& recvCount_xY, int& recvCount_Xy,
+		int& recvCount_xz, int& recvCount_XZ, int& recvCount_xZ, int& recvCount_Xz,
+		int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
+{
+	MPI_Request req1[18], req2[18];
+	MPI_Status stat1[18],stat2[18];
+	MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
+	MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
+	MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
+	MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
+	MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
+	MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
+	MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
+	MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
+	MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
+	MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
+	MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
+	MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
+
+	MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
+	MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
+	MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
+	MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
+	MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
+	MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
+	MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
+	MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
+
+	MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
+	MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
+	MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
+	MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
+	MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
+	MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
+	MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
+	MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
+
+	MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
+	MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
+	MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
+	MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
+	MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
+	MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
+	MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
+	MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
+	MPI_Waitall(18,req1,stat1);
+	MPI_Waitall(18,req2,stat2);
+	MPI_Barrier(Communicator);
+}
+
+
+//***************************************************************************************
+inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag, 
+		int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
+		int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
+		int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
+		int *sendList_yz, int *sendList_YZ, int *sendList_yZ, int *sendList_Yz,
+		int sendCount_x, int sendCount_y, int sendCount_z, int sendCount_X, int sendCount_Y, int sendCount_Z,
+		int sendCount_xy, int sendCount_XY, int sendCount_xY, int sendCount_Xy,
+		int sendCount_xz, int sendCount_XZ, int sendCount_xZ, int sendCount_Xz,
+		int sendCount_yz, int sendCount_YZ, int sendCount_yZ, int sendCount_Yz,
+		int *recvList_x, int *recvList_y, int *recvList_z, int *recvList_X, int *recvList_Y, int *recvList_Z,
+		int *recvList_xy, int *recvList_XY, int *recvList_xY, int *recvList_Xy,
+		int *recvList_xz, int *recvList_XZ, int *recvList_xZ, int *recvList_Xz,
+		int *recvList_yz, int *recvList_YZ, int *recvList_yZ, int *recvList_Yz,
+		int recvCount_x, int recvCount_y, int recvCount_z, int recvCount_X, int recvCount_Y, int recvCount_Z,
+		int recvCount_xy, int recvCount_XY, int recvCount_xY, int recvCount_Xy,
+		int recvCount_xz, int recvCount_XZ, int recvCount_xZ, int recvCount_Xz,
+		int recvCount_yz, int recvCount_YZ, int recvCount_yZ, int recvCount_Yz,
+		int rank_x, int rank_y, int rank_z, int rank_X, int rank_Y, int rank_Z, int rank_xy, int rank_XY, int rank_xY,
+		int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
+{
+	MPI_Request req1[18], req2[18];
+	MPI_Status stat1[18],stat2[18];
+	MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
+	MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
+	MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
+	MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
+	MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
+	MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
+	MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
+	MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
+	MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
+	MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
+	MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
+	MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
+
+	MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
+	MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
+	MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
+	MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
+	MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
+	MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
+	MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
+	MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
+
+	MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
+	MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
+	MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
+	MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
+	MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
+	MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
+	MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
+	MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
+
+	MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
+	MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
+	MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
+	MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
+	MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
+	MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
+	MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
+	MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
+	MPI_Waitall(18,req1,stat1);
+	MPI_Waitall(18,req2,stat2);
+}
+
+
+//***************************************************************************************
+inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
+		double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
+		double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
+		double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
+		double *sendbuf_yz,double *sendbuf_YZ,double *sendbuf_yZ,double *sendbuf_Yz,
+		double *recvbuf_x,double *recvbuf_y,double *recvbuf_z,double *recvbuf_X,double *recvbuf_Y,double *recvbuf_Z,
+		double *recvbuf_xy,double *recvbuf_XY,double *recvbuf_xY,double *recvbuf_Xy,
+		double *recvbuf_xz,double *recvbuf_XZ,double *recvbuf_xZ,double *recvbuf_Xz,
+		double *recvbuf_yz,double *recvbuf_YZ,double *recvbuf_yZ,double *recvbuf_Yz,
+		int *sendList_x,int *sendList_y,int *sendList_z,int *sendList_X,int *sendList_Y,int *sendList_Z,
+		int *sendList_xy,int *sendList_XY,int *sendList_xY,int *sendList_Xy,
+		int *sendList_xz,int *sendList_XZ,int *sendList_xZ,int *sendList_Xz,
+		int *sendList_yz,int *sendList_YZ,int *sendList_yZ,int *sendList_Yz,
+		int sendCount_x,int sendCount_y,int sendCount_z,int sendCount_X,int sendCount_Y,int sendCount_Z,
+		int sendCount_xy,int sendCount_XY,int sendCount_xY,int sendCount_Xy,
+		int sendCount_xz,int sendCount_XZ,int sendCount_xZ,int sendCount_Xz,
+		int sendCount_yz,int sendCount_YZ,int sendCount_yZ,int sendCount_Yz,
+		int *recvList_x,int *recvList_y,int *recvList_z,int *recvList_X,int *recvList_Y,int *recvList_Z,
+		int *recvList_xy,int *recvList_XY,int *recvList_xY,int *recvList_Xy,
+		int *recvList_xz,int *recvList_XZ,int *recvList_xZ,int *recvList_Xz,
+		int *recvList_yz,int *recvList_YZ,int *recvList_yZ,int *recvList_Yz,
+		int recvCount_x,int recvCount_y,int recvCount_z,int recvCount_X,int recvCount_Y,int recvCount_Z,
+		int recvCount_xy,int recvCount_XY,int recvCount_xY,int recvCount_Xy,
+		int recvCount_xz,int recvCount_XZ,int recvCount_xZ,int recvCount_Xz,
+		int recvCount_yz,int recvCount_YZ,int recvCount_yZ,int recvCount_Yz,
+		int rank_x,int rank_y,int rank_z,int rank_X,int rank_Y,int rank_Z,int rank_xy,int rank_XY,int rank_xY,
+		int rank_Xy,int rank_xz,int rank_XZ,int rank_xZ,int rank_Xz,int rank_yz,int rank_YZ,int rank_yZ,int rank_Yz)
+{
+	int sendtag, recvtag;
+	sendtag = recvtag = 7;
+    double *MeshData = Mesh.data();
+	PackMeshData(sendList_x, sendCount_x ,sendbuf_x, MeshData);
+	PackMeshData(sendList_X, sendCount_X ,sendbuf_X, MeshData);
+	PackMeshData(sendList_y, sendCount_y ,sendbuf_y, MeshData);
+	PackMeshData(sendList_Y, sendCount_Y ,sendbuf_Y, MeshData);
+	PackMeshData(sendList_z, sendCount_z ,sendbuf_z, MeshData);
+	PackMeshData(sendList_Z, sendCount_Z ,sendbuf_Z, MeshData);
+	PackMeshData(sendList_xy, sendCount_xy ,sendbuf_xy, MeshData);
+	PackMeshData(sendList_Xy, sendCount_Xy ,sendbuf_Xy, MeshData);
+	PackMeshData(sendList_xY, sendCount_xY ,sendbuf_xY, MeshData);
+	PackMeshData(sendList_XY, sendCount_XY ,sendbuf_XY, MeshData);
+	PackMeshData(sendList_xz, sendCount_xz ,sendbuf_xz, MeshData);
+	PackMeshData(sendList_Xz, sendCount_Xz ,sendbuf_Xz, MeshData);
+	PackMeshData(sendList_xZ, sendCount_xZ ,sendbuf_xZ, MeshData);
+	PackMeshData(sendList_XZ, sendCount_XZ ,sendbuf_XZ, MeshData);
+	PackMeshData(sendList_yz, sendCount_yz ,sendbuf_yz, MeshData);
+	PackMeshData(sendList_Yz, sendCount_Yz ,sendbuf_Yz, MeshData);
+	PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
+	PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
+	//......................................................................................
+	MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag,
+			recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag,
+			recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag,
+			recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,
+			recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag,
+			recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,
+			recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,
+			recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,
+			recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,
+			recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,
+			recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,
+			recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,
+			recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,
+			recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,
+			recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,
+			recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,
+			recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,
+			recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
+	MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,
+			recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
+	//........................................................................................
+	UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
+	UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);
+	UnpackMeshData(recvList_y, recvCount_y ,recvbuf_y, MeshData);
+	UnpackMeshData(recvList_Y, recvCount_Y ,recvbuf_Y, MeshData);
+	UnpackMeshData(recvList_z, recvCount_z ,recvbuf_z, MeshData);
+	UnpackMeshData(recvList_Z, recvCount_Z ,recvbuf_Z, MeshData);
+	UnpackMeshData(recvList_xy, recvCount_xy ,recvbuf_xy, MeshData);
+	UnpackMeshData(recvList_Xy, recvCount_Xy ,recvbuf_Xy, MeshData);
+	UnpackMeshData(recvList_xY, recvCount_xY ,recvbuf_xY, MeshData);
+	UnpackMeshData(recvList_XY, recvCount_XY ,recvbuf_XY, MeshData);
+	UnpackMeshData(recvList_xz, recvCount_xz ,recvbuf_xz, MeshData);
+	UnpackMeshData(recvList_Xz, recvCount_Xz ,recvbuf_Xz, MeshData);
+	UnpackMeshData(recvList_xZ, recvCount_xZ ,recvbuf_xZ, MeshData);
+	UnpackMeshData(recvList_XZ, recvCount_XZ ,recvbuf_XZ, MeshData);
+	UnpackMeshData(recvList_yz, recvCount_yz ,recvbuf_yz, MeshData);
+	UnpackMeshData(recvList_Yz, recvCount_Yz ,recvbuf_Yz, MeshData);
+	UnpackMeshData(recvList_yZ, recvCount_yZ ,recvbuf_yZ, MeshData);
+	UnpackMeshData(recvList_YZ, recvCount_YZ ,recvbuf_YZ, MeshData);
+}
+
+
+#endif
+
+#include "common/Communication.hpp"
+
+

common/Communication.hpp

@@ -0,0 +1,261 @@
+#ifndef COMMUNICATION_HPP_INC
+#define COMMUNICATION_HPP_INC
+
+#include "common/Communication.h"
+#include "common/MPI_Helpers.h"
+#include "common/Utilities.h"
+//#include "ProfilerApp.h"
+
+
+/********************************************************
+*  Structure to store the rank info                     *
+********************************************************/
+template<class TYPE>
+fillHalo<TYPE>::fillHalo( MPI_Comm comm0, const RankInfoStruct& info0, int nx0, int ny0, int nz0, 
+    int ngx0, int ngy0, int ngz0, int tag0, int depth0,
+    bool fill_face, bool fill_edge, bool fill_corner ):
+    info(info0), nx(nx0), ny(ny0), nz(nz0), ngx(ngx0), ngy(ngy0), ngz(ngz0), depth(depth0)
+{
+    comm = comm0;
+    datatype = getMPItype<TYPE>();
+    // Set the fill pattern
+    memset(fill_pattern,0,sizeof(fill_pattern));
+    if ( fill_face ) {
+        fill_pattern[0][1][1] = true;
+        fill_pattern[2][1][1] = true;
+        fill_pattern[1][0][1] = true;
+        fill_pattern[1][2][1] = true;
+        fill_pattern[1][1][0] = true;
+        fill_pattern[1][1][2] = true;
+    }
+    if ( fill_edge ) {
+        fill_pattern[0][0][1] = true;
+        fill_pattern[0][2][1] = true;
+        fill_pattern[2][0][1] = true;
+        fill_pattern[2][2][1] = true;
+        fill_pattern[0][1][0] = true;
+        fill_pattern[0][1][2] = true;
+        fill_pattern[2][1][0] = true;
+        fill_pattern[2][1][2] = true;
+        fill_pattern[1][0][0] = true;
+        fill_pattern[1][0][2] = true;
+        fill_pattern[1][2][0] = true;
+        fill_pattern[1][2][2] = true;
+    }
+    if ( fill_corner ) {
+        fill_pattern[0][0][0] = true;
+        fill_pattern[0][0][2] = true;
+        fill_pattern[0][2][0] = true;
+        fill_pattern[0][2][2] = true;
+        fill_pattern[2][0][0] = true;
+        fill_pattern[2][0][2] = true;
+        fill_pattern[2][2][0] = true;
+        fill_pattern[2][2][2] = true;
+    }
+    // Determine the number of elements for each send/recv
+    for (int i=0; i<3; i++) {
+        int ni = (i-1)==0 ? nx:ngx;
+        for (int j=0; j<3; j++) {
+            int nj = (j-1)==0 ? ny:ngy;
+            for (int k=0; k<3; k++) {
+                int nk = (k-1)==0 ? nz:ngz;
+                if ( fill_pattern[i][j][k] )
+                    N_send_recv[i][j][k] = ni*nj*nk;
+                else
+                    N_send_recv[i][j][k] = 0;
+            }
+        }
+    }
+    // Create send/recv buffers
+    size_t N_mem=0;
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++)
+                N_mem += N_send_recv[i][j][k];
+        }
+    }
+    mem = new TYPE[2*depth*N_mem];
+    size_t index = 0;
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++) {
+                send[i][j][k] = &mem[index];
+                index += depth*N_send_recv[i][j][k];
+                recv[i][j][k] = &mem[index];
+                index += depth*N_send_recv[i][j][k];
+            }
+        }
+    }
+    // Create the tags
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++) {
+                tag[i][j][k] = tag0 + i + j*3 + k*9;
+            }
+        }
+    }
+
+}
+template<class TYPE>
+fillHalo<TYPE>::~fillHalo( )
+{
+    delete [] mem;
+}
+template<class TYPE>
+void fillHalo<TYPE>::fill( Array<TYPE>& data )
+{
+    //PROFILE_START("fillHalo::fill",1);
+    int depth2 = data.size(3);
+    ASSERT((int)data.size(0)==nx+2*ngx);
+    ASSERT((int)data.size(1)==ny+2*ngy);
+    ASSERT((int)data.size(2)==nz+2*ngz);
+    ASSERT(depth2<=depth);
+    ASSERT(data.ndim()==3||data.ndim()==4);
+    // Start the recieves
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++) {
+                if ( !fill_pattern[i][j][k] )
+                    continue;
+                MPI_Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k], datatype, 
+                    info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] );
+            }
+        }
+    }
+    // Pack the src data and start the sends
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++) {
+                if ( !fill_pattern[i][j][k] )
+                    continue;
+                pack( data, i-1, j-1, k-1, send[i][j][k] );
+                MPI_Isend( send[i][j][k], depth2*N_send_recv[i][j][k], datatype, 
+                    info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] );
+            }
+        }
+    }
+    // Recv the dst data and unpack (we recive in reverse order to match the sends)
+    MPI_Status status;
+    for (int i=2; i>=0; i--) {
+        for (int j=2; j>=0; j--) {
+            for (int k=2; k>=0; k--) {
+                if ( !fill_pattern[i][j][k] )
+                    continue;
+                MPI_Wait(&recv_req[i][j][k],&status);
+                unpack( data, i-1, j-1, k-1, recv[i][j][k] );
+            }
+        }
+    }
+    // Wait until all sends have completed
+    for (int i=0; i<3; i++) {
+        for (int j=0; j<3; j++) {
+            for (int k=0; k<3; k++) {
+                if ( !fill_pattern[i][j][k] )
+                    continue;
+                MPI_Wait(&send_req[i][j][k],&status);
+            }
+        }
+    }
+    //PROFILE_STOP("fillHalo::fill",1);
+}
+template<class TYPE>
+void fillHalo<TYPE>::pack( const Array<TYPE>& data, int i0, int j0, int k0, TYPE *buffer )
+{
+    int depth2 = data.size(3);
+    int ni = i0==0 ? nx:ngx;
+    int nj = j0==0 ? ny:ngy;
+    int nk = k0==0 ? nz:ngz;
+    int is = i0==0 ? ngx:((i0==-1)?ngx:nx);
+    int js = j0==0 ? ngy:((j0==-1)?ngy:ny);
+    int ks = k0==0 ? ngz:((k0==-1)?ngz:nz);
+    for (int d=0; d<depth2; d++) {
+        for (int k=0; k<nk; k++) {
+            for (int j=0; j<nj; j++) {
+                for (int i=0; i<ni; i++) {
+                    buffer[i+j*ni+k*ni*nj+d*ni*nj*nk] = data(i+is,j+js,k+ks,d);
+                }
+            }
+        }
+    }
+}
+template<class TYPE>
+void fillHalo<TYPE>::unpack( Array<TYPE>& data, int i0, int j0, int k0, const TYPE *buffer )
+{
+    int depth2 = data.size(3);
+    int ni = i0==0 ? nx:ngx;
+    int nj = j0==0 ? ny:ngy;
+    int nk = k0==0 ? nz:ngz;
+    int is = i0==0 ? ngx:((i0==-1)?0:nx+ngx);
+    int js = j0==0 ? ngy:((j0==-1)?0:ny+ngy);
+    int ks = k0==0 ? ngz:((k0==-1)?0:nz+ngz);
+    for (int d=0; d<depth2; d++) {
+        for (int k=0; k<nk; k++) {
+            for (int j=0; j<nj; j++) {
+                for (int i=0; i<ni; i++) {
+                    data(i+is,j+js,k+ks,d) = buffer[i+j*ni+k*ni*nj+d*ni*nj*nk];
+                }
+            }
+        }
+    }
+}
+
+
+/********************************************************
+*  Function to remove the ghost halo                    *
+********************************************************/
+template<class TYPE>
+template<class TYPE1, class TYPE2>
+void fillHalo<TYPE>::copy( const Array<TYPE1>& src, Array<TYPE2>& dst )
+{
+    //PROFILE_START("fillHalo::copy",1);
+    ASSERT( (int)src.size(0)==nx || (int)src.size(0)==nx+2*ngx );
+    ASSERT( (int)dst.size(0)==nx || (int)dst.size(0)==nx+2*ngx );
+    bool src_halo = (int)src.size(0)==nx+2*ngx;
+    bool dst_halo = (int)dst.size(0)==nx+2*ngx;
+    if ( src_halo ) {
+        ASSERT((int)src.size(0)==nx+2*ngx);
+        ASSERT((int)src.size(1)==ny+2*ngy);
+        ASSERT((int)src.size(2)==nz+2*ngz);
+    } else {
+        ASSERT((int)src.size(0)==nx);
+        ASSERT((int)src.size(1)==ny);
+        ASSERT((int)src.size(2)==nz);
+    }
+    if ( dst_halo ) {
+        ASSERT((int)dst.size(0)==nx+2*ngx);
+        ASSERT((int)dst.size(1)==ny+2*ngy);
+        ASSERT((int)dst.size(2)==nz+2*ngz);
+    } else {
+        ASSERT((int)dst.size(0)==nx);
+        ASSERT((int)dst.size(1)==ny);
+        ASSERT((int)dst.size(2)==nz);
+    }
+    if ( src_halo == dst_halo ) {
+        // Src and dst halos match
+        for (size_t i=0; i<src.length(); i++)
+            dst(i) = src(i);
+    } else if ( src_halo && !dst_halo ) {
+        // Src has halos
+        for (int k=0; k<nz; k++) {
+            for (int j=0; j<ny; j++) {
+                for (int i=0; i<nx; i++) {
+                    dst(i,j,k) = src(i+ngx,j+ngy,k+ngz);
+                }
+            }
+        }
+    } else if ( !src_halo && dst_halo ) {
+        // Dst has halos
+        for (int k=0; k<nz; k++) {
+            for (int j=0; j<ny; j++) {
+                for (int i=0; i<nx; i++) {
+                    dst(i+ngx,j+ngy,k+ngz) = src(i,j,k);
+                }
+            }
+        }
+        fill(dst);
+    }
+    //PROFILE_STOP("fillHalo::copy",1);
+}
+
+
+#endif

common/Domain.h

@@ -0,0 +1,567 @@
+#ifndef Domain_INC
+#define Domain_INC
+// Created by James McClure
+// Copyright 2008-2013
+
+//#define _GLIBCXX_USE_CXX11_ABI 0
+#include <stdio.h>
+#include <stdlib.h>
+#include <iostream>
+#include <fstream>
+#include <math.h>
+#include <time.h>
+#include <exception>      // std::exception
+#include <stdexcept>
+
+#include "common/Array.h"
+#include "common/Utilities.h"
+#include "common/MPI_Helpers.h"
+#include "common/Communication.h"
+
+using namespace std;
+
+
+//! Read the domain information file
+void read_domain( int rank, int nprocs, MPI_Comm comm, 
+    int& nprocx, int& nprocy, int& nprocz, int& nx, int& ny, int& nz,
+    int& nspheres, double& Lx, double& Ly, double& Lz );
+
+
+//! Class to hold domain info
+struct Domain{
+    // Default constructor
+	Domain(int nx, int ny, int nz, int rnk, int npx, int npy, int npz, 
+			double lx, double ly, double lz, int BC);
+
+    // Destructor
+	~Domain();
+    
+	// Basic domain information
+	int Nx,Ny,Nz,N;
+	int iproc,jproc,kproc;
+ 	int nprocx,nprocy,nprocz;
+    double Lx,Ly,Lz,Volume;
+	int rank;
+	int BoundaryCondition;
+	RankInfoStruct rank_info;
+	MPI_Group Group;	// Group of processors associated with this domain
+	MPI_Comm Comm;		// MPI Communicator for this domain
+
+	//**********************************
+	// MPI ranks for all 18 neighbors
+	//**********************************
+	int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
+	int rank_xy,rank_XY,rank_xY,rank_Xy;
+	int rank_xz,rank_XZ,rank_xZ,rank_Xz;
+	int rank_yz,rank_YZ,rank_yZ,rank_Yz;
+	//**********************************
+	//......................................................................................
+	// Get the actual D3Q19 communication counts (based on location of solid phase)
+	// Discrete velocity set symmetry implies the sendcount = recvcount
+	//......................................................................................
+	int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
+	int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
+	int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
+	//......................................................................................
+	int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
+	int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
+	int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
+	//......................................................................................
+	int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
+	int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
+	int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ;
+	//......................................................................................
+	int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
+	int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
+	int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
+	//......................................................................................
+	int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
+	int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
+	int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
+	//......................................................................................
+	int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z;
+	int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ;
+	int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ;
+	//......................................................................................	
+	double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z;
+	double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ;
+	double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ;
+	double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z;
+	double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ;
+	double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ;
+
+	// Solid indicator function
+	char *id;
+
+	void InitializeRanks();
+	void CommInit(MPI_Comm comm);
+	void CommunicateMeshHalo(DoubleArray &Mesh);
+	void AssignComponentLabels(double *phase);
+
+	void TestCommInit(MPI_Comm comm);
+
+    //void MemoryOptimizedLayout(IntArray &Map, int *neighborList, int Np);
+
+private:
+
+	inline int getRankForBlock( int i, int j, int k )
+	{
+		int i2 = (i+nprocx)%nprocx;
+		int j2 = (j+nprocy)%nprocy;
+		int k2 = (k+nprocz)%nprocz;
+		return i2 + j2*nprocx + k2*nprocx*nprocy;
+	}
+};
+
+
+
+// Inline function to read line without a return argument
+static inline void fgetl( char * str, int num, FILE * stream )
+{
+    char* ptr = fgets( str, num, stream );
+    if ( 0 ) {char *temp = (char *)&ptr; temp++;}
+}
+
+
+
+inline double SSO(DoubleArray &Distance, char *ID, Domain &Dm, int timesteps){
+    /*
+     * This routine converts the data in the Distance array to a signed distance
+     * by solving the equation df/dt = sign(1-|grad f|), where Distance provides
+     * the values of f on the mesh associated with domain Dm
+     * It has been tested with segmented data initialized to values [-1,1]
+     * and will converge toward the signed distance to the surface bounding the associated phases
+     */
+
+    int Q=26;
+    int q,i,j,k;
+    double dt=0.1;
+    int in,jn,kn;
+    double Dqx,Dqy,Dqz,Dx,Dy,Dz,W;
+    double nx,ny,nz,Cqx,Cqy,Cqz,sign,norm;
+    double TotalVariation=0.0;
+
+    const static int D3Q27[26][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
+        {1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
+        {0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1},{1,1,1},{-1,-1,-1},{1,1,-1},{-1,-1,1},
+        {-1,1,-1},{1,-1,1},{1,-1,-1},{-1,1,1}};
+
+    double weights[26];
+    // Compute the weights from the finite differences
+    for (q=0; q<Q; q++){
+        weights[q] = sqrt(1.0*(D3Q27[q][0]*D3Q27[q][0]) + 1.0*(D3Q27[q][1]*D3Q27[q][1]) + 1.0*(D3Q27[q][2]*D3Q27[q][2]));
+    }
+
+    int xdim,ydim,zdim;
+    xdim=Dm.Nx-2;
+    ydim=Dm.Ny-2;
+    zdim=Dm.Nz-2;
+    fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
+
+    int count = 0;
+    while (count < timesteps){
+
+        // Communicate the halo of values
+        fillData.fill(Distance);
+
+        TotalVariation=0.0;
+        // Execute the next timestep
+        for (k=1;k<Dm.Nz-1;k++){
+            for (j=1;j<Dm.Ny-1;j++){
+                for (i=1;i<Dm.Nx-1;i++){
+
+                	int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
+			//sign = Distance(i,j,k) / fabs(Distance(i,j,k));
+		        sign = -1;
+		    	if (ID[n] == 1) sign = 1;
+              /*
+                    if (!(i+1<Nx))     nx=0.5*Distance(i,j,k);
+                    else             nx=0.5*Distance(i+1,j,k);;
+                    if (!(j+1<Ny))     ny=0.5*Distance(i,j,k);
+                    else             ny=0.5*Distance(i,j+1,k);
+                    if (!(k+1<Nz))     nz=0.5*Distance(i,j,k);
+                    else             nz=0.5*Distance(i,j,k+1);
+                    if (i<1)          nx-=0.5*Distance(i,j,k);
+                    else             nx-=0.5*Distance(i-1,j,k);
+                    if (j<1)         ny-=0.5*Distance(i,j,k);
+                    else             ny-=0.5*Distance(i,j-1,k);
+                    if (k<1)          nz-=0.5*Distance(i,j,k);
+                    else             nz-=0.5*Distance(i,j,k-1);
+                    */
+
+                    //............Compute the Gradient...................................
+                    nx = 0.5*(Distance(i+1,j,k) - Distance(i-1,j,k));
+                    ny = 0.5*(Distance(i,j+1,k) - Distance(i,j-1,k));
+                    nz = 0.5*(Distance(i,j,k+1) - Distance(i,j,k-1));
+
+                    W = 0.0;    Dx = Dy = Dz = 0.0;
+                    // also ignore places where the gradient is zero since this will not
+                    // result in any local change to Distance
+                    if (nx*nx+ny*ny+nz*nz > 0.0 ){
+                        for (q=0; q<26; q++){
+                            Cqx = 1.0*D3Q27[q][0];
+                            Cqy = 1.0*D3Q27[q][1];
+                            Cqz = 1.0*D3Q27[q][2];
+                            // get the associated neighbor
+                            in = i + D3Q27[q][0];
+                            jn = j + D3Q27[q][1];
+                            kn = k + D3Q27[q][2];
+
+                            // make sure the neighbor is in the domain (periodic BC)
+                            /*                if (in < 0 ) in +=Nx;
+                             *     don't need this in parallel since MPI handles the halos
+                             if (jn < 0 ) jn +=Ny;
+                             if (kn < 0 ) kn +=Nz;
+                             if (!(in < Nx) ) in -=Nx;
+                             if (!(jn < Ny) ) jn -=Ny;
+                             if (!(kn < Nz) ) kn -=Nz;
+                                         // symmetric boundary
+                            if (in < 0 ) in = i;
+                            if (jn < 0 ) jn = j;
+                            if (kn < 0 ) kn = k;
+                            if (!(in < Nx) ) in = i;
+                            if (!(jn < Ny) ) jn = k;
+                            if (!(kn < Nz) ) kn = k;
+                            */
+
+                            // Compute the gradient using upwind finite differences
+                            Dqx = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqx;
+                            Dqy = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqy;
+                            Dqz = weights[q]*(Distance(i,j,k) - Distance(in,jn,kn))*Cqz;
+
+                            // Only include upwind derivatives
+                            if (sign*(nx*Cqx + ny*Cqy + nz*Cqz) < 0.0 ){
+
+                            	Dx += Dqx;
+                            	Dy += Dqy;
+                            	Dz += Dqz;
+                            	W += weights[q];
+                            }
+                        }
+                        // Normalize by the weight to get the approximation to the gradient
+                        if (fabs(W) > 0.0){
+                        	Dx /= W;
+                        	Dy /= W;
+                        	Dz /= W;
+                        }
+                        norm = sqrt(Dx*Dx+Dy*Dy+Dz*Dz);
+                    }
+                    else{
+                        norm = 0.0;
+                    }
+                    Distance(i,j,k) += dt*sign*(1.0 - norm);
+                    TotalVariation +=  dt*sign*(1.0 - norm);
+                    // Disallow any change in phase
+                   // if (Distance(i,j,k)*2.0*(ID[n]-1.0) < 0) Distance(i,j,k) = -Distance(i,j,k);
+                }
+            }
+        }
+        TotalVariation /= (Dm.Nx-2)*(Dm.Ny-2)*(Dm.Nz-2);
+        count++;
+    }
+
+    return TotalVariation;
+}
+
+
+inline void ReadSpherePacking(int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad)
+{
+    // Read in the full sphere pack
+    //...... READ IN THE SPHERES...................................
+    cout << "Reading the packing file..." << endl;
+    FILE *fid = fopen("pack.out","rb");
+    INSIST(fid!=NULL,"Error opening pack.out");
+    //.........Trash the header lines..........
+    char line[100];
+    fgetl(line, 100, fid);
+    fgetl(line, 100, fid);
+    fgetl(line, 100, fid);
+    fgetl(line, 100, fid);
+    fgetl(line, 100, fid);
+    //........read the spheres..................
+    // We will read until a blank like or end-of-file is reached
+    int count = 0;
+    while ( !feof(fid) && fgets(line,100,fid)!=NULL ) {
+        char* line2 = line;
+        List_cx[count] = strtod(line2,&line2);
+        List_cy[count] = strtod(line2,&line2);
+        List_cz[count] = strtod(line2,&line2);
+        List_rad[count] = strtod(line2,&line2);
+        count++;
+    }
+    cout << "Number of spheres extracted is: " << count << endl;
+    INSIST( count==nspheres, "Specified number of spheres is probably incorrect!" );
+    // .............................................................
+}
+
+inline void AssignLocalSolidID(char *ID, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
+                          double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, 
+                          int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
+{
+    // Use sphere lists to determine which nodes are in porespace
+    // Write out binary file for nodes
+    char value;
+    int N = Nx*Ny*Nz;     // Domain size, including the halo
+    double hx,hy,hz;
+    double x,y,z;
+    double cx,cy,cz,r;
+    int imin,imax,jmin,jmax,kmin,kmax;
+    int p,i,j,k,n;
+    //............................................
+    double min_x,min_y,min_z;
+//    double max_x,max_y,max_z;
+    //............................................
+    // Lattice spacing for the entire domain
+    // It should generally be true that hx=hy=hz
+    // Otherwise, you will end up with ellipsoids
+    hx = Lx/(Nx*nprocx-1);
+    hy = Ly/(Ny*nprocy-1);
+    hz = Lz/(Nz*nprocz-1);    
+    //............................................
+    // Get maximum and minimum for this domain
+    // Halo is included !
+    min_x = double(iproc*Nx-1)*hx;
+    min_y = double(jproc*Ny-1)*hy;
+    min_z = double(kproc*Nz-1)*hz;
+//    max_x = ((iproc+1)*Nx+1)*hx;
+//    max_y = ((jproc+1)*Ny+1)*hy;
+//    max_z = ((kproc+1)*Nz+1)*hz;
+    //............................................
+
+    //............................................
+        // Pre-initialize local ID 
+    for (n=0;n<N;n++){
+        ID[n]=1;
+    }
+    //............................................
+
+    //............................................
+    // .........Loop over the spheres.............
+    for (p=0;p<nspheres;p++){
+        // Get the sphere from the list, map to local min
+        cx = List_cx[p] - min_x;
+        cy = List_cy[p] - min_y;
+        cz = List_cz[p] - min_z;
+        r = List_rad[p];
+        // Check if
+        // Range for this sphere in global indexing
+        imin = int ((cx-r)/hx)-1;
+        imax = int ((cx+r)/hx)+1;
+        jmin = int ((cy-r)/hy)-1;
+        jmax = int ((cy+r)/hy)+1;
+        kmin = int ((cz-r)/hz)-1;
+        kmax = int ((cz+r)/hz)+1;
+        // Obviously we have to do something at the edges
+        if (imin<0)        imin = 0;
+        if (imin>Nx)    imin = Nx;
+        if (imax<0)        imax = 0;
+        if (imax>Nx)    imax = Nx;
+        if (jmin<0)        jmin = 0;
+        if (jmin>Ny)    jmin = Ny;
+        if (jmax<0)        jmax = 0;
+        if (jmax>Ny)    jmax = Ny;
+        if (kmin<0)        kmin = 0;
+        if (kmin>Nz)    kmin = Nz;
+        if (kmax<0)        kmax = 0;
+        if (kmax>Nz)    kmax = Nz;
+        // Loop over the domain for this sphere (may be null)
+        for (i=imin;i<imax;i++){
+            for (j=jmin;j<jmax;j++){
+                for (k=kmin;k<kmax;k++){
+                    // Initialize ID value to 'fluid (=1)'
+                    x = i*hx;
+                    y = j*hy;
+                    z = k*hz;
+                    value = 1;
+                    // if inside sphere, set to zero
+                    if ( (cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z) < r*r){
+                        value=0;
+                    }
+                    // get the position in the list
+                    n = k*Nx*Ny+j*Nx+i;
+                    if ( ID[n] != 0 ){
+                        ID[n] = value;
+                    }
+                }
+            }
+        }
+    }
+}
+
+inline void SignedDistance(double *Distance, int nspheres, double *List_cx, double *List_cy, double *List_cz, double *List_rad,
+                          double Lx, double Ly, double Lz, int Nx, int Ny, int Nz, 
+                          int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz)
+{
+    // Use sphere lists to determine which nodes are in porespace
+    // Write out binary file for nodes
+    int N = Nx*Ny*Nz;     // Domain size, including the halo
+    double hx,hy,hz;
+    double x,y,z;
+    double cx,cy,cz,r;
+    int imin,imax,jmin,jmax,kmin,kmax;
+    int p,i,j,k,n;
+    //............................................
+    double min_x,min_y,min_z;
+    double distance;
+    //............................................
+    // Lattice spacing for the entire domain
+    // It should generally be true that hx=hy=hz
+    // Otherwise, you will end up with ellipsoids
+    hx = Lx/((Nx-2)*nprocx-1);
+    hy = Ly/((Ny-2)*nprocy-1);
+    hz = Lz/((Nz-2)*nprocz-1);    
+    //............................................
+    // Get maximum and minimum for this domain
+    // Halo is included !
+    min_x = double(iproc*(Nx-2)-1)*hx;
+    min_y = double(jproc*(Ny-2)-1)*hy;
+    min_z = double(kproc*(Nz-2)-1)*hz;
+    //............................................
+
+    //............................................
+        // Pre-initialize Distance 
+    for (n=0;n<N;n++){
+        Distance[n]=100.0;
+    }
+    //............................................
+
+    //............................................
+    // .........Loop over the spheres.............
+    for (p=0;p<nspheres;p++){
+        // Get the sphere from the list, map to local min
+        cx = List_cx[p] - min_x;
+        cy = List_cy[p] - min_y;
+        cz = List_cz[p] - min_z;
+        r = List_rad[p];
+        // Check if
+        // Range for this sphere in global indexing
+        imin = int ((cx-2*r)/hx);
+        imax = int ((cx+2*r)/hx)+2;
+        jmin = int ((cy-2*r)/hy);
+        jmax = int ((cy+2*r)/hy)+2;
+        kmin = int ((cz-2*r)/hz);
+        kmax = int ((cz+2*r)/hz)+2;
+        // Obviously we have to do something at the edges
+        if (imin<0)        imin = 0;
+        if (imin>Nx)    imin = Nx;
+        if (imax<0)        imax = 0;
+        if (imax>Nx)    imax = Nx;
+        if (jmin<0)        jmin = 0;
+        if (jmin>Ny)    jmin = Ny;
+        if (jmax<0)        jmax = 0;
+        if (jmax>Ny)    jmax = Ny;
+        if (kmin<0)        kmin = 0;
+        if (kmin>Nz)    kmin = Nz;
+        if (kmax<0)        kmax = 0;
+        if (kmax>Nz)    kmax = Nz;
+        // Loop over the domain for this sphere (may be null)
+        for (i=imin;i<imax;i++){
+            for (j=jmin;j<jmax;j++){
+                for (k=kmin;k<kmax;k++){
+                    // x,y,z is distance in physical units
+                    x = i*hx;
+                    y = j*hy;
+                    z = k*hz;
+                    // if inside sphere, set to zero
+                    // get the position in the list
+                    n = k*Nx*Ny+j*Nx+i;            
+                    // Compute the distance
+                    distance = sqrt((cx-x)*(cx-x)+(cy-y)*(cy-y)+(cz-z)*(cz-z)) - r;
+                    // Assign the minimum distance
+                    if (distance < Distance[n])        Distance[n] = distance;
+                
+                }
+            }
+        }
+    }
+    
+    // Map the distance to lattice units
+    for (n=0; n<N; n++)    Distance[n] = Distance[n]/hx;
+}
+
+inline void WriteLocalSolidID(char *FILENAME, char *ID, int N)
+{
+    char value;
+    ofstream File(FILENAME,ios::binary);
+    for (int n=0; n<N; n++){
+        value = ID[n];
+        File.write((char*) &value, sizeof(value));
+    }
+    File.close();
+}
+
+inline void WriteLocalSolidDistance(char *FILENAME, double *Distance, int N)
+{
+    double value;
+    ofstream File(FILENAME,ios::binary);
+    for (int n=0; n<N; n++){
+        value = Distance[n];
+        File.write((char*) &value, sizeof(value));
+    }
+    File.close();
+}
+
+
+inline void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, int Np)
+{
+    int q,n;
+    double value;
+    ofstream File(FILENAME,ios::binary);
+    for (n=0; n<Np; n++){
+        // Write the two density values
+        value = cDen[n];
+        File.write((char*) &value, sizeof(value));
+        value = cDen[Np+n];
+        File.write((char*) &value, sizeof(value));
+        // Write the even distributions
+        for (q=0; q<19; q++){
+            value = cfq[q*Np+n];
+            File.write((char*) &value, sizeof(value));
+        }
+    }
+    File.close();
+
+}
+
+inline void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, int Np)
+{
+    int q=0, n=0;
+    double value=0;
+    ifstream File(FILENAME,ios::binary);
+    for (n=0; n<Np; n++){
+        // Write the two density values
+        File.read((char*) &value, sizeof(value));
+        cDen[n] = value;
+        File.read((char*) &value, sizeof(value));
+        cDen[Np+n] = value;
+        // Read the even distributions
+        for (q=0; q<19; q++){
+            File.read((char*) &value, sizeof(value));
+            cfq[q*Np+n] = value;
+        }
+    }
+    File.close();
+}
+
+inline void ReadBinaryFile(char *FILENAME, double *Data, int N)
+{
+    int n;
+    double value;
+    ifstream File(FILENAME,ios::binary);
+    if (File.good()){
+        for (n=0; n<N; n++){
+            // Write the two density values
+            File.read((char*) &value, sizeof(value));
+            Data[n] = value;
+
+        }
+    }
+    else {
+        for (n=0; n<N; n++) Data[n] = 1.2e-34;
+    }
+    File.close();
+
+}
+
+
+#endif

common/FunctionTable.h

@@ -0,0 +1,81 @@
+#ifndef included_FunctionTable
+#define included_FunctionTable
+
+
+#include "common/Array.h"
+
+#include <functional>
+
+
+/*!
+ * Class FunctionTable is a serial function table class that defines
+ *   a series of operations that can be performed on the Array class.
+ *   Users can impliment additional versions of the function table that match
+ *   the interface to change the behavior of the array class.
+ */
+class FunctionTable final
+{
+public:
+    /*!
+     * Initialize the array with random values
+     * @param[in] x         The array to operate on
+     */
+    template<class TYPE, class FUN>
+    static void rand( Array<TYPE, FUN> &x );
+
+    /*!
+     * Perform a reduce operator y = f(x)
+     * @param[in] op        The function operation
+     *                      Note: the operator is a template parameter
+     *                      (compared to a std::function to improve performance)
+     * @param[in] A         The array to operate on
+     * @return              The reduction
+     */
+    template<class TYPE, class FUN, typename LAMBDA>
+    static inline TYPE reduce( LAMBDA &op, const Array<TYPE, FUN> &A );
+
+    /*!
+     * Perform a element-wise operation y = f(x)
+     * @param[in] fun       The function operation
+     *                      Note: the operator is a template parameter
+     *                      (compared to a std::function to improve performance)
+     * @param[in] x         The input array to operate on
+     * @param[out] y        The output array
+     */
+    template<class TYPE, class FUN, typename LAMBDA>
+    static inline void transform( LAMBDA &fun, const Array<TYPE, FUN> &x, Array<TYPE, FUN> &y );
+
+    /*!
+     * Perform a element-wise operation z = f(x,y)
+     * @param[in] fun       The function operation
+     *                      Note: the operator is a template parameter
+     *                      (compared to a std::function to improve performance)
+     * @param[in] x         The first array
+     * @param[in] y         The second array
+     * @param[out] z        The result
+     */
+    template<class TYPE, class FUN, typename LAMBDA>
+    static inline void transform(
+        LAMBDA &fun, const Array<TYPE, FUN> &x, const Array<TYPE, FUN> &y, Array<TYPE, FUN> &z );
+
+    /*!
+     * Multiply two arrays
+     * @param[in] a             The first array
+     * @param[in] b             The second array
+     * @param[out] c            The output array
+     */
+    template<class TYPE, class FUN>
+    static void multiply(
+        const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, Array<TYPE, FUN> &c );
+
+
+private:
+    FunctionTable();
+
+    template<class T>
+    static inline void rand( size_t N, T *x );
+};
+
+#include "common/FunctionTable.hpp"
+
+#endif

common/FunctionTable.hpp

@@ -0,0 +1,116 @@
+#ifndef included_FunctionTable_hpp
+#define included_FunctionTable_hpp
+
+#include "common/FunctionTable.h"
+#include "common/Utilities.h"
+
+#include <algorithm>
+#include <cstring>
+#include <limits>
+#include <random>
+
+
+/********************************************************
+ *  Random number initialization                         *
+ ********************************************************/
+template<class TYPE, class FUN>
+void FunctionTable::rand( Array<TYPE, FUN> &x )
+{
+    FunctionTable::rand<TYPE>( x.length(), x.data() );
+}
+template<>
+inline void FunctionTable::rand<double>( size_t N, double *x )
+{
+    std::random_device rd;
+    std::mt19937 gen( rd() );
+    std::uniform_real_distribution<> dis( 0, 1 );
+    for ( size_t i = 0; i < N; i++ )
+        x[i] = dis( gen );
+}
+template<>
+inline void FunctionTable::rand<float>( size_t N, float *x )
+{
+    std::random_device rd;
+    std::mt19937 gen( rd() );
+    std::uniform_real_distribution<> dis( 0, 1 );
+    for ( size_t i = 0; i < N; i++ )
+        x[i] = dis( gen );
+}
+template<>
+inline void FunctionTable::rand<int>( size_t N, int *x )
+{
+    std::random_device rd;
+    std::mt19937 gen( rd() );
+    std::uniform_int_distribution<> dis;
+    for ( size_t i = 0; i < N; i++ )
+        x[i] = dis( gen );
+}
+
+
+/********************************************************
+ *  Reduction                                            *
+ ********************************************************/
+template<class TYPE, class FUN, typename LAMBDA>
+inline TYPE FunctionTable::reduce( LAMBDA &op, const Array<TYPE, FUN> &A )
+{
+    if ( A.length() == 0 )
+        return TYPE();
+    const TYPE *x  = A.data();
+    TYPE y         = x[0];
+    const size_t N = A.length();
+    for ( size_t i = 1; i < N; i++ )
+        y = op( x[i], y );
+    return y;
+}
+
+
+/********************************************************
+ *  Unary transformation                                 *
+ ********************************************************/
+template<class TYPE, class FUN, typename LAMBDA>
+inline void FunctionTable::transform( LAMBDA &fun, const Array<TYPE, FUN> &x, Array<TYPE, FUN> &y )
+{
+    y.resize( x.size() );
+    const size_t N = x.length();
+    for ( size_t i = 0; i < N; i++ )
+        y( i ) = fun( x( i ) );
+}
+template<class TYPE, class FUN, typename LAMBDA>
+inline void FunctionTable::transform(
+    LAMBDA &fun, const Array<TYPE, FUN> &x, const Array<TYPE, FUN> &y, Array<TYPE, FUN> &z )
+{
+    if ( !x.sizeMatch( y ) )
+        throw std::logic_error( "Sizes of x and y do not match" );
+    z.resize( x.size() );
+    const size_t N = x.length();
+    for ( size_t i = 0; i < N; i++ )
+        z( i ) = fun( x( i ), y( i ) );
+}
+
+
+/********************************************************
+ *  Multiply two arrays                                  *
+ ********************************************************/
+template<class TYPE, class FUN>
+void FunctionTable::multiply(
+    const Array<TYPE, FUN> &a, const Array<TYPE, FUN> &b, Array<TYPE, FUN> &c )
+{
+    if ( a.ndim() <= 2 && b.ndim() <= 2 ) {
+        if ( a.size( 1 ) != b.size( 0 ) )
+            throw std::logic_error( "Inner dimensions must match" );
+        c.resize( a.size( 0 ), b.size( 1 ) );
+        c.fill( 0 );
+        for ( size_t k = 0; k < b.size( 1 ); k++ ) {
+            for ( size_t j = 0; j < a.size( 1 ); j++ ) {
+                for ( size_t i = 0; i < a.size( 0 ); i++ ) {
+                    c( i, k ) += a( i, j ) * b( j, k );
+                }
+            }
+        }
+    } else {
+        throw std::logic_error( "Not finished yet" );
+    }
+}
+
+
+#endif

common/MPI_Helpers.cpp

@@ -0,0 +1,266 @@
+#include "common/MPI_Helpers.h"
+#include "common/Utilities.h"
+
+
+/********************************************************
+* Return the MPI data type                              *
+********************************************************/
+template<> MPI_Datatype getMPItype<char>() {
+    return MPI_CHAR;
+}
+template<> MPI_Datatype getMPItype<unsigned char>() {
+    return MPI_UNSIGNED_CHAR;
+}
+template<> MPI_Datatype getMPItype<int>() {
+    return MPI_INT;
+}
+template<> MPI_Datatype getMPItype<long>() {
+    return MPI_LONG;
+}
+template<> MPI_Datatype getMPItype<unsigned long>() {
+    return MPI_UNSIGNED_LONG;
+}
+template<> MPI_Datatype getMPItype<long long>() {
+    return MPI_LONG_LONG;
+}
+template<> MPI_Datatype getMPItype<float>() {
+    return MPI_FLOAT;
+}
+template<> MPI_Datatype getMPItype<double>() {
+    return MPI_DOUBLE;
+}
+
+
+/********************************************************
+* Concrete implimentations for packing/unpacking        *
+********************************************************/
+// unsigned char
+template<>
+size_t packsize<unsigned char>( const unsigned char& rhs )
+{
+    return sizeof(unsigned char);
+}
+template<>
+void pack<unsigned char>( const unsigned char& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(unsigned char));
+}
+template<>
+void unpack<unsigned char>( unsigned char& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(unsigned char));
+}
+// char
+template<>
+size_t packsize<char>( const char& rhs )
+{
+    return sizeof(char);
+}
+template<>
+void pack<char>( const char& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(char));
+}
+template<>
+void unpack<char>( char& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(char));
+}
+// int
+template<>
+size_t packsize<int>( const int& rhs )
+{
+    return sizeof(int);
+}
+template<>
+void pack<int>( const int& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(int));
+}
+template<>
+void unpack<int>( int& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(int));
+}
+// unsigned int
+template<>
+size_t packsize<unsigned int>( const unsigned int& rhs )
+{
+    return sizeof(unsigned int);
+}
+template<>
+void pack<unsigned int>( const unsigned int& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(int));
+}
+template<>
+void unpack<unsigned int>( unsigned int& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(int));
+}
+// size_t
+template<>
+size_t packsize<size_t>( const size_t& rhs )
+{
+    return sizeof(size_t);
+}
+template<>
+void pack<size_t>( const size_t& rhs, char *buffer )
+{
+    memcpy(buffer,&rhs,sizeof(size_t));
+}
+template<>
+void unpack<size_t>( size_t& data, const char *buffer )
+{
+    memcpy(&data,buffer,sizeof(size_t));
+}
+// std::string
+template<>
+size_t packsize<std::string>( const std::string& rhs )
+{
+    return rhs.size()+1;
+}
+template<>
+void pack<std::string>( const std::string& rhs, char *buffer )
+{
+    memcpy(buffer,rhs.c_str(),rhs.size()+1);
+}
+template<>
+void unpack<std::string>( std::string& data, const char *buffer )
+{
+    data = std::string(buffer);
+}
+
+
+/********************************************************
+* Fake MPI routines                                     *
+********************************************************/
+#ifndef USE_MPI
+int MPI_Init(int*,char***)
+{
+    return 0;
+}
+int MPI_Init_thread(int*,char***, int required, int *provided )
+{
+    *provided = required;
+    return 0;
+}
+int MPI_Finalize()
+{
+    return 0;
+}
+int MPI_Comm_size( MPI_Comm, int *size )
+{
+    *size = 1;
+    return 0;
+}
+int MPI_Comm_rank( MPI_Comm, int *rank )
+{
+    *rank = 0;
+    return 0;
+}
+int MPI_Barrier( MPI_Comm )
+{
+    return 0;
+}
+int MPI_Waitall( int, MPI_Request[], MPI_Status[] )
+{
+    return 0;
+}
+int MPI_Wait( MPI_Request*, MPI_Status* )
+{
+    return 0;
+}
+int MPI_Bcast( void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm )
+{
+    return 0;
+}
+int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
+         MPI_Comm comm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
+         MPI_Comm comm, MPI_Status *status)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
+              MPI_Comm comm, MPI_Request *request)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
+              int tag, MPI_Comm comm, MPI_Request *request)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
+                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                  MPI_Comm comm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                   void *recvbuf, const int *recvcounts, const int *displs,
+                   MPI_Datatype recvtype, MPI_Comm comm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                int dest, int sendtag,
+                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                int source, int recvtag,
+                MPI_Comm comm, MPI_Status *status)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+               MPI_Op op, int root, MPI_Comm comm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Comm_group(MPI_Comm comm, MPI_Group *group)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
+{
+    ERROR("Not implimented yet");
+    return 0;
+}
+int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
+{
+    *newcomm = comm;
+    return 0;
+}
+double MPI_Wtime( void )
+{
+    return 0.0;
+}
+int MPI_Comm_free(MPI_Comm *group)
+{
+    return 0;
+}
+int MPI_Group_free(MPI_Group *group)
+{
+    return 0;
+}
+#endif
+
+

common/MPI_Helpers.h

@@ -0,0 +1,215 @@
+// This file contains wrappers for MPI routines and functions to pack/unpack data structures
+#ifndef MPI_WRAPPERS_INC
+#define MPI_WRAPPERS_INC
+
+#include <string.h>
+#include <vector>
+#include <set>
+#include <map>
+
+#ifdef USE_MPI
+    // Inlcude MPI
+    #include "mpi.h"
+#else
+    // Create fake MPI types
+    typedef int MPI_Comm;
+    typedef int MPI_Request;
+    typedef int MPI_Status;
+    #define MPI_COMM_WORLD 0
+    #define MPI_COMM_SELF 0
+    #define MPI_COMM_NULL -1
+    #define MPI_GROUP_NULL -2
+    #define MPI_STATUS_IGNORE NULL
+    enum MPI_Datatype { MPI_LOGICAL, MPI_CHAR, MPI_UNSIGNED_CHAR, MPI_INT, 
+        MPI_UNSIGNED, MPI_LONG, MPI_UNSIGNED_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE };
+    enum MPI_Op { MPI_MIN, MPI_MAX, MPI_SUM };
+    typedef int MPI_Group;
+    #define MPI_THREAD_SINGLE 0
+    #define MPI_THREAD_FUNNELED 1
+    #define MPI_THREAD_SERIALIZED 2
+    #define MPI_THREAD_MULTIPLE 3
+    // Fake MPI functions
+	int MPI_Init(int*,char***);
+    int MPI_Init_thread( int *argc, char ***argv, int required, int *provided );
+	int MPI_Finalize();
+    int MPI_Comm_size( MPI_Comm, int *size );
+    int MPI_Comm_rank( MPI_Comm, int *rank );
+    int MPI_Barrier(MPI_Comm);
+    int MPI_Wait(MPI_Request*,MPI_Status*);
+    int MPI_Waitall(int,MPI_Request[],MPI_Status[]);
+    int MPI_Bcast(void*,int,MPI_Datatype,int,MPI_Comm);
+    int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
+             MPI_Comm comm);
+    int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
+             MPI_Comm comm, MPI_Status *status);
+    int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
+              MPI_Comm comm, MPI_Request *request);
+    int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
+              int tag, MPI_Comm comm, MPI_Request *request);
+    int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
+                  MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
+    int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                  void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                  MPI_Comm comm);
+    int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                  void *recvbuf, const int *recvcounts, const int *displs,
+                  MPI_Datatype recvtype, MPI_Comm comm);
+    int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
+                int dest, int sendtag,
+                void *recvbuf, int recvcount, MPI_Datatype recvtype,
+                int source, int recvtag,
+                MPI_Comm comm, MPI_Status *status);
+    int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
+               MPI_Op op, int root, MPI_Comm comm);
+    double MPI_Wtime( void );
+    int MPI_Comm_group(MPI_Comm comm, MPI_Group *group);
+    int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
+    int MPI_Comm_free(MPI_Comm *group);
+    int MPI_Group_free(MPI_Group *group);
+    int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
+#endif
+
+
+//! Get the size of the MPI_Comm
+//  Note: this is a thread and interrupt safe function
+inline int comm_size( MPI_Comm comm ) {
+    int size = 1;
+    MPI_Comm_size( comm, &size );
+    return size;
+}
+    
+
+//! Get the rank of the MPI_Comm
+//  Note: this is a thread and interrupt safe function
+inline int comm_rank( MPI_Comm comm ) {
+    int rank = 1;
+    MPI_Comm_rank( comm, &rank );
+    return rank;
+}
+    
+
+//! Get the size of MPI_COMM_WORLD
+inline int MPI_WORLD_SIZE( ) {
+    return comm_size( MPI_COMM_WORLD );
+}
+
+//! Get the size of MPI_COMM_WORLD
+inline int MPI_WORLD_RANK( ) {
+    return comm_rank( MPI_COMM_WORLD );
+}
+
+//! Return the appropriate MPI datatype for a class
+template<class TYPE>
+MPI_Datatype getMPItype();
+
+
+//! Template function to return the buffer size required to pack a class
+template<class TYPE>
+size_t packsize( const TYPE& rhs );
+
+//! Template function to pack a class to a buffer
+template<class TYPE>
+void pack( const TYPE& rhs, char *buffer );
+
+//! Template function to unpack a class from a buffer
+template<class TYPE>
+void unpack( TYPE& data, const char *buffer );
+
+
+//! Template function to return the buffer size required to pack a std::vector
+template<class TYPE>
+size_t packsize( const std::vector<TYPE>& rhs );
+
+//! Template function to pack a class to a buffer
+template<class TYPE>
+void pack( const std::vector<TYPE>& rhs, char *buffer );
+
+//! Template function to pack a class to a buffer
+template<class TYPE>
+void unpack( std::vector<TYPE>& data, const char *buffer );
+
+
+//! Template function to return the buffer size required to pack a std::pair
+template<class TYPE1, class TYPE2>
+size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
+
+//! Template function to pack a class to a buffer
+template<class TYPE1, class TYPE2>
+void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
+
+//! Template function to pack a class to a buffer
+template<class TYPE1, class TYPE2>
+void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
+
+
+//! Template function to return the buffer size required to pack a std::map
+template<class TYPE1, class TYPE2>
+size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
+
+//! Template function to pack a class to a buffer
+template<class TYPE1, class TYPE2>
+void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
+
+//! Template function to pack a class to a buffer
+template<class TYPE1, class TYPE2>
+void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
+
+
+//! Template function to return the buffer size required to pack a std::set
+template<class TYPE>
+size_t packsize( const std::set<TYPE>& rhs );
+
+//! Template function to pack a class to a buffer
+template<class TYPE>
+void pack( const std::set<TYPE>& rhs, char *buffer );
+
+//! Template function to pack a class to a buffer
+template<class TYPE>
+void unpack( std::set<TYPE>& data, const char *buffer );
+
+
+
+// Helper functions
+inline float sumReduce( MPI_Comm comm, float x )
+{
+    float y = 0;
+	MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_SUM,comm);
+    return y;
+}
+inline int sumReduce( MPI_Comm comm, int x )
+{
+    int y = 0;
+	MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm);
+    return y;
+}
+inline bool sumReduce( MPI_Comm comm, bool x )
+{
+    int y = sumReduce( comm, x?1:0 );
+    return y>0;
+}
+inline std::vector<float> sumReduce( MPI_Comm comm, const std::vector<float>& x )
+{
+    auto y = x;
+	MPI_Allreduce(x.data(),y.data(),x.size(),MPI_FLOAT,MPI_SUM,comm);
+    return y;
+}
+inline std::vector<int> sumReduce( MPI_Comm comm, const std::vector<int>& x )
+{
+    auto y = x;
+	MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm);
+    return y;
+}
+inline float maxReduce( MPI_Comm comm, float x )
+{
+    float y = 0;
+	MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm);
+    return y;
+}
+
+
+#endif
+
+
+#include "common/MPI_Helpers.hpp"
+
+

common/MPI_Helpers.hpp

@@ -0,0 +1,154 @@
+// This file contains wrappers for MPI routines and functions to pack/unpack data structures
+#ifndef MPI_WRAPPERS_HPP
+#define MPI_WRAPPERS_HPP
+
+#include "common/MPI_Helpers.h"
+#include <string.h>
+#include <vector>
+#include <set>
+#include <map>
+
+
+
+/********************************************************
+* Default instantiations for std::vector                *
+********************************************************/
+template<class TYPE>
+size_t packsize( const std::vector<TYPE>& rhs )
+{
+    size_t bytes = sizeof(size_t);
+    for (size_t i=0; i<rhs.size(); i++)
+        bytes += packsize(rhs[i]);
+    return bytes;
+}
+template<class TYPE>
+void pack( const std::vector<TYPE>& rhs, char *buffer )
+{
+    size_t size = rhs.size();
+    memcpy(buffer,&size,sizeof(size_t));
+    size_t pos = sizeof(size_t);
+    for (size_t i=0; i<rhs.size(); i++) {
+        pack(rhs[i],&buffer[pos]);
+        pos += packsize(rhs[i]);
+    }
+}
+template<class TYPE>
+void unpack( std::vector<TYPE>& data, const char *buffer )
+{
+    size_t size;
+    memcpy(&size,buffer,sizeof(size_t));
+    data.clear();
+    data.resize(size);
+    size_t pos = sizeof(size_t);
+    for (size_t i=0; i<data.size(); i++) {
+        unpack(data[i],&buffer[pos]);
+        pos += packsize(data[i]);
+    }
+}
+
+
+/********************************************************
+* Default instantiations for std::pair                  *
+********************************************************/
+template<class TYPE1, class TYPE2>
+size_t packsize( const std::pair<TYPE1,TYPE2>& rhs )
+{
+    return packsize(rhs.first)+packsize(rhs.second);
+}
+template<class TYPE1, class TYPE2>
+void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer )
+{
+    pack(rhs.first,buffer);
+    pack(rhs.second,&buffer[packsize(rhs.first)]);
+}
+template<class TYPE1, class TYPE2>
+void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer )
+{
+    unpack(data.first,buffer);
+    unpack(data.second,&buffer[packsize(data.first)]);
+}
+
+
+/********************************************************
+* Default instantiations for std::map                   *
+********************************************************/
+template<class TYPE1, class TYPE2>
+size_t packsize( const std::map<TYPE1,TYPE2>& rhs )
+{
+    size_t bytes = sizeof(size_t);
+    typename std::map<TYPE1,TYPE2>::const_iterator it;
+    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+        bytes += packsize(it->first);
+        bytes += packsize(it->second);
+    }
+    return bytes;
+}
+template<class TYPE1, class TYPE2>
+void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer )
+{
+    size_t N = rhs.size();
+    pack(N,buffer);
+    size_t pos = sizeof(size_t);
+    typename std::map<TYPE1,TYPE2>::const_iterator it;
+    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+        pack(it->first,&buffer[pos]);   pos+=packsize(it->first);
+        pack(it->second,&buffer[pos]);  pos+=packsize(it->second);
+    }
+}
+template<class TYPE1, class TYPE2>
+void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer )
+{
+    size_t N = 0;
+    unpack(N,buffer);
+    size_t pos = sizeof(size_t);
+    data.clear();
+    for (size_t i=0; i<N; i++) {
+        std::pair<TYPE1,TYPE2> tmp;
+        unpack(tmp.first,&buffer[pos]);   pos+=packsize(tmp.first);
+        unpack(tmp.second,&buffer[pos]);  pos+=packsize(tmp.second);
+        data.insert(tmp);
+    }
+}
+
+
+/********************************************************
+* Default instantiations for std::set                   *
+********************************************************/
+template<class TYPE>
+size_t packsize( const std::set<TYPE>& rhs )
+{
+    size_t bytes = sizeof(size_t);
+    typename std::set<TYPE>::const_iterator it;
+    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+        bytes += packsize(*it);
+    }
+    return bytes;
+}
+template<class TYPE>
+void pack( const std::set<TYPE>& rhs, char *buffer )
+{
+    size_t N = rhs.size();
+    pack(N,buffer);
+    size_t pos = sizeof(size_t);
+    typename std::set<TYPE>::const_iterator it;
+    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+        pack(*it);   pos+=packsize(*it);
+    }
+}
+template<class TYPE>
+void unpack( std::set<TYPE>& data, const char *buffer )
+{
+    size_t N = 0;
+    unpack(N,buffer);
+    size_t pos = sizeof(size_t);
+    data.clear();
+    for (size_t i=0; i<N; i++) {
+        TYPE tmp;
+        unpack(tmp,&buffer[pos]);   pos+=packsize(tmp);
+        data.insert(tmp);
+    }
+}
+
+
+#endif
+

common/ScaLBL.h

@@ -0,0 +1,369 @@
+/* ScaLBL.h 
+ *  Header file for Scalable Lattice Boltzmann Library
+ *  Separate implementations for GPU and CPU must both follow the conventions defined in this header
+ *  This libarry contains the essential components of the LBM
+ *     - streaming implementations
+ *     - collision terms to model various physics
+ *     - communication framework for the LBM
+ *  Refer to Domain.h for setup of parallel domains
+ */
+#ifndef ScalLBL_H
+#define ScalLBL_H
+#include "common/Domain.h"
+
+extern "C" int ScaLBL_SetDevice(int rank);
+
+extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size);
+
+extern "C" void ScaLBL_FreeDeviceMemory(void* pointer);
+
+extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size);
+
+extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size);
+
+extern "C" void ScaLBL_AllocateZeroCopy(void** address, size_t size);
+
+extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t size);
+
+extern "C" void ScaLBL_DeviceBarrier();
+
+extern "C" void ScaLBL_D3Q19_Pack(int q, int *list, int start, int count, double *sendbuf, double *dist, int N);
+
+extern "C" void ScaLBL_D3Q19_Unpack(int q, int *list, int start, int count, double *recvbuf, double *dist, int N);
+
+extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list,  int start, int count, double *recvbuf, double *dist, int N);
+
+extern "C" void ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N);
+
+extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N);
+
+extern "C" void ScaLBL_Gradient_Unpack(double weight, double Cqx, double Cqy, double Cqz, 
+		int *list, int start, int count, double *recvbuf, double *phi, double *grad, int N);
+
+extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N);
+
+extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N);
+
+extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np);
+
+extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np);
+
+extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np);
+
+// BGK MODEL
+extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
+
+extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz);
+
+// MRT MODEL
+extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
+		double Fy, double Fz);
+
+extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int start, int finish, int Np,
+		double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz);
+
+// COLOR MODEL
+
+extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
+		double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Color(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double *Aq, double *Bq, 
+			double *Den, double *Phi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi, 
+			int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz);
+
+extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
+
+// Density functional hydrodynamics LBM
+extern "C" void ScaLBL_DFH_Init(double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
+		double *Gradient, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_DFH(int *neighborList, double *dist, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *Gradient, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAodd_DFH(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double *ColorGrad, double *Potential, int start, int finish, int Np);
+
+// BOUNDARY CONDITION ROUTINES
+
+//extern "C" void ScaLBL_D3Q19_Pressure_BC_z(double *disteven, double *distodd, double din,
+//		int Nx, int Ny, int Nz);
+//extern "C" void ScaLBL_D3Q19_Pressure_BC_Z(double *disteven, double *distodd, double dout,
+//		int Nx, int Ny, int Nz, int outlet);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *dist, double dout, int count, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_z(int *list, double *dist, double din, int count, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist, double dout, int count, int Np);
+
+extern "C" double ScaLBL_D3Q19_AAodd_Flux_BC_z(int *neighborList, int *list, double *dist, double flux, 
+		double area, int count, int N);
+
+extern "C" double ScaLBL_D3Q19_AAeven_Flux_BC_z(int *list, double *dist, double flux, double area, 
+		 int count, int N);
+
+//extern "C" void ScaLBL_Color_BC(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
+
+extern "C" void ScaLBL_Color_BC_z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
+
+extern "C" void ScaLBL_Color_BC_Z(int *list, int *Map, double *Phi, double *Den, double vA, double vB, int count, int Np);
+
+extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int Nz, int Slice);
+
+// LIST OF DEPRECATED FUNCTIONS (probably delete)
+
+//extern "C" double ScaLBL_D3Q19_Flux_BC_z(double *disteven, double *distodd, double flux,
+//		int Nx, int Ny, int Nz);
+
+//extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, double flux,
+//		int Nx, int Ny, int Nz, int outlet);
+
+//extern "C" void ScaLBL_D3Q19_Velocity_BC_z(double *disteven, double *distodd, double uz,
+//		int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_Velocity_BC_Z(double *disteven, double *distodd, double uz,
+//		int Nx, int Ny, int Nz, int outlet);
+
+//extern "C" void ScaLBL_Color_BC_z(double *Phi, double *Den, double *A_even, double *A_odd,
+//		double *B_even, double *B_odd, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_Color_BC_Z(double *Phi, double *Den, double *A_even, double *A_odd,
+//		double *B_even, double *B_odd, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_AAeven_Compact(char *ID,  double *d_dist, int Np);
+
+//extern "C" void ScaLBL_D3Q19_AAodd_Compact(char *ID,int *d_neighborList,  double *d_dist, int Np);
+
+//extern "C" void ScaLBL_ComputePhaseField(char *ID, double *Phi, double *Den, int N);
+
+//extern "C" void ScaLBL_D3Q7_Init(double *Dist, double *Den, int Np);
+
+//extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den,
+//		int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_AA_Init(double *f_even, double *f_odd, int Np);
+
+//extern "C" void ScaLBL_D3Q19_Init(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_Swap_Compact(int *neighborList, double *disteven, double *distodd, int Np);
+
+
+//extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB,
+//		double Fx, double Fy, double Fz,int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_Color_Init(char *ID, double *Den, double *Phi, double das, double dbs, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_ColorDistance_Init(char *ID, double *Den, double *Phi, double *Distance,
+//		double das, double dbs, double beta, double xp, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_ColorGradient(char *ID, double *phi, double *ColorGrad, int Nx, int Ny, int Nz);
+
+//extern "C" void ScaLBL_D3Q19_ColorCollide( char *ID, double *disteven, double *distodd, double *phi, double *ColorGrad,
+//		double *Velocity, int Nx, int Ny, int Nz,double rlx_setA, double rlx_setB,
+//		double alpha, double beta, double Fx, double Fy, double Fz);
+
+//extern "C" void ScaLBL_D3Q7_ColorCollideMass(char *ID, double *A_even, double *A_odd, double *B_even, double *B_odd, 
+//		double *Den, double *Phi, double *ColorGrad, double *Velocity, double beta, int N, bool pBC);
+
+
+//extern "C" void ScaLBL_D3Q19_AAeven_Color(double *dist, double *Aq, double *Bq, double *Den, double *Vel,
+//		double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int start, int finish, int Np);
+
+//extern "C" void ScaLBL_D3Q19_AAodd_Color(int *d_neighborList, double *dist, double *Aq, double *Bq, double *Den, double *Vel,
+//		double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+//		double Fx, double Fy, double Fz, int start, int finish, int Np);
+
+/*extern "C" void ScaLBL_D3Q19_AAeven_ColorMomentum(double *dist, double *Den, double *Vel,
+		double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_ColorMomentum(int *d_neighborList, double *dist, double *Den, double *Vel,
+		double *ColorGrad, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
+		double Fx, double Fy, double Fz, int start, int finish, int Np);
+		
+extern "C" void ScaLBL_D3Q19_AAeven_ColorMass(double *Aq, double *Bq, double *Den, double *Vel,
+		double *ColorGrad, double beta, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_ColorMass(int *d_neighborList, double *Aq, double *Bq, double *Den, double *Vel,
+		double *ColorGrad, double beta, int start, int finish, int Np);
+*/
+/*
+class ScaLBL_Color{
+public:
+	ScaLBL_Color(Domain &Dm);
+	~ScaLBL_Color();
+
+	int *NeighborList;
+	int *dvcMap;
+	//		double *f_even,*f_odd;
+	double *fq, *Aq, *Bq;
+	double *Den, *Phi;
+	double *ColorGrad;
+	double *Vel;
+	double *Pressure;
+	ScaLBL_Communicator ScaLBL_Comm;
+	//Create a second communicator based on the regular data layout
+	ScaLBL_Communicator ScaLBL_Comm_Regular;
+
+	void Initialize(Domain &Dm);
+	void Run(int &timestep);
+	
+private:
+
+};
+
+void ScaLBL_Color::ScaLBL_Color(Domain &Dm){
+	
+}
+
+void ScaLBL_Color::Initialize(Domain &Dm){
+	
+}
+
+void ScaLBL_Color::Run(int &timestep){
+	
+}
+*/
+
+class ScaLBL_Communicator{
+public:
+	//......................................................................................
+	ScaLBL_Communicator(Domain &Dm);
+
+	//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
+	~ScaLBL_Communicator();
+	//......................................................................................
+	unsigned long int CommunicationCount,SendCount,RecvCount;
+	int Nx,Ny,Nz,N;
+	int next;
+	int first_interior,last_interior;
+	int BoundaryCondition;
+	//......................................................................................
+	//  Set up for D319 distributions
+	// 		- determines how much memory is allocated
+	//		- buffers are reused to send D3Q7 distributions and halo exchange as needed
+	//......................................................................................
+	// Buffers to store data sent and recieved by this MPI process
+	double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
+	double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
+	double *sendbuf_xY, *sendbuf_yZ, *sendbuf_Xz, *sendbuf_XY, *sendbuf_YZ, *sendbuf_XZ;
+	double *recvbuf_x, *recvbuf_y, *recvbuf_z, *recvbuf_X, *recvbuf_Y, *recvbuf_Z;
+	double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ;
+	double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
+	//......................................................................................
+
+	int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, char *id, int Np);
+	void MemoryOptimizedLayout(IntArray &Map, int *neighborList, char *id, int Np);
+	void MemoryOptimizedLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
+	void MemoryDenseLayout(IntArray &Map, int *neighborList, char *id, int Np);
+	void MemoryDenseLayoutFull(IntArray &Map, int *neighborList, char *id, int Np);
+	void SendD3Q19(double *f_even, double *f_odd);
+	void RecvD3Q19(double *f_even, double *f_odd);
+	void SendD3Q19AA(double *f_even, double *f_odd);
+	void RecvD3Q19AA(double *f_even, double *f_odd);
+	void SendD3Q19AA(double *dist);
+	void RecvD3Q19AA(double *dist);
+	void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
+	void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
+	void BiSendD3Q7AA(double *Aq, double *Bq);
+	void BiRecvD3Q7AA(double *Aq, double *Bq);
+	void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
+	void TriRecvD3Q7AA(double *Aq, double *Bq, double *Cq);
+	void SendHalo(double *data);
+	void RecvHalo(double *data);
+	void RecvGrad(double *Phi, double *Gradient);
+	void RegularLayout(IntArray map, double *data, DoubleArray &regdata);
+
+	// Routines to set boundary conditions
+	void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
+	void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB);
+	void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time);
+	void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
+	double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
+
+	void TestSendD3Q19(double *f_even, double *f_odd);
+	void TestRecvD3Q19(double *f_even, double *f_odd);
+
+	// Debugging and unit testing functions
+	void PrintD3Q19();
+
+private:
+	//void D3Q19_MapRecv_OLD(int q, int Cqx, int Cqy, int Cqz, int *list,  int start, int count, int *d3q19_recvlist);
+	void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list,  int start, int count, int *d3q19_recvlist);
+
+	bool Lock; 	// use Lock to make sure only one call at a time to protect data in transit
+	// only one set of Send requests can be active at any time (per instance)
+	int i,j,k,n;
+	int iproc,jproc,kproc;
+	int nprocx,nprocy,nprocz;
+	int sendtag,recvtag;
+	// Give the object it's own MPI communicator
+	RankInfoStruct rank_info;
+	MPI_Group Group;	// Group of processors associated with this domain
+	MPI_Comm MPI_COMM_SCALBL;		// MPI Communicator for this domain
+	MPI_Request req1[18],req2[18];
+	MPI_Status stat1[18],stat2[18];
+	//......................................................................................
+	// MPI ranks for all 18 neighbors
+	//......................................................................................
+	// These variables are all private to prevent external things from modifying them!!
+	//......................................................................................
+	int rank;
+	int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
+	int rank_xy,rank_XY,rank_xY,rank_Xy;
+	int rank_xz,rank_XZ,rank_xZ,rank_Xz;
+	int rank_yz,rank_YZ,rank_yZ,rank_Yz;
+	//......................................................................................
+
+	//......................................................................................
+	int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
+	int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
+	int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
+	//......................................................................................
+	int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
+	int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
+	int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
+	//......................................................................................
+	// Send buffers that reside on the compute device
+	int *dvcSendList_x, *dvcSendList_y, *dvcSendList_z, *dvcSendList_X, *dvcSendList_Y, *dvcSendList_Z;
+	int *dvcSendList_xy, *dvcSendList_yz, *dvcSendList_xz, *dvcSendList_Xy, *dvcSendList_Yz, *dvcSendList_xZ;
+	int *dvcSendList_xY, *dvcSendList_yZ, *dvcSendList_Xz, *dvcSendList_XY, *dvcSendList_YZ, *dvcSendList_XZ;
+	// Recieve buffers that reside on the compute device
+	int *dvcRecvList_x, *dvcRecvList_y, *dvcRecvList_z, *dvcRecvList_X, *dvcRecvList_Y, *dvcRecvList_Z;
+	int *dvcRecvList_xy, *dvcRecvList_yz, *dvcRecvList_xz, *dvcRecvList_Xy, *dvcRecvList_Yz, *dvcRecvList_xZ;
+	int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ;
+	// Recieve buffers for the distributions
+	int *dvcRecvDist_x, *dvcRecvDist_y, *dvcRecvDist_z, *dvcRecvDist_X, *dvcRecvDist_Y, *dvcRecvDist_Z;
+	int *dvcRecvDist_xy, *dvcRecvDist_yz, *dvcRecvDist_xz, *dvcRecvDist_Xy, *dvcRecvDist_Yz, *dvcRecvDist_xZ;
+	int *dvcRecvDist_xY, *dvcRecvDist_yZ, *dvcRecvDist_Xz, *dvcRecvDist_XY, *dvcRecvDist_YZ, *dvcRecvDist_XZ;
+	//......................................................................................
+
+};
+
+
+#endif

common/StackTrace.h

@@ -0,0 +1,242 @@
+#ifndef included_StackTrace
+#define included_StackTrace
+
+#include <functional>
+#include <iostream>
+#include <set>
+#include <thread>
+#include <vector>
+
+
+// Check for and include MPI
+// clang-format off
+#if defined(USE_MPI) || defined(USE_EXT_MPI)
+    #include "mpi.h"
+#elif defined(__has_include)
+    #if __has_include("mpi.h")
+        #include "mpi.h"
+    #else
+        typedef int MPI_Comm;
+    #endif
+#else
+    typedef int MPI_Comm;
+#endif
+// clang-format on
+
+
+namespace StackTrace {
+
+
+struct stack_info {
+    void *address;
+    void *address2;
+    std::string object;
+    std::string function;
+    std::string filename;
+    int line;
+    //! Default constructor
+    stack_info() : address( nullptr ), address2( nullptr ), line( 0 ) {}
+    //! Reset the stack
+    void clear();
+    //! Operator==
+    bool operator==( const stack_info &rhs ) const;
+    //! Operator!=
+    bool operator!=( const stack_info &rhs ) const;
+    //! Get the minimum width to print the addresses
+    int getAddressWidth() const;
+    //! Print the stack info
+    std::string print( int widthAddress = 16, int widthObject = 20, int widthFunction = 32 ) const;
+    //! Compute the number of bytes needed to store the object
+    size_t size() const;
+    //! Pack the data to a byte array, returning a pointer to the end of the data
+    char *pack( char *ptr ) const;
+    //! Unpack the data from a byte array, returning a pointer to the end of the data
+    const char *unpack( const char *ptr );
+    //! Pack a vector of data to a memory block
+    static std::vector<char> packArray( const std::vector<stack_info> &data );
+    //! Unpack a vector of data from a memory block
+    static std::vector<stack_info> unpackArray( const char *data );
+};
+
+
+struct multi_stack_info {
+    int N;                                  // Number of threads/processes
+    stack_info stack;                       // Current stack item
+    std::vector<multi_stack_info> children; // Children
+    //! Default constructor
+    multi_stack_info() : N( 0 ) {}
+    //! Construct from a simple call stack
+    explicit multi_stack_info( const std::vector<stack_info> & );
+    //! Copy constructor from a simple call stack
+    multi_stack_info &operator=( const std::vector<stack_info> & );
+    //! Reset the stack
+    void clear();
+    //! Add the given stack to the multistack
+    void add( size_t len, const stack_info *stack );
+    //! Print the stack info
+    std::vector<std::string> print( const std::string &prefix = std::string() ) const;
+
+private:
+    void print2( const std::string &prefix, int w[3], std::vector<std::string> &text ) const;
+    int getAddressWidth() const;
+    int getObjectWidth() const;
+    int getFunctionWidth() const;
+};
+
+
+/*!
+ * @brief  Get the current call stack
+ * @details  This function returns the current call stack for the current thread
+ * @return      Returns vector containing the stack
+ */
+std::vector<stack_info> getCallStack();
+
+
+/*!
+ * @brief  Get the current call stack for a thread
+ * @details  This function returns the current call stack for the given thread
+ * @param[in] id    The thread id of the stack we want to return
+ * @return          Returns vector containing the stack
+ */
+std::vector<stack_info> getCallStack( std::thread::native_handle_type id );
+
+
+/*!
+ * @brief  Get the current call stack for all threads
+ * @details  This function returns the current call stack for all threads
+ *    in the current process.
+ *    Note: This functionality may not be availible on all platforms
+ * @return          Returns vector containing the stack
+ */
+multi_stack_info getAllCallStacks();
+
+
+/*!
+ * @brief  Get the current call stack for all threads/processes
+ * @details  This function returns the current call stack for all threads
+ *    for all processes in the current process.  This function requires
+ *    the user to call globalCallStackInitialize() before calling this
+ *    routine, and globalCallStackFinalize() before exiting.
+ *    Note: This functionality may not be availible on all platforms
+ * @return          Returns vector containing the stack
+ */
+multi_stack_info getGlobalCallStacks();
+
+
+/*!
+ * @brief  Clean up the stack trace
+ * @details  This function modifies the stack trace to remove entries
+ *    related to acquiring the stack trace in an attempt to make it
+ *    more useful for display/users.
+ * @param[in,out] stack     The stack trace to modify
+ */
+void cleanupStackTrace( multi_stack_info &stack );
+
+
+//! Function to return the current call stack for the current thread
+std::vector<void *> backtrace();
+
+//! Function to return the current call stack for the given thread
+std::vector<void *> backtrace( std::thread::native_handle_type id );
+
+//! Function to return the current call stack for all threads
+std::vector<std::vector<void *>> backtraceAll();
+
+
+//! Function to return the stack info for a given address
+stack_info getStackInfo( void *address );
+
+
+//! Function to return the stack info for a given address
+std::vector<stack_info> getStackInfo( const std::vector<void *> &address );
+
+
+//! Function to return the signal name
+std::string signalName( int signal );
+
+
+/*!
+ * Return the symbols from the current executable (not availible for all platforms)
+ * @return      Returns 0 if sucessful
+ */
+int getSymbols( std::vector<void *> &address,
+                std::vector<char> &type,
+                std::vector<std::string> &obj );
+
+
+/*!
+ * Return the name of the executable
+ * @return      Returns the name of the executable (usually the full path)
+ */
+std::string getExecutable();
+
+
+/*!
+ * Return the search path for the symbols
+ * @return      Returns the search path for the symbols
+ */
+std::string getSymPaths();
+
+
+//!< Terminate type
+enum class terminateType { signal, exception };
+
+/*!
+ * Set the error handlers
+ * @param[in] abort     Function to terminate the program: abort(msg,type)
+ */
+void setErrorHandlers( std::function<void( std::string, terminateType )> abort );
+
+
+/*!
+ * Set the given signals to the handler
+ * @param[in] signals   Signals to handle
+ * @param[in] handler   Function to terminate the program: abort(msg,type)
+ */
+void setSignals( const std::vector<int> &signals, void ( *handler )( int ) );
+
+
+//! Clear a signal set by setSignals
+void clearSignal( int signal );
+
+
+//! Clear all signals set by setSignals
+void clearSignals();
+
+
+//! Return a list of all signals that can be caught
+std::vector<int> allSignalsToCatch();
+
+//! Return a default list of signals to catch
+std::vector<int> defaultSignalsToCatch();
+
+
+//! Get a list of the active threads
+std::set<std::thread::native_handle_type> activeThreads();
+
+//! Get a handle to this thread
+std::thread::native_handle_type thisThread();
+
+
+//! Initialize globalCallStack functionallity
+void globalCallStackInitialize( MPI_Comm comm );
+
+//! Clean up globalCallStack functionallity
+void globalCallStackFinalize();
+
+
+/*!
+ * @brief  Call system command
+ * @details  This function calls a system command, waits for the program
+ *   to execute, captures and returns the output and exit code.
+ * @param[in] cmd           Command to execute
+ * @param[out] exit_code    Exit code returned from child process
+ * @return                  Returns string containing the output
+ */
+std::string exec( const std::string &cmd, int &exit_code );
+
+
+} // namespace StackTrace
+
+
+#endif

common/UnitTest.cpp

@@ -0,0 +1,379 @@
+#include "common/UnitTest.h"
+#include "common/Utilities.h"
+#include <cstring>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+
+#define pout std::cout
+#define printp printf
+
+
+/********************************************************************
+ *  Constructor/Destructor                                           *
+ ********************************************************************/
+UnitTest::UnitTest()
+{
+#ifdef USE_MPI
+    comm = MPI_COMM_WORLD;
+#endif
+}
+UnitTest::~UnitTest() { reset(); }
+void UnitTest::reset()
+{
+    mutex.lock();
+    // Clear the data forcing a reallocation
+    std::vector<std::string>().swap( pass_messages );
+    std::vector<std::string>().swap( fail_messages );
+    std::vector<std::string>().swap( expected_fail_messages );
+    mutex.unlock();
+}
+
+
+/********************************************************************
+ *  Add a pass, fail, expected failure message in a thread-safe way  *
+ ********************************************************************/
+void UnitTest::passes( const std::string &in )
+{
+    mutex.lock();
+    pass_messages.push_back( in );
+    mutex.unlock();
+}
+void UnitTest::failure( const std::string &in )
+{
+    mutex.lock();
+    fail_messages.push_back( in );
+    mutex.unlock();
+}
+void UnitTest::expected_failure( const std::string &in )
+{
+    mutex.lock();
+    expected_fail_messages.push_back( in );
+    mutex.unlock();
+}
+
+
+/********************************************************************
+ *  Print a global report                                            *
+ *  Note: only rank 0 will print, all messages will be aggregated    *
+ ********************************************************************/
+inline std::vector<int> UnitTest::allGather( int value ) const
+{
+    int size = getSize();
+    std::vector<int> data( size, value );
+#ifdef USE_MPI
+    if ( size > 1 )
+        MPI_Allgather( &value, 1, MPI_INT, data.data(), 1, MPI_INT, comm );
+#endif
+    return data;
+}
+inline void UnitTest::barrier() const
+{
+#ifdef USE_MPI
+    if ( getSize() > 1 )
+        MPI_Barrier( comm );
+#endif
+}
+static inline void print_messages( const std::vector<std::vector<std::string>> &messages )
+{
+    if ( messages.size() > 1 ) {
+        for ( size_t i = 0; i < messages.size(); i++ ) {
+            if ( !messages[i].empty() ) {
+                printp( "     Proccessor %i:\n", static_cast<int>( i ) );
+                for ( const auto &j : messages[i] )
+                    pout << "        " << j << std::endl;
+            }
+        }
+    } else {
+        for ( const auto &j : messages[0] )
+            pout << "    " << j << std::endl;
+    }
+}
+void UnitTest::report( const int level0 ) const
+{
+    mutex.lock();
+    int size = getSize();
+    int rank = getRank();
+    // Broadcast the print level from rank 0
+    int level = level0;
+#ifdef USE_MPI
+    if ( getSize() > 1 )
+        MPI_Bcast( &level, 1, MPI_INT, 0, comm );
+#endif
+    if ( level < 0 || level > 2 )
+        ERROR( "Invalid print level" );
+    // Perform a global all gather to get the number of failures per processor
+    auto N_pass             = allGather( pass_messages.size() );
+    auto N_fail             = allGather( fail_messages.size() );
+    auto N_expected_fail    = allGather( expected_fail_messages.size() );
+    int N_pass_tot          = 0;
+    int N_fail_tot          = 0;
+    int N_expected_fail_tot = 0;
+    for ( int i = 0; i < size; i++ ) {
+        N_pass_tot += N_pass[i];
+        N_fail_tot += N_fail[i];
+        N_expected_fail_tot += N_expected_fail[i];
+    }
+    // Send all messages to rank 0 (if needed)
+    std::vector<std::vector<std::string>> pass_messages_rank( size );
+    std::vector<std::vector<std::string>> fail_messages_rank( size );
+    std::vector<std::vector<std::string>> expected_fail_rank( size );
+    // Get the pass messages
+    if ( ( level == 1 && N_pass_tot <= 20 ) || level == 2 )
+        pass_messages_rank = UnitTest::gatherMessages( pass_messages, 1 );
+    // Get the fail messages
+    if ( level == 1 || level == 2 )
+        fail_messages_rank = UnitTest::gatherMessages( fail_messages, 2 );
+    // Get the expected_fail messages
+    if ( ( level == 1 && N_expected_fail_tot <= 50 ) || level == 2 )
+        expected_fail_rank = UnitTest::gatherMessages( expected_fail_messages, 2 );
+    // Print the results of all messages (only rank 0 will print)
+    if ( rank == 0 ) {
+        pout << std::endl;
+        // Print the passed tests
+        pout << "Tests passed" << std::endl;
+        if ( level == 0 || ( level == 1 && N_pass_tot > 20 ) ) {
+            // We want to print a summary
+            if ( size > 8 ) {
+                // Print 1 summary for all processors
+                printp( "     %i tests passed (use report level 2 for more detail)\n", N_pass_tot );
+            } else {
+                // Print a summary for each processor
+                for ( int i = 0; i < size; i++ )
+                    printp( "     %i tests passed (proc %i) (use report level 2 for more detail)\n",
+                        N_pass[i], i );
+            }
+        } else {
+            // We want to print all messages
+            for ( int i = 0; i < size; i++ )
+                ASSERT( (int) pass_messages_rank[i].size() == N_pass[i] );
+            print_messages( pass_messages_rank );
+        }
+        pout << std::endl;
+        // Print the tests that failed
+        pout << "Tests failed" << std::endl;
+        if ( level == 0 ) {
+            // We want to print a summary
+            if ( size > 8 ) {
+                // Print 1 summary for all processors
+                printp( "     %i tests failed (use report level 2 for more detail)\n", N_fail_tot );
+            } else {
+                // Print a summary for each processor
+                for ( int i = 0; i < size; i++ )
+                    printp( "     %i tests failed (proc %i) (use report level 2 for more detail)\n",
+                        N_fail[i], i );
+            }
+        } else {
+            // We want to print all messages
+            for ( int i = 0; i < size; i++ )
+                ASSERT( (int) fail_messages_rank[i].size() == N_fail[i] );
+            print_messages( fail_messages_rank );
+        }
+        pout << std::endl;
+        // Print the tests that expected failed
+        pout << "Tests expected failed" << std::endl;
+        if ( level == 0 || ( level == 1 && N_expected_fail_tot > 50 ) ) {
+            // We want to print a summary
+            if ( size > 8 ) {
+                // Print 1 summary for all processors
+                printp( "     %i tests expected failed (use report level 2 for more detail)\n",
+                    N_expected_fail_tot );
+            } else {
+                // Print a summary for each processor
+                for ( int i = 0; i < size; i++ )
+                    printp( "     %i tests expected failed (proc %i) (use report level 2 for more "
+                            "detail)\n",
+                        N_expected_fail[i], i );
+            }
+        } else {
+            // We want to print all messages
+            for ( int i = 0; i < size; i++ )
+                ASSERT( (int) expected_fail_rank[i].size() == N_expected_fail[i] );
+            print_messages( expected_fail_rank );
+        }
+        pout << std::endl;
+    }
+    // Add a barrier to synchronize all processors (rank 0 is much slower)
+    barrier();
+    Utilities::sleep_ms( 10 ); // Need a brief pause to allow any printing to finish
+    mutex.unlock();
+}
+
+
+/********************************************************************
+ *  Gather the messages to rank 0                                    *
+ ********************************************************************/
+std::vector<std::vector<std::string>> UnitTest::gatherMessages(
+    const std::vector<std::string> &local_messages, int tag ) const
+{
+    const int rank = getRank();
+    const int size = getSize();
+    std::vector<std::vector<std::string>> messages( size );
+    if ( rank == 0 ) {
+        // Rank 0 should receive all messages
+        for ( int i = 0; i < size; i++ ) {
+            if ( i == 0 )
+                messages[i] = local_messages;
+            else
+                messages[i] = unpack_message_stream( i, tag );
+        }
+    } else {
+        // All other ranks send their message (use non-blocking communication)
+        pack_message_stream( local_messages, 0, tag );
+    }
+    return messages;
+}
+
+
+/********************************************************************
+ *  Pack and send the given messages                                 *
+ ********************************************************************/
+void UnitTest::pack_message_stream(
+    const std::vector<std::string> &messages, const int rank, const int tag ) const
+{
+#ifdef USE_MPI
+    // Get the size of the messages
+    auto N_messages  = (int) messages.size();
+    auto *msg_size   = new int[N_messages];
+    int msg_size_tot = 0;
+    for ( int i = 0; i < N_messages; i++ ) {
+        msg_size[i] = (int) messages[i].size();
+        msg_size_tot += msg_size[i];
+    }
+    // Allocate space for the message stream
+    size_t size_data = ( N_messages + 1 ) * sizeof( int ) + msg_size_tot;
+    auto *data       = new char[size_data];
+    // Pack the message stream
+    memcpy( data, &N_messages, sizeof( int ) );
+    memcpy( &data[sizeof( int )], msg_size, N_messages * sizeof( int ) );
+    size_t k = ( N_messages + 1 ) * sizeof( int );
+    for ( int i = 0; i < N_messages; i++ ) {
+        messages[i].copy( &data[k], msg_size[i] );
+        k += msg_size[i];
+    }
+    // Send the message stream (using a non-blocking send)
+    MPI_Request request;
+    MPI_Isend( data, size_data, MPI_CHAR, rank, tag, comm, &request );
+    // Wait for the communication to send and free the temporary memory
+    MPI_Status status;
+    MPI_Wait( &request, &status );
+    delete[] data;
+    delete[] msg_size;
+#else
+    NULL_USE( messages );
+    NULL_USE( rank );
+    NULL_USE( tag );
+#endif
+}
+
+
+/********************************************************************
+ *  Receive and unpack a message stream                              *
+ ********************************************************************/
+std::vector<std::string> UnitTest::unpack_message_stream( const int rank, const int tag ) const
+{
+#ifdef USE_MPI
+    // Probe the message to get the message size
+    MPI_Status status;
+    MPI_Probe( rank, tag, comm, &status );
+    int size_data = -1;
+    MPI_Get_count( &status, MPI_BYTE, &size_data );
+    ASSERT( size_data >= 0 );
+    // Allocate memory to receive the data
+    auto *data = new char[size_data];
+    // receive the data (using a non-blocking receive)
+    MPI_Request request;
+    MPI_Irecv( data, size_data, MPI_CHAR, rank, tag, comm, &request );
+    // Wait for the communication to be received
+    MPI_Wait( &request, &status );
+    // Unpack the message stream
+    int N_messages = 0;
+    memcpy( &N_messages, data, sizeof( int ) );
+    if ( N_messages == 0 ) {
+        delete[] data;
+        return std::vector<std::string>();
+    }
+    std::vector<int> msg_size( N_messages );
+    std::vector<std::string> messages( N_messages );
+    memcpy( msg_size.data(), &data[sizeof( int )], N_messages * sizeof( int ) );
+    int k = ( N_messages + 1 ) * sizeof( int );
+    for ( int i = 0; i < N_messages; i++ ) {
+        messages[i] = std::string( &data[k], msg_size[i] );
+        k += msg_size[i];
+    }
+    delete[] data;
+    return messages;
+#else
+    NULL_USE( rank );
+    NULL_USE( tag );
+    return std::vector<std::string>();
+#endif
+}
+
+
+/********************************************************************
+ *  Other functions                                                  *
+ ********************************************************************/
+int UnitTest::getRank() const
+{
+    int rank = 0;
+#ifdef USE_MPI
+    int flag = 0;
+    MPI_Initialized( &flag );
+    if ( flag )
+        MPI_Comm_rank( comm, &rank );
+#endif
+    return rank;
+}
+int UnitTest::getSize() const
+{
+    int size = 1;
+#ifdef USE_MPI
+    int flag = 0;
+    MPI_Initialized( &flag );
+    if ( flag )
+        MPI_Comm_size( comm, &size );
+#endif
+    return size;
+}
+size_t UnitTest::NumPassGlobal() const
+{
+    size_t num = pass_messages.size();
+#ifdef USE_MPI
+    if ( getSize() > 1 ) {
+        auto send = static_cast<int>( num );
+        int sum   = 0;
+        MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
+        num = static_cast<size_t>( sum );
+    }
+#endif
+    return num;
+}
+size_t UnitTest::NumFailGlobal() const
+{
+    size_t num = fail_messages.size();
+#ifdef USE_MPI
+    if ( getSize() > 1 ) {
+        auto send = static_cast<int>( num );
+        int sum   = 0;
+        MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
+        num = static_cast<size_t>( sum );
+    }
+#endif
+    return num;
+}
+size_t UnitTest::NumExpectedFailGlobal() const
+{
+    size_t num = expected_fail_messages.size();
+#ifdef USE_MPI
+    if ( getSize() > 1 ) {
+        auto send = static_cast<int>( num );
+        int sum   = 0;
+        MPI_Allreduce( &send, &sum, 1, MPI_INT, MPI_SUM, comm );
+        num = static_cast<size_t>( sum );
+    }
+#endif
+    return num;
+}

common/UnitTest.h

@@ -0,0 +1,120 @@
+#ifndef included_UnitTest
+#define included_UnitTest
+
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <vector>
+#ifdef USE_MPI
+#include "mpi.h"
+#endif
+
+
+/*!
+ * @brief Class UnitTest is simple utility for running unit tests.
+ * It provides basic routines for tracing success or failure of tests,
+ * and reporting the results.
+ * \par Code Sample:
+ * \code
+    try {
+        std::cout << "Testing tstOne" << std::endl;
+        tstOne(&ut);
+        ut.passes("Test XXX passed");
+    } catch( ... ) {
+        ut.failure("An unknown exception was thrown");
+    }
+    ut.report();
+    return ut.NumFail();
+ * \endcode
+
+ */
+class UnitTest
+{
+public:
+    //! Constructor
+    UnitTest();
+
+    //! Destructor
+    virtual ~UnitTest();
+
+    //! Indicate a passed test (thread-safe)
+    virtual void passes( const std::string &in );
+
+    //! Indicate a failed test (thread-safe)
+    virtual void failure( const std::string &in );
+
+    //! Indicate an expected failed test (thread-safe)
+    virtual void expected_failure( const std::string &in );
+
+    //! Return the number of passed tests locally
+    virtual size_t NumPassLocal() const { return pass_messages.size(); }
+
+    //! Return the number of failed tests locally
+    virtual size_t NumFailLocal() const { return fail_messages.size(); }
+
+    //! Return the number of expected failed tests locally
+    virtual size_t NumExpectedFailLocal() const { return expected_fail_messages.size(); }
+
+    //! Return the number of passed tests locally
+    virtual size_t NumPassGlobal() const;
+
+    //! Return the number of failed tests locally
+    virtual size_t NumFailGlobal() const;
+
+    //! Return the number of expected failed tests locally
+    virtual size_t NumExpectedFailGlobal() const;
+
+    //! Return the rank of the current processor
+    int getRank() const;
+
+    //! Return the number of processors
+    int getSize() const;
+
+    /*!
+     * Print a report of the passed and failed tests.
+     * Note: This is a blocking call that all processors must execute together.
+     * Note: Only rank 0 will print the messages (this is necessary as other ranks may not be able
+     * to print correctly).
+     * @param level     Optional integer specifying the level of reporting (default: 1)
+     *                  0: Report the number of tests passed, failed, and expected failures.
+     *                  1: Report the number of passed tests (if <=20) or the number passed
+     *                     otherwise, report all failures, report the number of expected
+     *                     failed tests (if <=50) or the number passed otherwise.
+     *                  2: Report all passed, failed, and expected failed tests.
+     */
+    virtual void report( const int level = 1 ) const;
+
+    //! Clear the messages
+    void reset();
+
+protected:
+    std::vector<std::string> pass_messages;
+    std::vector<std::string> fail_messages;
+    std::vector<std::string> expected_fail_messages;
+    mutable std::mutex mutex;
+#ifdef USE_MPI
+    MPI_Comm comm;
+#endif
+
+private:
+    // Make the copy constructor private
+    UnitTest( const UnitTest & ) {}
+
+    // Function to pack the messages into a single data stream and send to the given processor
+    // Note: This function does not return until the message stream has been sent
+    void pack_message_stream(
+        const std::vector<std::string> &messages, const int rank, const int tag ) const;
+
+    // Function to unpack the messages from a single data stream
+    // Note: This function does not return until the message stream has been received
+    std::vector<std::string> unpack_message_stream( const int rank, const int tag ) const;
+
+    // Helper functions
+    inline void barrier() const;
+    inline std::vector<int> allGather( int value ) const;
+    inline std::vector<std::vector<std::string>> gatherMessages(
+        const std::vector<std::string> &local_messages, int tag ) const;
+};
+
+
+#endif

common/Utilities.cpp

@@ -0,0 +1,357 @@
+#include "common/Utilities.h"
+#include "common/StackTrace.h"
+
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <fstream>
+#include <string.h>
+#include <signal.h>
+#include <math.h>
+#include <algorithm>
+
+#ifdef USE_MPI
+    #include "mpi.h"
+#endif
+
+// Detect the OS and include system dependent headers
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) || defined(_MSC_VER)
+    // Note: windows has not been testeds
+    #define USE_WINDOWS
+    #include <windows.h>
+    #include <process.h>
+    #include <stdio.h>   
+    #include <tchar.h>
+    #include <psapi.h>
+    #include <DbgHelp.h>
+    #define mkdir(path, mode) _mkdir(path)
+    //#pragma comment(lib, psapi.lib) //added
+    //#pragma comment(linker, /DEFAULTLIB:psapi.lib)
+#elif defined(__APPLE__)
+    #define USE_MAC
+    #include <sys/time.h>
+    #include <signal.h>
+    #include <execinfo.h>
+    #include <dlfcn.h>
+    #include <mach/mach.h>
+    #include <unistd.h>
+#elif defined(__linux) || defined(__unix) || defined(__posix)
+    #define USE_LINUX
+    #include <sys/time.h>
+    #include <execinfo.h>
+    #include <dlfcn.h>
+    #include <malloc.h>
+    #include <unistd.h>
+#else
+    #error Unknown OS
+#endif
+
+
+#ifdef __GNUC__
+    #define USE_ABI
+    #include <cxxabi.h>
+#endif
+
+
+/****************************************************************************
+*  Function to terminate the program                                        *
+****************************************************************************/
+static bool abort_printMemory = true;
+static bool abort_printStack = true;
+static bool abort_throwException = false;
+static int force_exit = 0;
+void Utilities::setAbortBehavior( bool printMemory, bool printStack, bool throwException )
+{
+    abort_printMemory = printMemory;
+    abort_printStack = printStack;
+    abort_throwException = throwException;
+}
+void Utilities::abort(const std::string &message, const std::string &filename, const int line) 
+{
+    std::stringstream msg;
+    msg << "Program abort called in file `" << filename << "' at line " << line << std::endl;
+    // Add the memory usage and call stack to the error message
+    if ( abort_printMemory ) {
+        size_t N_bytes = Utilities::getMemoryUsage();
+        msg << "Bytes used = " << N_bytes << std::endl;
+    }
+    if ( abort_printStack ) {
+        std::vector<StackTrace::stack_info> stack = StackTrace::getCallStack();
+        msg << std::endl;
+        msg << "Stack Trace:\n";
+        for (size_t i=0; i<stack.size(); i++)
+            msg << "   " << stack[i].print() << std::endl;
+    }
+    msg << std::endl << message << std::endl;
+    // Print the message and abort
+    if ( force_exit>1 ) {
+        exit(-1);
+    } else if ( !abort_throwException ) {
+        // Use MPI_abort (will terminate all processes)
+        force_exit = 2;
+        std::cerr << msg.str();
+        #if defined(USE_MPI) || defined(HAVE_MPI)
+            int initialized=0, finalized=0;
+            MPI_Initialized(&initialized);
+            MPI_Finalized(&finalized);
+            if ( initialized!=0 && finalized==0 )
+                MPI_Abort(MPI_COMM_WORLD,-1);
+        #endif
+        exit(-1);
+    } else if ( force_exit>0 ) {
+        exit(-1);
+    } else {
+        // Throw and standard exception (allows the use of try, catch)
+        throw std::logic_error(msg.str());
+    }
+}
+
+
+/****************************************************************************
+*  Function to handle MPI errors                                            *
+****************************************************************************/
+/*#if defined(USE_MPI) || defined(HAVE_MPI)
+MPI_Errhandler mpierr;
+void MPI_error_handler_fun( MPI_Comm *comm, int *err, ... )
+{
+    if ( *err==MPI_ERR_COMM && *comm==MPI_COMM_WORLD ) {
+        // Special error handling for an invalid MPI_COMM_WORLD
+        std::cerr << "Error invalid MPI_COMM_WORLD";
+        exit(-1);
+    }
+    int msg_len=0;
+    char message[1000];
+    MPI_Error_string( *err, message, &msg_len );
+    if ( msg_len <= 0 )
+         abort("Unkown error in MPI");
+    abort( "Error calling MPI routine:\n" + std::string(message) );
+}
+#endif*/
+
+
+/****************************************************************************
+*  Function to handle unhandled exceptions                                  *
+****************************************************************************/
+bool tried_MPI_Abort=false;
+void term_func_abort(int err) 
+{
+    printf("Exiting due to abort (%i)\n",err);
+    std::vector<StackTrace::stack_info> stack = StackTrace::getCallStack();
+    std::string message = "Stack Trace:\n";
+    for (size_t i=0; i<stack.size(); i++)
+        message += "   " + stack[i].print() += "\n";
+    message += "\nExiting\n";
+    // Print the message and abort
+    std::cerr << message;
+    #ifdef USE_MPI
+        if ( !abort_throwException && !tried_MPI_Abort ) {
+            tried_MPI_Abort = true;
+            MPI_Abort(MPI_COMM_WORLD,-1);
+        }
+    #endif
+    exit(-1);
+}
+#if defined(USE_LINUX) || defined(USE_MAC)
+    static int tried_throw = 0;
+#endif
+void term_func() 
+{
+    // Try to re-throw the last error to get the last message
+    std::string last_message;
+    #if defined(USE_LINUX) || defined(USE_MAC)
+        try {
+            if ( tried_throw==0 ) { 
+                tried_throw = 1;
+                throw;
+            }
+            // No active exception
+        } catch (const std::exception &err) {
+            // Caught a std::runtime_error
+            last_message = err.what();
+        } catch (...) {
+            // Caught an unknown exception
+            last_message = "unknown exception occurred.";
+        }
+    #endif
+    std::stringstream msg;
+    msg << "Unhandled exception:" << std::endl;
+    msg << "   " << last_message << std::endl;
+    Utilities::abort( msg.str(), __FILE__, __LINE__ );
+}
+
+
+/****************************************************************************
+*  Functions to set the error handler                                       *
+****************************************************************************/
+static void setTerminateErrorHandler()
+{
+    std::set_terminate( term_func );
+    signal(SIGABRT,&term_func_abort);
+    signal(SIGFPE,&term_func_abort);
+    signal(SIGILL,&term_func_abort);
+    signal(SIGINT,&term_func_abort);
+    signal(SIGSEGV,&term_func_abort);
+    signal(SIGTERM,&term_func_abort);
+}
+void Utilities::setErrorHandlers()
+{
+    //d_use_MPI_Abort = use_MPI_Abort;
+    //setMPIErrorHandler( SAMRAI::tbox::SAMRAI_MPI::getSAMRAIWorld() );
+    setTerminateErrorHandler();
+}
+/*void Utilities::setMPIErrorHandler( const SAMRAI::tbox::SAMRAI_MPI& mpi )
+{
+    #if defined(USE_MPI) || defined(HAVE_MPI)
+        if ( mpierr.get()==NULL ) {
+            mpierr = boost::shared_ptr<MPI_Errhandler>( new MPI_Errhandler );
+            MPI_Comm_create_errhandler( MPI_error_handler_fun, mpierr.get() );
+        }
+        MPI_Comm_set_errhandler( mpi.getCommunicator(), *mpierr );
+        MPI_Comm_set_errhandler( MPI_COMM_WORLD, *mpierr );
+    #endif
+}
+void Utilities::clearMPIErrorHandler(  )
+{
+    #if defined(USE_MPI) || defined(HAVE_MPI)
+        if ( mpierr.get()!=NULL )
+            MPI_Errhandler_free( mpierr.get() );    // Delete the error handler
+        mpierr.reset();
+        MPI_Comm_set_errhandler( MPI_COMM_SELF, MPI_ERRORS_ARE_FATAL );
+        MPI_Comm_set_errhandler( MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL );
+    #endif
+}*/
+
+
+/****************************************************************************
+*  Function to get the memory usage                                         *
+*  Note: this function should be thread-safe                                *
+****************************************************************************/
+#if defined(USE_MAC)
+    // Get the page size on mac
+    static size_t page_size = static_cast<size_t>(sysconf(_SC_PAGESIZE));
+#endif
+static size_t N_bytes_initialization = Utilities::getMemoryUsage();
+size_t Utilities::getMemoryUsage()
+{
+    size_t N_bytes = 0;
+    #if defined(USE_LINUX)
+        struct mallinfo meminfo = mallinfo();
+        size_t size_hblkhd = static_cast<unsigned int>( meminfo.hblkhd );
+        size_t size_uordblks = static_cast<unsigned int>( meminfo.uordblks );
+        N_bytes = size_hblkhd + size_uordblks;
+    #elif defined(USE_MAC)
+        struct task_basic_info t_info;
+        mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
+        if (KERN_SUCCESS != task_info(mach_task_self(),
+                              TASK_BASIC_INFO, (task_info_t)&t_info, 
+                              &t_info_count)) {
+            return 0;
+        }
+        N_bytes = t_info.virtual_size;
+    #elif defined(USE_WINDOWS)
+        PROCESS_MEMORY_COUNTERS memCounter;
+        GetProcessMemoryInfo( GetCurrentProcess(), &memCounter, sizeof(memCounter) );
+        N_bytes = memCounter.WorkingSetSize;
+    #endif
+    return N_bytes;
+}
+
+
+/****************************************************************************
+*  Functions to get the time and timer resolution                           *
+****************************************************************************/
+#if defined(USE_WINDOWS)
+    double Utilities::time() 
+    { 
+        LARGE_INTEGER end, f;
+        QueryPerformanceFrequency(&f);
+        QueryPerformanceCounter(&end);       
+        double time = ((double)end.QuadPart)/((double)f.QuadPart);
+        return time;
+    }
+    double Utilities::tick() 
+    { 
+        LARGE_INTEGER f;
+        QueryPerformanceFrequency(&f);
+        double resolution = ((double)1.0)/((double)f.QuadPart);
+        return resolution;
+    }
+#elif defined(USE_LINUX) || defined(USE_MAC)
+    double Utilities::time() 
+    { 
+        timeval current_time;
+        gettimeofday(&current_time,NULL);
+        double time = ((double)current_time.tv_sec)+1e-6*((double)current_time.tv_usec);
+        return time;
+    }
+    double Utilities::tick() 
+    { 
+        timeval start, end;
+        gettimeofday(&start,NULL);
+        gettimeofday(&end,NULL);
+        while ( end.tv_sec==start.tv_sec &&  end.tv_usec==start.tv_usec )
+            gettimeofday(&end,NULL);
+        double resolution = ((double)(end.tv_sec-start.tv_sec))+1e-6*((double)(end.tv_usec-start.tv_usec));
+        return resolution;
+    }
+#else
+    #error Unknown OS
+#endif
+
+
+// Factor a number into it's prime factors
+std::vector<int> Utilities::factor(size_t number)
+{
+    if ( number<=3 ) 
+        return std::vector<int>(1,(int)number);
+    size_t i, n, n_max;
+    bool factor_found;
+    // Compute the maximum number of factors
+    int N_primes_max = 1;
+    n = number;
+    while (n >>= 1) ++N_primes_max;
+    // Initialize n, factors 
+    n = number;
+    std::vector<int> factors;
+    factors.reserve(N_primes_max);
+    while ( 1 ) {
+        // Check if n is a trivial prime number
+        if ( n==2 || n==3 || n==5 ) {
+            factors.push_back( (int) n );
+            break;
+        } 
+        // Check if n is divisible by 2
+        if ( n%2 == 0 ) {
+            factors.push_back( 2 );
+            n/=2;
+            continue;
+        } 
+        // Check each odd number until a factor is reached
+        n_max = (size_t) floor(sqrt((double) n));
+        factor_found = false;
+        for (i=3; i<=n_max; i+=2) {
+            if ( n%i == 0 ) {
+                factors.push_back( i );
+                n/=i;
+                factor_found = true;
+                break;
+            } 
+        }
+        if ( factor_found )
+            continue;
+        // No factors were found, the number must be prime
+        factors.push_back( (int) n );
+        break;
+    }
+    // Sort the factors
+    std::sort( factors.begin(), factors.end() );
+    return factors;
+}
+
+
+// Dummy function to prevent compiler from optimizing away variable
+void Utilities::nullUse( void* data )
+{
+    NULL_USE(data);
+}
+

common/Utilities.h

@@ -0,0 +1,107 @@
+#ifndef included_Utilities
+#define included_Utilities
+
+#include <chrono>
+#include <cstdarg>
+#include <iostream>
+#include <mutex>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <thread>
+#include <vector>
+
+
+namespace Utilities {
+
+
+/*!
+ * Aborts the run after printing an error message with file and
+ * linenumber information.
+ */
+void abort( const std::string &message, const std::string &filename, const int line );
+
+
+/*!
+ * Set the behavior of abort
+ * @param printMemory       Print the current memory usage (default is true)
+ * @param printStack        Print the current call stack (default is true)
+ * @param throwException    Throw an exception instead of MPI_Abort (default is false)
+ */
+void setAbortBehavior( bool printMemory, bool printStack, bool throwException );
+
+//! Function to set the error handlers
+void setErrorHandlers();
+
+
+/*!
+ * Function to get the memory availible.
+ * This function will return the total memory availible
+ * Note: depending on the implimentation, this number may be rounded to
+ * to a multiple of the page size.
+ * If this function fails, it will return 0.
+ */
+size_t getSystemMemory();
+
+
+/*!
+ * Function to get the memory usage.
+ * This function will return the total memory used by the application.
+ * Note: depending on the implimentation, this number may be rounded to
+ * to a multiple of the page size.
+ * If this function fails, it will return 0.
+ */
+size_t getMemoryUsage();
+
+
+//! Function to get an arbitrary point in time
+double time();
+
+
+//! Function to get the resolution of time
+double tick();
+
+
+//! std::string version of sprintf
+inline std::string stringf( const char *format, ... );
+
+
+/*!
+ * Sleep for X ms
+ * @param N         Time to sleep (ms)
+ */
+inline void sleep_ms( int N ) { std::this_thread::sleep_for( std::chrono::milliseconds( N ) ); }
+
+
+/*!
+ * Sleep for X s
+ * @param N         Time to sleep (s)
+ */
+inline void sleep_s( int N ) { std::this_thread::sleep_for( std::chrono::seconds( N ) ); }
+
+
+//! Factor a number into it's prime factors
+std::vector<int> factor(size_t number);
+
+//! Print AMP Banner
+void nullUse( void* );
+
+} // namespace Utilities
+
+
+#include "common/UtilityMacros.h"
+
+
+// stringf
+inline std::string Utilities::stringf( const char *format, ... )
+{
+    va_list ap;
+    va_start( ap, format );
+    char tmp[4096];
+    vsprintf( tmp, format, ap );
+    va_end( ap );
+    return std::string( tmp );
+}
+
+
+#endif

common/UtilityMacros.h

@@ -0,0 +1,182 @@
+// This file contains useful macros including ERROR, WARNING, INSIST, ASSERT, etc.
+#ifndef included_UtilityMacros
+#define included_UtilityMacros
+
+#include "common/Utilities.h"
+
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+
+
+/*! \defgroup Macros Set of utility macro functions
+ *  \details  These functions are a list of C++ macros that are used
+ *     for common operations, including checking for errors.
+ *  \addtogroup Macros
+ *  @{
+ */
+
+
+/*! \def NULL_STATEMENT
+ *  \brief    A null statement
+ *  \details  A statement that does nothing, for insure++ make it something
+ * more complex than a simple C null statement to avoid a warning.
+ */
+#ifndef NULL_STATEMENT
+#ifdef __INSURE__
+#define NULL_STATEMENT            \
+    do {                          \
+        if ( 0 )                  \
+            int nullstatement = 0 \
+    } while ( 0 )
+#else
+#define NULL_STATEMENT
+#endif
+#endif
+
+
+/*! \def NULL_USE(variable)
+ *  \brief    A null use of a variable
+ *  \details  A null use of a variable, use to avoid GNU compiler warnings about unused variables.
+ *  \param variable  Variable to pretend to use
+ */
+#ifndef NULL_USE
+#define NULL_USE( variable )                \
+    do {                                    \
+        if ( 0 ) {                          \
+            auto temp = (char *) &variable; \
+            temp++;                         \
+        }                                   \
+    } while ( 0 )
+#endif
+
+
+/*! \def ERROR(MSG)
+ *  \brief      Throw error
+ *  \details    Throw an error exception from within any C++ source code.  The 
+ *     macro argument may be any standard ostream expression.  The file and
+ *     line number of the abort are also printed.
+ *  \param MSG  Error message to print
+ */
+#define ERROR(MSG)                                                \
+    do {                                                          \
+        ::Utilities::abort( MSG, __FILE__, __LINE__ );            \
+    } while ( 0 )
+
+
+/*! \def WARNING(MSG)
+ *  \brief   Print a warning
+ *  \details Print a warning without exit.  Print file and line number of the warning.
+ *  \param MSG  Warning message to print
+ */
+#define WARNING(MSG)                                                    \
+    do {                                                                \
+        std::stringstream tboxos;                                       \
+        tboxos << MSG << std::ends;                                     \
+        printf("WARNING: %s\n   Warning called in %s on line %i\n",     \
+            tboxos.str().c_str(),__FILE__,__LINE__);                    \
+    }while(0)
+
+
+/*! \def ASSERT(EXP)
+ *  \brief Assert error
+ *  \details Throw an error exception from within any C++ source code if the
+ *     given expression is not true.  This is a parallel-friendly version
+ *     of assert.
+ *     The file and line number of the abort are printed along with the stack trace (if availible).
+ *  \param EXP  Expression to evaluate
+ */
+#define ASSERT(EXP)                                                     \
+    do {                                                                \
+        if ( !(EXP) ) {                                                 \
+            std::stringstream tboxos;                                   \
+            tboxos << "Failed assertion: " << #EXP << std::ends;        \
+            ::Utilities::abort(tboxos.str(), __FILE__, __LINE__);       \
+        }                                                               \
+    }while(0)
+
+
+/*! \def INSIST(EXP,MSG)
+ *  \brief Insist error
+ *  \details Throw an error exception from within any C++ source code if the
+ *     given expression is not true.  This will also print the given message.
+ *     This is a parallel-friendly version of assert.
+ *     The file and line number of the abort are printed along with the stack trace (if availible).
+ *  \param EXP  Expression to evaluate
+ *  \param MSG  Debug message to print
+ */
+#define INSIST(EXP,MSG) do {                                        \
+    if ( !(EXP) ) {                                                 \
+        std::stringstream tboxos;                                   \
+        tboxos << "Failed insist: " << #EXP << std::endl;           \
+        tboxos << "Message: " << MSG << std::ends;                  \
+        ::Utilities::abort(tboxos.str(), __FILE__, __LINE__);         \
+    }                                                               \
+}while(0)
+
+
+/**
+ * Macro for use when assertions are to be included
+ * only when debugging.
+ */
+/*! \def CHECK_ASSERT(EXP)
+ *  \brief Assert error (debug only)
+ *  \details Throw an error exception from within any C++ source code if the
+ *     given expression is not true.  This only runs if DEBUG_CHECK_ASSERTIONS
+ *     is enabled.  If enabled, this is the same as a call to ASSERT.
+ *  \param EXP  Expression to evaluate
+ */
+#ifdef DEBUG_CHECK_ASSERTIONS
+    #define CHECK_ASSERT(EXP) ASSERT(EXP)
+#else
+    #define CHECK_ASSERT(EXP) 
+#endif
+
+
+/*! \def DISABLE_WARNINGS
+ *  \brief Reenable warnings
+ *  \details This will re-enable warnings after a call to DIASABLE_WARNINGS
+ */
+/*! \def ENABLE_WARNINGS
+ *  \brief Supress all warnings
+ *  \details This will start to supress all compile warnings.
+ *      Be sure to follow with ENABLE_WARNINGS
+ */
+// clang-format off
+#ifdef DISABLE_WARNINGS
+    // Macros previously defined
+#elif defined( USING_MSVC )
+    #define DISABLE_WARNINGS __pragma( warning( push, 0 ) )
+    #define ENABLE_WARNINGS __pragma( warning( pop ) )
+#elif defined( USING_CLANG )
+    #define DISABLE_WARNINGS                                                \
+        _Pragma( "clang diagnostic push" ) _Pragma( "clang diagnostic ignored \"-Wall\"" ) \
+        _Pragma( "clang diagnostic ignored \"-Wextra\"" )                   \
+        _Pragma( "clang diagnostic ignored \"-Wunused-private-field\"" )    \
+        _Pragma( "clang diagnostic ignored \"-Wmismatched-new-delete\"" )
+    #define ENABLE_WARNINGS _Pragma( "clang diagnostic pop" )
+#elif defined( USING_GCC )
+    // Note: We cannot disable the -Wliteral-suffix message with this macro because the
+    // pragma command cannot suppress warnings from the C++ preprocessor.  See gcc bug #53431.
+    #define DISABLE_WARNINGS                                                \
+        _Pragma( "GCC diagnostic push" ) _Pragma( "GCC diagnostic ignored \"-Wall\"" ) \
+        _Pragma( "GCC diagnostic ignored \"-Wextra\"" )                     \
+        _Pragma( "GCC diagnostic ignored \"-Wpragmas\"" )                     \
+        _Pragma( "GCC diagnostic ignored \"-Wunused-local-typedefs\"" )     \
+        _Pragma( "GCC diagnostic ignored \"-Woverloaded-virtual\"" )        \
+        _Pragma( "GCC diagnostic ignored \"-Wunused-parameter\"" )          \
+        _Pragma( "GCC diagnostic ignored \"-Warray-bounds\"" )              \
+        _Pragma( "GCC diagnostic ignored \"-Wterminate\"" )
+    #define ENABLE_WARNINGS _Pragma( "GCC diagnostic pop" )
+#else
+    #define DISABLE_WARNINGS
+    #define ENABLE_WARNINGS
+#endif
+// clang-format on
+
+
+
+/*! @} */
+
+
+#endif

cpu/BGK.cpp

@@ -0,0 +1,278 @@
+extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
+	int n;
+	// conserved momemnts
+	double rho,ux,uy,uz,uu;
+	// non-conserved moments
+	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
+
+	for (int n=start; n<finish; n++){
+		// q=0
+		f0 = dist[n];
+		f1 = dist[2*Np+n];
+		f2 = dist[1*Np+n];
+		f3 = dist[4*Np+n];
+		f4 = dist[3*Np+n];
+		f5 = dist[6*Np+n];
+		f6 = dist[5*Np+n];
+		f7 = dist[8*Np+n];
+		f8 = dist[7*Np+n];
+		f9 = dist[10*Np+n];
+		f10 = dist[9*Np+n];
+		f11 = dist[12*Np+n];
+		f12 = dist[11*Np+n];
+		f13 = dist[14*Np+n];
+		f14 = dist[13*Np+n];
+		f15 = dist[16*Np+n];
+		f16 = dist[15*Np+n];
+		f17 = dist[18*Np+n];
+		f18 = dist[17*Np+n];
+
+		rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
+		ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+		uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+		uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+		uu = 1.5*(ux*ux+uy*uy+uz*uz);
+
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu);
+
+		// q = 1
+		dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx;
+
+		// q=2
+		dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)-  0.16666666*Fx;
+
+		// q = 3
+		dist[3*Np+n] = f3*(1.0-rlx) +
+				rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy;
+
+		// q = 4
+		dist[4*Np+n] = f4*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy;
+
+		// q = 5
+		dist[5*Np+n] = f5*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz;
+
+		// q = 6
+		dist[6*Np+n] = f6*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz;
+
+		// q = 7
+		dist[7*Np+n] = f7*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy);
+
+		// q = 8
+		dist[8*Np+n] = f8*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy);
+
+		// q = 9
+		dist[9*Np+n] = f9*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy);
+
+		// q = 10
+		dist[10*Np+n] = f10*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy);
+
+		// q = 11
+		dist[11*Np+n] = f11*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz);
+
+		// q = 12
+		dist[12*Np+n] = f12*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu)  - 0.08333333333*(Fx+Fz);
+
+		// q = 13
+		dist[13*Np+n] = f13*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz);
+
+		// q= 14
+		dist[14*Np+n] = f14*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz);
+
+		// q = 15
+		dist[15*Np+n] = f15*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz);
+
+		// q = 16
+		dist[16*Np+n] = f16*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz);
+
+		// q = 17
+		dist[17*Np+n] = f17*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz);
+
+		// q = 18
+		dist[18*Np+n] = f18*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
+
+		//........................................................................
+	}
+}
+
+extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
+	int n;
+	// conserved momemnts
+	double rho,ux,uy,uz,uu;
+	// non-conserved moments
+	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
+	int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
+
+	int nread;
+	for (int n=start; n<finish; n++){
+		
+		// q=0
+		f0 = dist[n];
+		// q=1
+		nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
+		f1 = dist[nr1]; // reading the f1 data into register fq
+
+		nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
+		f2 = dist[nr2];  // reading the f2 data into register fq
+
+		// q=3
+		nr3 = neighborList[n+2*Np]; // neighbor 4
+		f3 = dist[nr3];
+
+		// q = 4
+		nr4 = neighborList[n+3*Np]; // neighbor 3
+		f4 = dist[nr4];
+
+		// q=5
+		nr5 = neighborList[n+4*Np];
+		f5 = dist[nr5];
+
+		// q = 6
+		nr6 = neighborList[n+5*Np];
+		f6 = dist[nr6];
+		
+		// q=7
+		nr7 = neighborList[n+6*Np];
+		f7 = dist[nr7];
+
+		// q = 8
+		nr8 = neighborList[n+7*Np];
+		f8 = dist[nr8];
+
+		// q=9
+		nr9 = neighborList[n+8*Np];
+		f9 = dist[nr9];
+
+		// q = 10
+		nr10 = neighborList[n+9*Np];
+		f10 = dist[nr10];
+
+		// q=11
+		nr11 = neighborList[n+10*Np];
+		f11 = dist[nr11];
+
+		// q=12
+		nr12 = neighborList[n+11*Np];
+		f12 = dist[nr12];
+
+		// q=13
+		nr13 = neighborList[n+12*Np];
+		f13 = dist[nr13];
+
+		// q=14
+		nr14 = neighborList[n+13*Np];
+		f14 = dist[nr14];
+
+		// q=15
+		nr15 = neighborList[n+14*Np];
+		f15 = dist[nr15];
+
+		// q=16
+		nr16 = neighborList[n+15*Np];
+		f16 = dist[nr16];
+
+		// q=17
+		//fq = dist[18*Np+n];
+		nr17 = neighborList[n+16*Np];
+		f17 = dist[nr17];
+
+		// q=18
+		nr18 = neighborList[n+17*Np];
+		f18 = dist[nr18];
+
+		rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
+		ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+		uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+		uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+		uu = 1.5*(ux*ux+uy*uy+uz*uz);
+
+		// q=0
+		dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu);
+
+		// q = 1
+		dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx;
+
+		// q=2
+		dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)-  0.16666666*Fx;
+
+		// q = 3
+		dist[nr4] = f3*(1.0-rlx) +
+				rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy;
+
+		// q = 4
+		dist[nr3] = f4*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy;
+
+		// q = 5
+		dist[nr6] = f5*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz;
+
+		// q = 6
+		dist[nr5] = f6*(1.0-rlx) + 
+				rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz;
+
+		// q = 7
+		dist[nr8] = f7*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy);
+
+		// q = 8
+		dist[nr7] = f8*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy);
+
+		// q = 9
+		dist[nr10] = f9*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy);
+
+		// q = 10
+		dist[nr9] = f10*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy);
+
+		// q = 11
+		dist[nr12] = f11*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz);
+
+		// q = 12
+		dist[nr11] = f12*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu)  - 0.08333333333*(Fx+Fz);
+
+		// q = 13
+		dist[nr14] = f13*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz);
+
+		// q= 14
+		dist[nr13] = f14*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz);
+
+		// q = 15
+		dist[nr16] = f15*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz);
+
+		// q = 16
+		dist[nr15] = f16*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz);
+
+		// q = 17
+		dist[nr18] = f17*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz);
+
+		// q = 18
+		dist[nr17] = f18*(1.0-rlx) + 
+				rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
+
+	}
+}

cpu/D3Q7.cpp

@@ -0,0 +1,187 @@
+// CPU Functions for D3Q7 Lattice Boltzmann Methods
+
+extern "C" void ScaLBL_Scalar_Pack(int *list, int count, double *sendbuf, double *Data, int N){
+	//....................................................................................
+	// Pack distribution q into the send buffer for the listed lattice sites
+	// dist may be even or odd distributions stored by stream layout
+	//....................................................................................
+	int idx,n;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		sendbuf[idx] = Data[n];
+	}
+}
+extern "C" void ScaLBL_Scalar_Unpack(int *list, int count, double *recvbuf, double *Data, int N){
+	//....................................................................................
+	// Pack distribution q into the send buffer for the listed lattice sites
+	// dist may be even or odd distributions stored by stream layout
+	//....................................................................................
+	int idx,n;
+	for (idx=0; idx<count; idx++){
+		n = list[idx];
+		Data[n] = recvbuf[idx];
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Unpack(int q, int *list,  int start, int count,
+		double *recvbuf, double *dist, int N){
+	//....................................................................................
+	// Unack distribution from the recv buffer
+	// Distribution q matche Cqx, Cqy, Cqz
+	// swap rule means that the distributions in recvbuf are OPPOSITE of q
+	// dist may be even or odd distributions stored by stream layout
+	//....................................................................................
+	int n,idx;
+	for (idx=0; idx<count; idx++){
+		// Get the value from the list -- note that n is the index is from the send (non-local) process
+		n = list[idx];
+		// unpack the distribution to the proper location
+		if (!(n<0)) dist[q*N+n] = recvbuf[start+idx];
+		//dist[q*N+n] = recvbuf[start+idx];
+
+	}
+}
+
+
+extern "C" void ScaLBL_PackDenD3Q7(int *list, int count, double *sendbuf, int number, double *Data, int N){
+	//....................................................................................
+	// Pack distribution into the send buffer for the listed lattice sites
+	//....................................................................................
+	int idx,n,component;
+	for (idx=0; idx<count; idx++){
+		for (component=0; component<number; component++){
+			n = list[idx];
+			sendbuf[idx*number+component] = Data[number*n+component];
+			Data[number*n+component] = 0.0;	// Set the data value to zero once it's in the buffer!
+		}
+	}
+}
+
+
+extern  "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int number, double *Data, int N){
+	//....................................................................................
+	// Unack distribution from the recv buffer
+	// Sum to the existing density value
+	//....................................................................................
+	int idx,n,component;
+	for (idx=0; idx<count; idx++){
+		for (component=0; component<number; component++){
+			n = list[idx];
+			Data[number*n+component] += recvbuf[idx*number+component];
+		}
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_Init(char *ID, double *f_even, double *f_odd, double *Den, int Nx, int Ny, int Nz)
+{
+	int n,N;
+	N = Nx*Ny*Nz;
+	double value;
+
+	for (n=0; n<N; n++){
+
+		if (ID[n] > 0){
+			value = Den[n];
+			f_even[n] = 0.3333333333333333*value;
+			f_odd[n] = 0.1111111111111111*value;		//double(100*n)+1.f;
+			f_even[N+n] = 0.1111111111111111*value;	//double(100*n)+2.f;
+			f_odd[N+n] = 0.1111111111111111*value;	//double(100*n)+3.f;
+			f_even[2*N+n] = 0.1111111111111111*value;	//double(100*n)+4.f;
+			f_odd[2*N+n] = 0.1111111111111111*value;	//double(100*n)+5.f;
+			f_even[3*N+n] = 0.1111111111111111*value;	//double(100*n)+6.f;
+		}
+		else{
+			for(int q=0; q<3; q++){
+				f_even[q*N+n] = -1.0;
+				f_odd[q*N+n] = -1.0;
+			}
+			f_even[3*N+n] = -1.0;
+		}
+	}
+}
+
+//*************************************************************************
+extern "C" void ScaLBL_D3Q7_Swap(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz)
+{
+	int i,j,k,n,nn,N;
+	// distributions
+	double f1,f2,f3,f4,f5,f6;
+	
+	N = Nx*Ny*Nz;
+	
+	for (n=0; n<N; n++){
+		//.......Back out the 3-D indices for node n..............
+		k = n/(Nx*Ny);
+		j = (n-Nx*Ny*k)/Nx;
+		i = n-Nx*Ny*k-Nz*j;
+		
+		if (ID[n] > 0){
+			//........................................................................
+			// Retrieve even distributions from the local node (swap convention)
+			//		f0 = disteven[n];  // Does not particupate in streaming
+			f1 = distodd[n];
+			f3 = distodd[N+n];
+			f5 = distodd[2*N+n];
+			//........................................................................
+			
+			//........................................................................
+			// Retrieve odd distributions from neighboring nodes (swap convention)
+			//........................................................................
+			nn = n+1;							// neighbor index (pull convention)
+			if (!(i+1<Nx))	nn -= Nx;			// periodic BC along the x-boundary
+			//if (i+1<Nx){
+			f2 = disteven[N+nn];					// pull neighbor for distribution 2
+			if (!(f2 < 0.0)){
+				distodd[n] = f2;
+				disteven[N+nn] = f1;
+			}
+			//}
+			//........................................................................
+			nn = n+Nx;							// neighbor index (pull convention)
+			if (!(j+1<Ny))	nn -= Nx*Ny;		// Perioidic BC along the y-boundary
+			//if (j+1<Ny){
+			f4 = disteven[2*N+nn];				// pull neighbor for distribution 4
+			if (!(f4 < 0.0)){
+				distodd[N+n] = f4;
+				disteven[2*N+nn] = f3;
+				//	}
+			}
+			//........................................................................
+			nn = n+Nx*Ny;						// neighbor index (pull convention)
+			if (!(k+1<Nz))	nn -= Nx*Ny*Nz;		// Perioidic BC along the z-boundary
+			//if (k+1<Nz){
+			f6 = disteven[3*N+nn];				// pull neighbor for distribution 6
+			if (!(f6 < 0.0)){
+				distodd[2*N+n] = f6;
+				disteven[3*N+nn] = f5;
+				//	}
+			}
+		}
+	}
+}
+
+//*************************************************************************
+extern "C" void ScaLBL_D3Q7_Density(char *ID, double *disteven, double *distodd, double *Den, 
+										int Nx, int Ny, int Nz)
+{
+	char id;
+	int n;
+	double f0,f1,f2,f3,f4,f5,f6;
+	int N = Nx*Ny*Nz;
+	
+	for (n=0; n<N; n++){
+		id = ID[n];
+		if (id > 0 ){
+			// Read the distributions
+			f0 = disteven[n];
+			f2 = disteven[N+n];
+			f4 = disteven[2*N+n];
+			f6 = disteven[3*N+n];
+			f1 = distodd[n];
+			f3 = distodd[N+n];
+			f5 = distodd[2*N+n];
+			// Compute the density
+			Den[n] = f0+f1+f2+f3+f4+f5+f6;
+		}
+	}
+}

cpu/Extras.cpp

@@ -0,0 +1,53 @@
+// Basic cuda functions callable from C/C++ code
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <mm_malloc.h>
+
+extern "C" int ScaLBL_SetDevice(int rank){
+	return 0;
+}
+
+extern "C" void ScaLBL_AllocateZeroCopy(void** address, size_t size){
+	//cudaMalloc(address,size);
+	(*address) = _mm_malloc(size,64);
+	memset(*address,0,size);
+	
+	if (*address==NULL){
+		printf("Memory allocation failed! \n");
+	}
+}
+
+extern "C" void ScaLBL_AllocateDeviceMemory(void** address, size_t size){
+	//cudaMalloc(address,size);
+	(*address) = _mm_malloc(size,64);
+	memset(*address,0,size);
+	
+	if (*address==NULL){
+		printf("Memory allocation failed! \n");
+	}
+}
+
+extern "C" void ScaLBL_FreeDeviceMemory(void* pointer){
+	_mm_free(pointer);
+}
+
+extern "C" void ScaLBL_CopyToDevice(void* dest, const void* source, size_t size){
+//	cudaMemcpy(dest,source,size,cudaMemcpyHostToDevice);
+	memcpy(dest, source, size);
+}
+
+
+extern "C" void ScaLBL_CopyToHost(void* dest, const void* source, size_t size){
+//	cudaMemcpy(dest,source,size,cudaMemcpyDeviceToHost);
+	memcpy(dest, source, size);
+}
+
+extern "C" void ScaLBL_CopyToZeroCopy(void* dest, const void* source, size_t size){
+//	cudaMemcpy(dest,source,size,cudaMemcpyDeviceToHost);
+	memcpy(dest, source, size);
+}
+
+extern "C" void ScaLBL_DeviceBarrier(){
+//	cudaDeviceSynchronize();
+}

cpu/MRT.cpp

@@ -0,0 +1,277 @@
+
+extern "C" void INITIALIZE(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz)
+{
+	int n,N;
+	N = Nx*Ny*Nz;
+
+	for (n=0; n<N; n++){
+
+			if (ID[n] > 0){
+				f_even[n] = 0.3333333333333333;
+				f_odd[n] = 0.055555555555555555;		//double(100*n)+1.f;
+				f_even[N+n] = 0.055555555555555555;	//double(100*n)+2.f;
+				f_odd[N+n] = 0.055555555555555555;	//double(100*n)+3.f;
+				f_even[2*N+n] = 0.055555555555555555;	//double(100*n)+4.f;
+				f_odd[2*N+n] = 0.055555555555555555;	//double(100*n)+5.f;
+				f_even[3*N+n] = 0.055555555555555555;	//double(100*n)+6.f;
+				f_odd[3*N+n] = 0.0277777777777778;   //double(100*n)+7.f;
+				f_even[4*N+n] = 0.0277777777777778;   //double(100*n)+8.f;
+				f_odd[4*N+n] = 0.0277777777777778;   //double(100*n)+9.f;
+				f_even[5*N+n] = 0.0277777777777778;  //double(100*n)+10.f;
+				f_odd[5*N+n] = 0.0277777777777778;  //double(100*n)+11.f;
+				f_even[6*N+n] = 0.0277777777777778;  //double(100*n)+12.f;
+				f_odd[6*N+n] = 0.0277777777777778;  //double(100*n)+13.f;
+				f_even[7*N+n] = 0.0277777777777778;  //double(100*n)+14.f;
+				f_odd[7*N+n] = 0.0277777777777778;  //double(100*n)+15.f;
+				f_even[8*N+n] = 0.0277777777777778;  //double(100*n)+16.f;
+				f_odd[8*N+n] = 0.0277777777777778;  //double(100*n)+17.f;
+				f_even[9*N+n] = 0.0277777777777778;  //double(100*n)+18.f;
+			}
+			else{
+				for(int q=0; q<9; q++){
+					f_even[q*N+n] = -1.0;
+					f_odd[q*N+n] = -1.0;
+				}
+				f_even[9*N+n] = -1.0;
+			}
+		}
+	
+}
+
+extern "C" void Compute_VELOCITY(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz)
+{
+	int n,N;
+	// distributions
+	double f1,f2,f3,f4,f5,f6,f7,f8,f9;
+	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
+	double vx,vy,vz;
+
+	N = Nx*Ny*Nz;
+	for (n=0; n<N; n++){
+
+			if (ID[n] > 0){
+				//........................................................................
+				// Registers to store the distributions
+				//........................................................................
+				f2 = disteven[N+n];
+				f4 = disteven[2*N+n];
+				f6 = disteven[3*N+n];
+				f8 = disteven[4*N+n];
+				f10 = disteven[5*N+n];
+				f12 = disteven[6*N+n];
+				f14 = disteven[7*N+n];
+				f16 = disteven[8*N+n];
+				f18 = disteven[9*N+n];
+				//........................................................................
+				f1 = distodd[n];
+				f3 = distodd[1*N+n];
+				f5 = distodd[2*N+n];
+				f7 = distodd[3*N+n];
+				f9 = distodd[4*N+n];
+				f11 = distodd[5*N+n];
+				f13 = distodd[6*N+n];
+				f15 = distodd[7*N+n];
+				f17 = distodd[8*N+n];
+				//.................Compute the velocity...................................
+				vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+				vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+				vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+				//..................Write the velocity.....................................
+				vel[n] = vx;
+				vel[N+n] = vy;
+				vel[2*N+n] = vz;
+				//........................................................................
+
+			}
+		}
+	}
+
+
+//*************************************************************************
+extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz,
+					double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz)
+{
+
+	int n,N;
+	// distributions
+	double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
+	double f10,f11,f12,f13,f14,f15,f16,f17,f18;
+
+	// conserved momemnts
+	double rho,jx,jy,jz;
+	// non-conserved moments
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+
+	N = Nx*Ny*Nz;
+
+	char id;
+	for (n=0; n<N; n++){
+		id = ID[n];
+
+			if (id > 0){
+				//........................................................................
+				// Registers to store the distributions - read based on swap convention
+				//........................................................................
+				f2 = distodd[n];
+				f4 = distodd[N+n];
+				f6 = distodd[2*N+n];
+				f8 = distodd[3*N+n];
+				f10 = distodd[4*N+n];
+				f12 = distodd[5*N+n];
+				f14 = distodd[6*N+n];
+				f16 = distodd[7*N+n];
+				f18 = distodd[8*N+n];
+				//........................................................................
+				f0 = disteven[n];
+				f1 = disteven[N+n];
+				f3 = disteven[2*N+n];
+				f5 = disteven[3*N+n];
+				f7 = disteven[4*N+n];
+				f9 = disteven[5*N+n];
+				f11 = disteven[6*N+n];
+				f13 = disteven[7*N+n];
+				f15 = disteven[8*N+n];
+				f17 = disteven[9*N+n];
+				//........................................................................
+				//....................compute the moments...............................................
+				rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
+				m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17);
+				m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
+				jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
+				m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14;
+				jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
+				m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18;
+				jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
+				m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18;
+				m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18);
+				m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17);
+				m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13;
+				m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13;
+				m13 = f8+f7-f10-f9;
+				m14 = f16+f15-f18-f17;
+				m15 = f12+f11-f14-f13;
+				m16 = f7-f8+f9-f10-f11+f12-f13+f14;
+				m17 = -f7+f8+f9-f10+f15-f16+f17-f18;
+				m18 = f11-f12-f13+f14-f15+f16+f17-f18;
+				//..............incorporate external force................................................
+				//jx += 0.5*Fx;
+				//jy += 0.5*Fy;
+				//jz += 0.5*Fz;
+				//..............carry out relaxation process...............................................
+				m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1);
+				m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2);
+				m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
+				m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
+				m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
+				m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9);
+				m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
+				m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11);
+				m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12);
+				m13 = m13 + rlx_setA*((jx*jy/rho) - m13);
+				m14 = m14 + rlx_setA*((jy*jz/rho) - m14);
+				m15 = m15 + rlx_setA*((jx*jz/rho) - m15);
+				m16 = m16 + rlx_setB*( - m16);
+				m17 = m17 + rlx_setB*( - m17);
+				m18 = m18 + rlx_setB*( - m18);
+				//.................inverse transformation......................................................
+				f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2;
+				f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(jx-m4)+0.05555555555555555*(m9-m10);
+				f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(m4-jx)+0.05555555555555555*(m9-m10);
+				f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
+				f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
+				f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
+				f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+				+0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
+				f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6)
+				+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+				+0.04166666666666666*m12+0.25*m13+0.125*(m16-m17);
+				f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6)
+				+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+				+0.04166666666666666*m12+0.25*m13+0.125*(m17-m16);
+				f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6)
+				+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+				+0.04166666666666666*m12-0.25*m13+0.125*(m16+m17);
+				f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4)
+				+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+				+0.04166666666666666*m12-0.25*m13-0.125*(m16+m17);
+				f11 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8)
+				+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
+				-0.04166666666666666*m12+0.25*m15+0.125*(m18-m16);
+				f12 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8)
+				+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
+				-0.04166666666666666*m12+0.25*m15+0.125*(m16-m18);
+				f13 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8)
+				+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
+				-0.04166666666666666*m12-0.25*m15-0.125*(m16+m18);
+				f14 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4)
+				+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
+				-0.04166666666666666*m12-0.25*m15+0.125*(m16+m18);
+				f15 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8)
+				-0.05555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18);
+				f16 =  0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8)
+				-0.05555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17);
+				f17 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8)
+				-0.05555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18);
+				f18 = 0.05263157894736842*rho+0.003341687552213868*m1
+				+0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6)
+				-0.05555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18);
+				//.......................................................................................................
+				// incorporate external force
+				f1 += 0.16666666*Fx;
+				f2 -= 0.16666666*Fx;
+				f3 += 0.16666666*Fy;
+				f4 -= 0.16666666*Fy;
+				f5 += 0.16666666*Fz;
+				f6 -= 0.16666666*Fz;
+				f7 += 0.08333333333*(Fx+Fy);
+				f8 -= 0.08333333333*(Fx+Fy);
+				f9 += 0.08333333333*(Fx-Fy);
+				f10 -= 0.08333333333*(Fx-Fy);
+				f11 += 0.08333333333*(Fx+Fz);
+				f12 -= 0.08333333333*(Fx+Fz);
+				f13 += 0.08333333333*(Fx-Fz);
+				f14 -= 0.08333333333*(Fx-Fz);
+				f15 += 0.08333333333*(Fy+Fz);
+				f16 -= 0.08333333333*(Fy+Fz);
+				f17 += 0.08333333333*(Fy-Fz);
+				f18 -= 0.08333333333*(Fy-Fz);
+				//.......................................................................................................
+				// Write data based on un-swapped convention
+				disteven[n] = f0;
+				disteven[N+n] = f2;
+				disteven[2*N+n] = f4;
+				disteven[3*N+n] = f6;
+				disteven[4*N+n] = f8;
+				disteven[5*N+n] = f10;
+				disteven[6*N+n] = f12;
+				disteven[7*N+n] = f14;
+				disteven[8*N+n] = f16;
+				disteven[9*N+n] = f18;
+
+				distodd[n] = f1;
+				distodd[N+n] = f3;
+				distodd[2*N+n] = f5;
+				distodd[3*N+n] = f7;
+				distodd[4*N+n] = f9;
+				distodd[5*N+n] = f11;
+				distodd[6*N+n] = f13;
+				distodd[7*N+n] = f15;
+				distodd[8*N+n] = f17;
+				//.......................................................................................................
+			}
+		}
+	}
+
+

cpu/Makefile

@@ -0,0 +1,46 @@
+CXX=mpicxx
+FLAGS=-O3
+INC=../include
+
+all:ColorLBM-cpu ColorLBM-CBUB
+
+ColorLBM-cpu:Extras.o D3Q19.o D3Q7.o Color.o lb2_Color_wia_mpi.o
+	$(CXX) $(FLAGS) -I$(INC) -o ColorLBM-cpu lb2_Color_wia_mpi.o D3Q19.o D3Q7.o Color.o Extras.o
+
+ColorLBM-CBUB:Extras.o D3Q19.o D3Q7.o Color.o lb2_Color_wia_mpi-CBUB.o
+	$(CXX) $(FLAGS) -I$(INC) -DCBUB -o ColorLBM-CBUB lb2_Color_wia_mpi-CBUB.o D3Q19.o D3Q7.o Color.o Extras.o
+
+ColorLBM-BUB:Extras.o D3Q19.o D3Q7.o Color.o lb2_Color_wia_mpi_bubble.o
+	$(CXX) $(FLAGS) -I$(INC) -DCBUB -o ColorLBM-BUB lb2_Color_wia_mpi_bubble.o D3Q19.o D3Q7.o Color.o Extras.o
+
+D3Q19.o:D3Q19.cpp
+	$(CXX) $(FLAGS) -c -o D3Q19.o D3Q19.cpp
+
+D3Q7.o:D3Q7.cpp
+	$(CXX) $(FLAGS) -c -o D3Q7.o D3Q7.cpp
+
+Extras.o:Extras.cpp
+	$(CXX) $(FLAGS) -c -o Extras.o Extras.cpp
+
+Color.o:Color.cpp
+	$(CXX) $(FLAGS) -c -o Color.o Color.cpp
+
+lb2_Color_wia_mpi.o:lb2_Color_wia_mpi.cpp
+	$(CXX) $(FLAGS) -I$(INC) -c -o lb2_Color_wia_mpi.o lb2_Color_wia_mpi.cpp
+
+lb2_Color_wia_mpi-CBUB.o:lb2_Color_wia_mpi.cpp
+	$(CXX) $(FLAGS) -I$(INC) -DCBUB -c -o lb2_Color_wia_mpi-CBUB.o lb2_Color_wia_mpi.cpp
+
+lb2_Color_wia_mpi_bubble.o:lb2_Color_wia_mpi_bubble.cpp
+	$(CXX) $(FLAGS) -I$(INC) -DCBUB -c -o lb2_Color_wia_mpi_bubble.o lb2_Color_wia_mpi_bubble.cpp
+
+#MRT-MPI.o:lb1_MRT_mpi.cpp
+#	$(CXX) -c -o MRT-MPI.o lb1_MRT_mpi.cpp
+
+#MRT-MPI:D3Q19.o MRT-MPI.o
+#	$(CXX) -o MRT-MPI D3Q19.o MRT-MPI.o
+
+clean:
+	rm *.o
+#	rm bin/*
+#	rm lib/*

cpu/exe/CMakeLists.txt

@@ -0,0 +1 @@
+INSTALL_LBPM_EXE( lb2_Color_mpi )

cpu/thermal.cpp

@@ -0,0 +1,5 @@
+// cpu implementation for thermal lattice boltzmann methods
+// copyright James McClure, 2014
+
+
+

ctest_script.cmake

@@ -0,0 +1,211 @@
+# ctest script for building, running, and submitting the test results 
+# Usage:  ctest -s script,build
+#   build = debug / optimized / valgrind
+# Note: this test will use use the number of processors defined in the variable N_PROCS,
+#   the enviornmental variable N_PROCS, or the number of processors availible (if not specified)
+
+# Set platform specific variables
+SITE_NAME( HOSTNAME )
+IF( ${HOSTNAME} STREQUAL "lap0086227" )
+    SET( COVERAGE_COMMAND /usr/bin/gcov )
+    SET( VALGRIND_COMMAND /usr/bin/valgrind )
+    SET( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
+    SET( CC "mpicc" )
+    SET( CXX "mpicxx" )
+    SET( C_FLAGS "-DCBUB" )
+    SET( CXX_FLAGS "-DCBUB" )
+    SET( MPIEXEC "mpirun" )
+ELSEIF( ${HOSTNAME} MATCHES "vayu" ) 
+    SET( COVERAGE_COMMAND /usr/bin/gcov )
+    SET( VALGRIND_COMMAND /usr/local/bin/valgrind )
+    SET( CUDA_FLAGS "--use_fast_math -Xptxas=-v -arch=sm_20" )
+    SET( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
+    SET( CC "mpicc" )
+    SET( CXX "mpicxx" )
+    SET( C_FLAGS "-DCBUB" )
+    SET( CXX_FLAGS "-DCBUB" )
+    SET( MPIEXEC "mpirun" )
+ELSEIF( ${HOSTNAME} MATCHES "titan.*" ) 
+    SET( COVERAGE_COMMAND "" )
+    SET( VALGRIND_COMMAND "" )
+    SET( CUDA_FLAGS "-arch sm_35" )
+    SET( CUDA_HOST_COMPILER "/usr/bin/g++" )
+    SET( CTEST_CMAKE_GENERATOR "Unix Makefiles" )
+    SET( CC "cc" )
+    SET( CXX "CC" )
+    SET( C_FLAGS "-DCBUB" )
+    SET( CXX_FLAGS "-DCBUB" )
+    SET( MPIEXEC "aprun" )
+    SET( N_PROCS 16 )
+ELSE()
+    MESSAGE( FATAL_ERROR "Unknown host: ${HOSTNAME}" )
+ENDIF()
+
+
+# Get the source directory based on the current directory
+SET( LBPM_DIR "${CMAKE_CURRENT_LIST_DIR}" )
+
+
+# Check that we specified the build type to run
+IF( NOT CTEST_SCRIPT_ARG )
+    MESSAGE(FATAL_ERROR "No build specified: ctest -S /path/to/script,build (debug/optimized/valgrind")
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "debug" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-debug" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( CTEST_COVERAGE_COMMAND ${COVERAGE_COMMAND} )
+    SET( ENABLE_GCOV "true" )
+    SET( USE_VALGRIND FALSE )
+    SET( USE_CUDA FALSE )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "debug-cuda" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-debug-cuda" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( CTEST_COVERAGE_COMMAND ${COVERAGE_COMMAND} )
+    SET( ENABLE_GCOV "true" )
+    SET( USE_VALGRIND FALSE )
+    SET( USE_CUDA TRUE )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "optimized") OR (${CTEST_SCRIPT_ARG} STREQUAL "opt") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-opt" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+    SET( CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+    SET( USE_VALGRIND FALSE )
+    SET( USE_CUDA FALSE )
+ELSEIF( (${CTEST_SCRIPT_ARG} STREQUAL "optimized-cuda") OR (${CTEST_SCRIPT_ARG} STREQUAL "opt-cuda") )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-opt-cuda" )
+    SET( CMAKE_BUILD_TYPE "Release" )
+    SET( CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+    SET( USE_VALGRIND FALSE )
+    SET( USE_CUDA TRUE )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "valgrind" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-valgrind" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+    SET( USE_VALGRIND TRUE )
+    SET( USE_CUDA FALSE )
+ELSEIF( ${CTEST_SCRIPT_ARG} STREQUAL "valgrind-cuda" )
+    SET( CTEST_BUILD_NAME "LBPM-WIA-valgrind-cuda" )
+    SET( CMAKE_BUILD_TYPE "Debug" )
+    SET( CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+    SET( USE_VALGRIND TRUE )
+    SET( USE_CUDA TRUE )
+ELSE()
+    MESSAGE(FATAL_ERROR "Invalid build (${CTEST_SCRIPT_ARG}): ctest -S /path/to/script,build (debug/opt/valgrind")
+ENDIF()
+IF ( NOT CTEST_COVERAGE_COMMAND )
+    SET( ENABLE_GCOV "false" )
+ENDIF()
+
+
+# Set the number of processors
+IF( NOT DEFINED N_PROCS )
+    SET( N_PROCS $ENV{N_PROCS} )
+ENDIF()
+IF( NOT DEFINED N_PROCS )
+    SET(N_PROCS 1)
+    # Linux:
+    SET(cpuinfo_file "/proc/cpuinfo")
+    IF(EXISTS "${cpuinfo_file}")
+        FILE(STRINGS "${cpuinfo_file}" procs REGEX "^processor.: [0-9]+$")
+        list(LENGTH procs N_PROCS)
+    ENDIF()
+    # Mac:
+    IF(APPLE)
+        find_program(cmd_sys_pro "system_profiler")
+        if(cmd_sys_pro)
+            execute_process(COMMAND ${cmd_sys_pro} OUTPUT_VARIABLE info)
+            STRING(REGEX REPLACE "^.*Total Number Of Cores: ([0-9]+).*$" "\\1" N_PROCS "${info}")
+        ENDIF()
+    ENDIF()
+    # Windows:
+    IF(WIN32)
+        SET(N_PROCS "$ENV{NUMBER_OF_PROCESSORS}")
+    ENDIF()
+ENDIF()
+
+
+# Set basic variables
+SET( CTEST_PROJECT_NAME "LBPM-WIA" )
+SET( CTEST_SOURCE_DIRECTORY "${LBPM_DIR}" )
+SET( CTEST_BINARY_DIRECTORY "." )
+SET( CTEST_DASHBOARD "Nightly" )
+SET( CTEST_TEST_TIMEOUT 300 )
+SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 500 )
+SET( CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 500 )
+SET( CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 10000 )
+SET( CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 10000 )
+SET( NIGHTLY_START_TIME "18:00:00 EST" )
+SET( CTEST_NIGHTLY_START_TIME "22:00:00 EST" )
+SET( CTEST_COMMAND "\"${CTEST_EXECUTABLE_NAME}\" -D ${CTEST_DASHBOARD}" )
+SET( CTEST_BUILD_COMMAND "make -i -j ${N_PROCS} install" )
+
+
+# Set valgrind options
+#SET (VALGRIND_COMMAND_OPTIONS "--tool=memcheck --leak-check=yes --track-fds=yes --num-callers=50 --show-reachable=yes --track-origins=yes --malloc-fill=0xff --free-fill=0xfe --suppressions=${LBPM_DIR}/ValgrindSuppresionFile" )
+SET( VALGRIND_COMMAND_OPTIONS  "--tool=memcheck --leak-check=yes --track-fds=yes --num-callers=50 --show-reachable=yes --suppressions=${LBPM_DIR}/ValgrindSuppresionFile" )
+IF ( USE_VALGRIND )
+    SET( MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
+    SET( MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECK_COMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECKCOMMAND ${VALGRIND_COMMAND} )
+    SET( CTEST_MEMORYCHECK_COMMAND_OPTIONS ${VALGRIND_COMMAND_OPTIONS} )
+    SET( CTEST_MEMORYCHECKCOMMAND_OPTIONS  ${VALGRIND_COMMAND_OPTIONS} )
+ENDIF()
+
+
+# Clear the binary directory and create an initial cache
+CTEST_EMPTY_BINARY_DIRECTORY (${CTEST_BINARY_DIRECTORY})
+FILE(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" "CTEST_TEST_CTEST:BOOL=1")
+
+# Set the configure options
+SET( CTEST_OPTIONS )
+SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
+IF ( USE_CUDA )
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" )
+ELSE()
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=false" )
+ENDIF()
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDLIBS:STRING=\"${LDLIBS}\"" )
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
+
+# Configure and run the tests
+SET( CTEST_SITE ${HOSTNAME} )
+CTEST_START("${CTEST_DASHBOARD}")
+CTEST_UPDATE()
+CTEST_CONFIGURE(
+    BUILD   ${CTEST_BINARY_DIRECTORY}
+    SOURCE  ${CTEST_SOURCE_DIRECTORY}
+    OPTIONS "${CTEST_OPTIONS}"
+)
+CTEST_BUILD()
+IF ( USE_VALGRIND_MATLAB )
+    CTEST_TEST( INCLUDE MATLAB--test_hello_world  PARALLEL_LEVEL ${N_PROCS} )
+ELSEIF ( USE_VALGRIND )
+    CTEST_MEMCHECK( EXCLUDE procs   PARALLEL_LEVEL ${N_PROCS} )
+ELSE()
+    # CTEST_TEST( EXCLUDE WEEKLY  PARALLEL_LEVEL ${N_PROCS} )
+    CTEST_TEST( PARALLEL_LEVEL ${N_PROCS} )
+ENDIF()
+IF( CTEST_COVERAGE_COMMAND )
+    CTEST_COVERAGE()
+ENDIF()
+
+
+# Submit the results to oblivion
+SET( CTEST_DROP_METHOD "http" )
+SET( CTEST_DROP_SITE "oblivion.engr.colostate.edu" )
+SET( CTEST_DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
+SET( CTEST_DROP_SITE_CDASH TRUE )
+SET( DROP_SITE_CDASH TRUE )
+CTEST_SUBMIT()
+
+
+# Clean up
+# exec_program("make distclean")
+
+

doxygen/header.tex

@@ -0,0 +1,56 @@
+\batchmode
+\documentclass[a4paper]{book}
+\usepackage{a4wide}
+\usepackage{makeidx}
+\usepackage{graphicx}
+\usepackage{multicol}
+\usepackage{float}
+\usepackage{listings}
+\usepackage{color}
+\usepackage{textcomp}
+\usepackage{alltt}
+\usepackage{times}
+\usepackage{ifpdf}
+\ifpdf
+\usepackage[pdftex,
+            pagebackref=true,
+            colorlinks=true,
+            linkcolor=blue,
+            unicode
+           ]{hyperref}
+\else
+\usepackage[ps2pdf,
+            pagebackref=true,
+            colorlinks=true,
+            linkcolor=blue,
+            unicode
+           ]{hyperref}
+\usepackage{pspicture}
+\fi
+\usepackage[utf8]{inputenc}
+\usepackage{doxygen}
+\lstset{language=C++,inputencoding=utf8,basicstyle=\footnotesize,breaklines=true,breakatwhitespace=true,tabsize=8,numbers=left }
+\makeindex
+\setcounter{tocdepth}{3}
+\renewcommand{\footrulewidth}{0.4pt}
+\begin{document}
+\hypersetup{pageanchor=false}
+\begin{titlepage}
+\vspace*{7cm}
+\begin{center}
+{\Huge Title}\\
+\vspace*{1cm}
+%%{\large Generated by Doxygen 1.6.3}\\
+\vspace*{0.5cm}
+%%{\small Tue Jan 24 16:37:18 2012}\\
+\end{center}
+\end{titlepage}
+\clearemptydoublepage
+\pagenumbering{roman}
+\tableofcontents
+\clearemptydoublepage
+\pagenumbering{arabic}
+\hypersetup{pageanchor=true}
+
+%% $title, $datetime, $date, $doxygenversion, $projectname, $projectnumber. Do
+

doxygen/html/footer.html

@@ -0,0 +1,23 @@
+<p>
+<hr size=4>
+<table cols=2 width=100% padding=2>
+  <tr>
+    <td align=left>
+    <small>
+        Title<br />
+    </small>
+  </td>
+  <td align=right>
+    <small>
+     This page automatically produced from the <br> source code by 
+     <a href="http://www.stack.nl/~dimitri/doxygen/index.html">
+     <img src="doxygen.png" alt="doxygen" align=center border=0 
+      width=59 height=26></a> <br>
+      Last updated: $datetime.  <br>
+      <a href="mailto:mail@server.org"> <i>Comments on this page</i></a>
+    </small>
+  </td>
+</tr>
+<hr size=4>
+</BODY>
+</HTML>

doxygen/html/header.html

@@ -0,0 +1,18 @@
+<HTML>
+<HEAD>
+   <TITLE> $title </TITLE>
+   <LINK HREF="doxygen.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="tabs.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="draco.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../doxygen.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../tabs.css" REL="stylesheet" TYPE="text/css" >
+   <LINK HREF="../../draco.css" REL="stylesheet" TYPE="text/css" >
+</HEAD>
+<BODY BGCOLOR="#FFFFFF">
+
+<div class="header">
+   <br>
+   On-Line Documentation 
+</div>
+
+

example/Bubble/BlueRidge.pbs

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#PBS -l walltime=05:00:00
+#PBS -l nodes=1:ppn=1:mic
+#PBS -W group_list=arcadm
+#PBS -q normal_q
+#PBS -A blueridgetest
+
+cd $PBS_O_WORKDIR
+
+#mpirun -np 1 ~/save-LBPM-WIA/cpu/ColorLBM-CBUB
+mpirun -np 1 ~/install-LBPM-WIA/tests/TestBubble
+
+exit;

example/Bubble/Color.in

@@ -0,0 +1,6 @@
+1.0
+1.0e-2 0.95 0.8
+0.7
+0.0 0.0 0.0
+0 0 1.0 1.0
+200 1000 1e-5

example/Bubble/Domain.in

@@ -0,0 +1,3 @@
+1 1 1
+80 80 80
+1.0 1.0 1.0

example/Bubble/ExampleOutput.txt

@@ -0,0 +1,5 @@
+8 0.34914 0.33333 0.001388 0.25364 0.33333 0.33333 0.33333 -1.6662e-06 2.1469e-06 -2.3454e-06
+10 0.34583 0.33333 0.0022065 0.20135 0.33333 0.33333 0.33333 -8.5941e-07 1.734e-06 -2.071e-06
+12 0.34377 0.33332 0.0031884 0.1675 0.33333 0.33333 0.33333 -7.089e-07 1.4714e-06 -1.7527e-06
+15 0.3417 0.33332 0.0050284 0.13345 0.33333 0.33333 0.33333 -7.8226e-07 1.4551e-06 -1.6459e-06
+

example/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copy the examples to the install folder
+INSTALL_EXAMPLE (NonNewtonianChannelFlow )
+INSTALL_EXAMPLE( Bubble )
+INSTALL_EXAMPLE( ConstrainedBubble )
+INSTALL_EXAMPLE( Piston )
+INSTALL_EXAMPLE( InkBottle )
+INSTALL_EXAMPLE( Sph1896 )
+INSTALL_EXAMPLE( drainage )
+INSTALL_EXAMPLE( imbibition )
+INSTALL_EXAMPLE( relperm )
+INSTALL_EXAMPLE( Poiseuille )
+INSTALL_EXAMPLE( Juanes )
+INSTALL_EXAMPLE( MicroModel )
+INSTALL_EXAMPLE( Tiff )
+
+
+# Create unit tests for each example
+#   TEST_EXAMPLE( example exe N_procs )
+#TEST_EXAMPLE( Bubble            TestBubble        1  )
+#TEST_EXAMPLE( Sph1896           lb2_Color_wia_mpi 27 )
+#TEST_EXAMPLE( ConstrainedBubble lb2_Color_wia_mpi 1  )
+
+
+
+
+

example/ConstrainedBubble/BlobSize.in

@@ -0,0 +1 @@
+0 

example/ConstrainedBubble/Color.in

@@ -0,0 +1,6 @@
+1.0
+1.0e-2 0.95 0.8
+0.7
+0.0 0.0 0.0
+0 0 1.0 1.0
+20000 1000 1e-5

example/ConstrainedBubble/Domain.in

@@ -0,0 +1,4 @@
+1 1 1
+100 100 100
+229
+1.0 1.0 1.0

example/ConstrainedBubble/Titan-Submit.pbs

@@ -0,0 +1,18 @@
+#!/bin/bash
+#PBS -A GEO019
+#PBS -N ConstrainedBubble
+#PBS -j oe
+#PBS -l walltime=2:00:00,nodes=1
+#PBS -l gres=widow2%widow3
+
+#cd /tmp/work/$USER
+date
+
+cd $PBS_O_WORKDIR
+
+#echo "PBS_O_WORKDIR: `echo $PBS_O_WORKDIR`"
+source $MODULESHOME/init/bash
+module swap cray-mpich2 cray-mpich2/5.6.3
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}
+export MPICH_RDMA_ENABLED_CUDA=1
+aprun -n 1 -N 1 ~/LBPM-WIA/bin/Color-WIA-CBUB