Merge branch 'master' into greyscale_dev

CMakeLists.txt

@@ -150,7 +150,7 @@ IF ( NOT ONLY_BUILD_DOCS )
     CONFIGURE_NETCDF()
     CONFIGURE_SILO()
     CONFIGURE_LBPM()
-    CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
+    CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" FALSE )
     CONFIGURE_LINE_COVERAGE()
     # Set the external library link list
     SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )

IO/IOHelpers.h

@@ -10,9 +10,9 @@ namespace IO {
 // Find a character in a line
 inline size_t find( const char *line, char token )
 {
-    size_t i=0;
+    size_t i = 0;
     while ( 1 ) {
-        if ( line[i]==token || line[i]<32 || line[i]==0 )
+        if ( line[i] == token || line[i] < 32 || line[i] == 0 )
             break;
         ++i;
     }
@@ -21,17 +21,17 @@ inline size_t find( const char *line, char token )
 
 
 // Remove preceeding/trailing whitespace
-inline std::string deblank( const std::string& str )
+inline std::string deblank( const std::string &str )
 {
     size_t i1 = str.size();
     size_t i2 = 0;
-    for (size_t i=0; i<str.size(); i++) {
+    for ( size_t i = 0; i < str.size(); i++ ) {
-        if ( str[i]!=' ' && str[i]>=32 ) {
+        if ( str[i] != ' ' && str[i] >= 32 ) {
-            i1 = std::min(i1,i);
+            i1 = std::min( i1, i );
-            i2 = std::max(i2,i);
+            i2 = std::max( i2, i );
         }
     }
-    return str.substr(i1,i2-i1+1);
+    return str.substr( i1, i2 - i1 + 1 );
 }
 
 
@@ -42,14 +42,14 @@ inline std::vector<std::string> splitList( const char *line, const char token )
     size_t i1 = 0;
     size_t i2 = 0;
     while ( 1 ) {
-        if ( line[i2]==token || line[i2]<32 ) {
+        if ( line[i2] == token || line[i2] < 32 ) {
-            std::string tmp(&line[i1],i2-i1);
+            std::string tmp( &line[i1], i2 - i1 );
-            tmp = deblank(tmp);
+            tmp = deblank( tmp );
             if ( !tmp.empty() )
-                list.push_back(tmp);
+                list.push_back( tmp );
-            i1 = i2+1;
+            i1 = i2 + 1;
         }
-        if ( line[i2]==0 )
+        if ( line[i2] == 0 )
             break;
         i2++;
     }
@@ -57,8 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
 }
 
 
-
+}; // namespace IO
-};
 
 #endif
-

IO/Mesh.cpp

@@ -1,4 +1,5 @@
 #include "Mesh.h"
+#include "IO/IOHelpers.h"
 #include "common/Utilities.h"
 
 #include <limits>
@@ -19,104 +20,110 @@ inline Point nullPoint()
 
 
 /****************************************************
-* Mesh                                              *
+ * Mesh                                              *
-****************************************************/
+ ****************************************************/
-Mesh::Mesh( )
+Mesh::Mesh() {}
-{
+Mesh::~Mesh() {}
-}
-Mesh::~Mesh( )
-{
-}
 
 
 /****************************************************
-* MeshDataStruct                                    *
+ * MeshDataStruct                                    *
-****************************************************/
+ ****************************************************/
-bool MeshDataStruct::check() const
+#define checkResult( pass, msg ) \
+    do {                         \
+        if ( !( pass ) ) {       \
+            if ( abort )         \
+                ERROR( msg );    \
+            return false;        \
+        }                        \
+    } while ( 0 )
+bool MeshDataStruct::check( bool abort ) const
 {
-    enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
+    for ( const auto &var : vars ) {
-    bool pass = mesh != nullptr;
+        checkResult( var->type == VariableType::NodeVariable ||
-    for ( const auto& var : vars ) {
+                         var->type == VariableType::EdgeVariable ||
-        pass = pass && static_cast<int>(var->type)>=1 && static_cast<int>(var->type)<=3;
+                         var->type == VariableType::SurfaceVariable ||
-        pass = pass && !var->data.empty();
+                         var->type == VariableType::VolumeVariable,
+            "Invalid data type" );
+        checkResult( !var->data.empty(), "Variable data is empty" );
     }
-    if ( !pass )
+    const std::string &meshClass = mesh->className();
-        return false;
-    const std::string& meshClass = mesh->className();
     if ( meshClass == "PointList" ) {
-        const auto mesh2 = dynamic_cast<IO::PointList*>( mesh.get() );
+        auto mesh2 = dynamic_cast<IO::PointList *>( mesh.get() );
-        if ( mesh2 == nullptr )
+        ASSERT( mesh2 );
-            return false;
+        for ( const auto &var : vars ) {
-        for ( const auto& var : vars ) {
             if ( var->type == IO::VariableType::NodeVariable ) {
-                pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim;
+                size_t N_points = mesh2->points.size();
+                checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
+                checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
             } else if ( var->type == IO::VariableType::EdgeVariable ) {
-                ERROR("Invalid type for PointList");
+                ERROR( "Invalid type for PointList" );
             } else if ( var->type == IO::VariableType::SurfaceVariable ) {
-                ERROR("Invalid type for PointList");
+                ERROR( "Invalid type for PointList" );
             } else if ( var->type == IO::VariableType::VolumeVariable ) {
-                ERROR("Invalid type for PointList");
+                ERROR( "Invalid type for PointList" );
             } else {
-                ERROR("Invalid variable type");
+                ERROR( "Invalid variable type" );
             }
         }
     } else if ( meshClass == "TriMesh" || meshClass == "TriList" ) {
-        const auto mesh2 = getTriMesh( mesh );
+        auto mesh2 = getTriMesh( mesh );
-        if ( mesh2 == nullptr )
+        ASSERT( mesh2 );
-            return false;
+        for ( const auto &var : vars ) {
-        for ( const auto& var : vars ) {
             if ( var->type == IO::VariableType::NodeVariable ) {
-                pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim;
+                size_t N_points = mesh2->vertices->points.size();
+                checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
+                checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
             } else if ( var->type == IO::VariableType::EdgeVariable ) {
-                ERROR("Not finished");
+                ERROR( "Not finished" );
             } else if ( var->type == IO::VariableType::SurfaceVariable ) {
-                ERROR("Not finished");
+                ERROR( "Not finished" );
             } else if ( var->type == IO::VariableType::VolumeVariable ) {
-                pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim;
+                checkResult( var->data.size( 0 ) == mesh2->A.size(), "sizeof VolumeVariable" );
+                checkResult( var->data.size( 1 ) == var->dim, "sizeof VolumeVariable" );
             } else {
-                ERROR("Invalid variable type");
+                ERROR( "Invalid variable type" );
             }
         }
     } else if ( meshClass == "DomainMesh" ) {
-        const auto mesh2 = dynamic_cast<IO::DomainMesh*>( mesh.get() );
+        auto mesh2 = dynamic_cast<IO::DomainMesh *>( mesh.get() );
-        if ( mesh2 == nullptr )
+        ASSERT( mesh2 );
-            return false;
+        for ( const auto &var : vars ) {
-        for ( const auto& var : vars ) {
+            ArraySize varSize;
             if ( var->type == IO::VariableType::NodeVariable ) {
-                pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1)
+                varSize = ArraySize( mesh2->nx + 1, mesh2->ny + 1, mesh2->nz + 1, var->dim );
-                            && (int) var->data.size(2)==(mesh2->nz+1) &&       var->data.size(3)==var->dim;
             } else if ( var->type == IO::VariableType::EdgeVariable ) {
-                ERROR("Not finished");
+                ERROR( "Not finished" );
             } else if ( var->type == IO::VariableType::SurfaceVariable ) {
-                ERROR("Not finished");
+                ERROR( "Not finished" );
             } else if ( var->type == IO::VariableType::VolumeVariable ) {
-                pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny
+                varSize = ArraySize( mesh2->nx, mesh2->ny, mesh2->nz, var->dim );
-                            && (int) var->data.size(2)==mesh2->nz &&       var->data.size(3)==var->dim;
             } else {
-                ERROR("Invalid variable type");
+                ERROR( "Invalid variable type" );
             }
+            if ( var->data.size( 0 ) == varSize[0] * varSize[1] * varSize[2] &&
+                 var->data.size( 1 ) == varSize[3] )
+                var->data.resize( varSize );
+            for ( int d = 0; d < 4; d++ )
+                checkResult( var->data.size( d ) == varSize[d], "DomainMesh Variable" );
         }
     } else {
-        ERROR("Unknown mesh class: "+mesh->className());
+        ERROR( "Unknown mesh class: " + mesh->className() );
     }
-    return pass;
+    return true;
 }
 
 
 /****************************************************
-* PointList                                         *
+ * PointList                                         *
-****************************************************/
+ ****************************************************/
-PointList::PointList( )
+PointList::PointList() {}
-{
-}
 PointList::PointList( size_t N )
 {
     Point tmp = nullPoint();
-    points.resize(N,tmp);
+    points.resize( N, tmp );
-}
-PointList::~PointList( )
-{
 }
+PointList::~PointList() {}
 size_t PointList::numberPointsVar( VariableType type ) const
 {
     size_t N = 0;
@@ -124,174 +131,168 @@ size_t PointList::numberPointsVar( VariableType type ) const
         N = points.size();
     return N;
 }
-std::pair<size_t,void*> PointList::pack( int level ) const
+std::pair<size_t, void *> PointList::pack( int level ) const
 {
-    std::pair<size_t,void*> data_out(0,NULL);
+    std::pair<size_t, void *> data_out( 0, NULL );
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        data_out.first = (2+3*points.size())*sizeof(double);
+        data_out.first     = ( 2 + 3 * points.size() ) * sizeof( double );
-        double *data_ptr = new double[2+3*points.size()];
+        double *data_ptr   = new double[2 + 3 * points.size()];
-        data_out.second = data_ptr;
+        data_out.second    = data_ptr;
-        uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
+        uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
-        data_int[0] = level;
+        data_int[0]        = level;
-        data_int[1] = points.size();
+        data_int[1]        = points.size();
-        double *data = &data_ptr[2];
+        double *data       = &data_ptr[2];
-        for (size_t i=0; i<points.size(); i++) {
+        for ( size_t i = 0; i < points.size(); i++ ) {
-            data[3*i+0] = points[i].x;
+            data[3 * i + 0] = points[i].x;
-            data[3*i+1] = points[i].y;
+            data[3 * i + 1] = points[i].y;
-            data[3*i+2] = points[i].z;
+            data[3 * i + 2] = points[i].z;
         }
     }
     return data_out;
 }
-void PointList::unpack( const std::pair<size_t,void*>& data_in )
+void PointList::unpack( const std::pair<size_t, void *> &data_in )
 {
-    uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
+    uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
-    const double *data = reinterpret_cast<const double*>(data_in.second);
+    const double *data = reinterpret_cast<const double *>( data_in.second );
-    int level = data_int[0];
+    int level          = data_int[0];
-    uint64_t N = data_int[1];
+    uint64_t N         = data_int[1];
-    data = &data[2];
+    data               = &data[2];
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        ASSERT((2+3*N)*sizeof(double)==data_in.first);
+        ASSERT( ( 2 + 3 * N ) * sizeof( double ) == data_in.first );
-        points.resize(N);
+        points.resize( N );
-        for (size_t i=0; i<points.size(); i++) {
+        for ( size_t i = 0; i < points.size(); i++ ) {
-            points[i].x = data[3*i+0];
+            points[i].x = data[3 * i + 0];
-            points[i].y = data[3*i+1];
+            points[i].y = data[3 * i + 1];
-            points[i].z = data[3*i+2];
+            points[i].z = data[3 * i + 2];
         }
     }
 }
 
 
 /****************************************************
-* TriList                                           *
+ * TriList                                           *
-****************************************************/
+ ****************************************************/
-TriList::TriList( )
+TriList::TriList() {}
-{
-}
 TriList::TriList( size_t N_tri )
 {
     Point tmp = nullPoint();
-    A.resize(N_tri,tmp);
+    A.resize( N_tri, tmp );
-    B.resize(N_tri,tmp);
+    B.resize( N_tri, tmp );
-    C.resize(N_tri,tmp);
+    C.resize( N_tri, tmp );
 }
-TriList::TriList( const TriMesh& mesh )
+TriList::TriList( const TriMesh &mesh )
 {
     Point tmp = nullPoint();
-    A.resize(mesh.A.size(),tmp);
+    A.resize( mesh.A.size(), tmp );
-    B.resize(mesh.B.size(),tmp);
+    B.resize( mesh.B.size(), tmp );
-    C.resize(mesh.C.size(),tmp);
+    C.resize( mesh.C.size(), tmp );
-    ASSERT(mesh.vertices.get()!=NULL);
+    ASSERT( mesh.vertices.get() != NULL );
-    const std::vector<Point>& P = mesh.vertices->points;
+    const std::vector<Point> &P = mesh.vertices->points;
-    for (size_t i=0; i<A.size(); i++)
+    for ( size_t i = 0; i < A.size(); i++ )
         A[i] = P[mesh.A[i]];
-    for (size_t i=0; i<B.size(); i++)
+    for ( size_t i = 0; i < B.size(); i++ )
         B[i] = P[mesh.B[i]];
-    for (size_t i=0; i<C.size(); i++)
+    for ( size_t i = 0; i < C.size(); i++ )
         C[i] = P[mesh.C[i]];
 }
-TriList::~TriList( )
+TriList::~TriList() {}
-{
-}
 size_t TriList::numberPointsVar( VariableType type ) const
 {
     size_t N = 0;
-    if ( type==VariableType::NodeVariable )
+    if ( type == VariableType::NodeVariable )
-        N = 3*A.size();
+        N = 3 * A.size();
-    else if ( type==VariableType::SurfaceVariable ||  type==VariableType::VolumeVariable )
+    else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
         N = A.size();
     return N;
 }
-std::pair<size_t,void*> TriList::pack( int level ) const
+std::pair<size_t, void *> TriList::pack( int level ) const
 {
-    std::pair<size_t,void*> data_out(0,NULL);
+    std::pair<size_t, void *> data_out( 0, NULL );
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        data_out.first = (2+9*A.size())*sizeof(double);
+        data_out.first     = ( 2 + 9 * A.size() ) * sizeof( double );
-        double *data_ptr = new double[2+9*A.size()];
+        double *data_ptr   = new double[2 + 9 * A.size()];
-        data_out.second = data_ptr;
+        data_out.second    = data_ptr;
-        uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
+        uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
-        data_int[0] = level;
+        data_int[0]        = level;
-        data_int[1] = A.size();
+        data_int[1]        = A.size();
-        double *data = &data_ptr[2];
+        double *data       = &data_ptr[2];
-        for (size_t i=0; i<A.size(); i++) {
+        for ( size_t i = 0; i < A.size(); i++ ) {
-            data[9*i+0] = A[i].x;
+            data[9 * i + 0] = A[i].x;
-            data[9*i+1] = A[i].y;
+            data[9 * i + 1] = A[i].y;
-            data[9*i+2] = A[i].z;
+            data[9 * i + 2] = A[i].z;
-            data[9*i+3] = B[i].x;
+            data[9 * i + 3] = B[i].x;
-            data[9*i+4] = B[i].y;
+            data[9 * i + 4] = B[i].y;
-            data[9*i+5] = B[i].z;
+            data[9 * i + 5] = B[i].z;
-            data[9*i+6] = C[i].x;
+            data[9 * i + 6] = C[i].x;
-            data[9*i+7] = C[i].y;
+            data[9 * i + 7] = C[i].y;
-            data[9*i+8] = C[i].z;
+            data[9 * i + 8] = C[i].z;
         }
     }
     return data_out;
 }
-void TriList::unpack( const std::pair<size_t,void*>& data_in )
+void TriList::unpack( const std::pair<size_t, void *> &data_in )
 {
-    uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
+    uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
-    const double *data = reinterpret_cast<const double*>(data_in.second);
+    const double *data = reinterpret_cast<const double *>( data_in.second );
-    int level = data_int[0];
+    int level          = data_int[0];
-    uint64_t N = data_int[1];
+    uint64_t N         = data_int[1];
-    data = &data[2];
+    data               = &data[2];
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        ASSERT((2+9*N)*sizeof(double)==data_in.first);
+        ASSERT( ( 2 + 9 * N ) * sizeof( double ) == data_in.first );
-        A.resize(N);
+        A.resize( N );
-        B.resize(N);
+        B.resize( N );
-        C.resize(N);
+        C.resize( N );
-        for (size_t i=0; i<A.size(); i++) {
+        for ( size_t i = 0; i < A.size(); i++ ) {
-            A[i].x = data[9*i+0];
+            A[i].x = data[9 * i + 0];
-            A[i].y = data[9*i+1];
+            A[i].y = data[9 * i + 1];
-            A[i].z = data[9*i+2];
+            A[i].z = data[9 * i + 2];
-            B[i].x = data[9*i+3];
+            B[i].x = data[9 * i + 3];
-            B[i].y = data[9*i+4];
+            B[i].y = data[9 * i + 4];
-            B[i].z = data[9*i+5];
+            B[i].z = data[9 * i + 5];
-            C[i].x = data[9*i+6];
+            C[i].x = data[9 * i + 6];
-            C[i].y = data[9*i+7];
+            C[i].y = data[9 * i + 7];
-            C[i].z = data[9*i+8];
+            C[i].z = data[9 * i + 8];
         }
     }
 }
 
 
 /****************************************************
-* TriMesh                                           *
+ * TriMesh                                           *
-****************************************************/
+ ****************************************************/
-TriMesh::TriMesh(  )
+TriMesh::TriMesh() {}
-{
-}
 TriMesh::TriMesh( size_t N_tri, size_t N_point )
 {
-    vertices.reset( new PointList(N_point) );
+    vertices.reset( new PointList( N_point ) );
-    A.resize(N_tri,-1);
+    A.resize( N_tri, -1 );
-    B.resize(N_tri,-1);
+    B.resize( N_tri, -1 );
-    C.resize(N_tri,-1);
+    C.resize( N_tri, -1 );
 }
 TriMesh::TriMesh( size_t N_tri, std::shared_ptr<PointList> points )
 {
     vertices = points;
-    A.resize(N_tri,-1);
+    A.resize( N_tri, -1 );
-    B.resize(N_tri,-1);
+    B.resize( N_tri, -1 );
-    C.resize(N_tri,-1);
+    C.resize( N_tri, -1 );
 }
-TriMesh::TriMesh( const TriList& mesh )
+TriMesh::TriMesh( const TriList &mesh )
 {
     // For simlicity we will just create a mesh with ~3x the verticies for now
-    ASSERT(mesh.A.size()==mesh.B.size()&&mesh.A.size()==mesh.C.size());
+    ASSERT( mesh.A.size() == mesh.B.size() && mesh.A.size() == mesh.C.size() );
-    A.resize(mesh.A.size());
+    A.resize( mesh.A.size() );
-    B.resize(mesh.B.size());
+    B.resize( mesh.B.size() );
-    C.resize(mesh.C.size());
+    C.resize( mesh.C.size() );
-    vertices.reset( new PointList(3*mesh.A.size()) );
+    vertices.reset( new PointList( 3 * mesh.A.size() ) );
-    for (size_t i=0; i<A.size(); i++) {
+    for ( size_t i = 0; i < A.size(); i++ ) {
-        A[i] = 3*i+0;
+        A[i]                   = 3 * i + 0;
-        B[i] = 3*i+1;
+        B[i]                   = 3 * i + 1;
-        C[i] = 3*i+2;
+        C[i]                   = 3 * i + 2;
         vertices->points[A[i]] = mesh.A[i];
         vertices->points[B[i]] = mesh.B[i];
         vertices->points[C[i]] = mesh.C[i];
     }
 }
-TriMesh::~TriMesh( )
+TriMesh::~TriMesh()
 {
     vertices.reset();
     A.clear();
@@ -301,181 +302,319 @@ TriMesh::~TriMesh( )
 size_t TriMesh::numberPointsVar( VariableType type ) const
 {
     size_t N = 0;
-    if ( type==VariableType::NodeVariable )
+    if ( type == VariableType::NodeVariable )
         N = vertices->points.size();
-    else if ( type==VariableType::SurfaceVariable ||  type==VariableType::VolumeVariable )
+    else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
         N = A.size();
     return N;
 }
-std::pair<size_t,void*> TriMesh::pack( int level ) const
+std::pair<size_t, void *> TriMesh::pack( int level ) const
 {
-    std::pair<size_t,void*> data_out(0,NULL);
+    std::pair<size_t, void *> data_out( 0, NULL );
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        const std::vector<Point>& points = vertices->points;
+        const std::vector<Point> &points = vertices->points;
-        data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int);
+        data_out.first =
-        double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)];
+            ( 3 + 3 * points.size() ) * sizeof( double ) + 3 * A.size() * sizeof( int );
-        data_out.second = data_ptr;
+        double *data_ptr =
-        uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_ptr);
+            new double[4 + 3 * points.size() + ( 3 * A.size() * sizeof( int ) ) / sizeof( double )];
-        data_int64[0] = level;
+        data_out.second      = data_ptr;
-        data_int64[1] = points.size();
+        uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_ptr );
-        data_int64[2] = A.size();
+        data_int64[0]        = level;
-        double *data = &data_ptr[3];
+        data_int64[1]        = points.size();
-        for (size_t i=0; i<points.size(); i++) {
+        data_int64[2]        = A.size();
-            data[3*i+0] = points[i].x;
+        double *data         = &data_ptr[3];
-            data[3*i+1] = points[i].y;
+        for ( size_t i = 0; i < points.size(); i++ ) {
-            data[3*i+2] = points[i].z;
+            data[3 * i + 0] = points[i].x;
+            data[3 * i + 1] = points[i].y;
+            data[3 * i + 2] = points[i].z;
         }
-        int *data_int = reinterpret_cast<int*>(&data[3*points.size()]);
+        int *data_int = reinterpret_cast<int *>( &data[3 * points.size()] );
-        for (size_t i=0; i<A.size(); i++) {
+        for ( size_t i = 0; i < A.size(); i++ ) {
-            data_int[3*i+0] = A[i];
+            data_int[3 * i + 0] = A[i];
-            data_int[3*i+1] = B[i];
+            data_int[3 * i + 1] = B[i];
-            data_int[3*i+2] = C[i];
+            data_int[3 * i + 2] = C[i];
         }
     }
     return data_out;
 }
-void TriMesh::unpack( const std::pair<size_t,void*>& data_in )
+void TriMesh::unpack( const std::pair<size_t, void *> &data_in )
 {
-    uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_in.second);
+    uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_in.second );
-    const double *data = reinterpret_cast<const double*>(data_in.second);
+    const double *data   = reinterpret_cast<const double *>( data_in.second );
-    int level = data_int64[0];
+    int level            = data_int64[0];
-    uint64_t N_P = data_int64[1];
+    uint64_t N_P         = data_int64[1];
-    uint64_t N_A = data_int64[2];
+    uint64_t N_A         = data_int64[2];
-    data = &data[3];
+    data                 = &data[3];
-    if ( level==0 ) {
+    if ( level == 0 ) {
-        size_t size = (3+3*N_P)*sizeof(double)+3*N_A*sizeof(int);
+        size_t size = ( 3 + 3 * N_P ) * sizeof( double ) + 3 * N_A * sizeof( int );
-        ASSERT(size==data_in.first);
+        ASSERT( size == data_in.first );
-        vertices.reset( new PointList(N_P) );
+        vertices.reset( new PointList( N_P ) );
-        std::vector<Point>& points = vertices->points;
+        std::vector<Point> &points = vertices->points;
-        for (size_t i=0; i<points.size(); i++) {
+        for ( size_t i = 0; i < points.size(); i++ ) {
-            points[i].x = data[3*i+0];
+            points[i].x = data[3 * i + 0];
-            points[i].y = data[3*i+1];
+            points[i].y = data[3 * i + 1];
-            points[i].z = data[3*i+2];
+            points[i].z = data[3 * i + 2];
         }
-        const int *data_int = reinterpret_cast<const int*>(&data[3*N_P]);
+        const int *data_int = reinterpret_cast<const int *>( &data[3 * N_P] );
-        A.resize(N_A);
+        A.resize( N_A );
-        B.resize(N_A);
+        B.resize( N_A );
-        C.resize(N_A);
+        C.resize( N_A );
-        for (size_t i=0; i<A.size(); i++) {
+        for ( size_t i = 0; i < A.size(); i++ ) {
-            A[i] = data_int[3*i+0];
+            A[i] = data_int[3 * i + 0];
-            B[i] = data_int[3*i+1];
+            B[i] = data_int[3 * i + 1];
-            C[i] = data_int[3*i+2];
+            C[i] = data_int[3 * i + 2];
         }
     }
 }
 
 
 /****************************************************
-* Domain mesh                                       *
+ * Domain mesh                                       *
-****************************************************/
+ ****************************************************/
-DomainMesh::DomainMesh():
+DomainMesh::DomainMesh()
-    nprocx(0), nprocy(0), nprocz(0), rank(0),
+    : nprocx( 0 ),
-    nx(0), ny(0), nz(0),
+      nprocy( 0 ),
-    Lx(0), Ly(0), Lz(0)
+      nprocz( 0 ),
+      rank( 0 ),
+      nx( 0 ),
+      ny( 0 ),
+      nz( 0 ),
+      Lx( 0 ),
+      Ly( 0 ),
+      Lz( 0 )
 {
 }
-DomainMesh::DomainMesh( RankInfoStruct data, 
+DomainMesh::DomainMesh(
-    int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ):
+    RankInfoStruct data, int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 )
-    nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]),
+    : nprocx( data.nx ),
-    nx(nx2), ny(ny2), nz(nz2),
+      nprocy( data.ny ),
-    Lx(Lx2), Ly(Ly2), Lz(Lz2)
+      nprocz( data.nz ),
-{
+      rank( data.rank[1][1][1] ),
-}
+      nx( nx2 ),
-DomainMesh::~DomainMesh()
+      ny( ny2 ),
+      nz( nz2 ),
+      Lx( Lx2 ),
+      Ly( Ly2 ),
+      Lz( Lz2 )
 {
 }
+DomainMesh::~DomainMesh() {}
 size_t DomainMesh::numberPointsVar( VariableType type ) const
 {
     size_t N = 0;
-    if ( type==VariableType::NodeVariable )
+    if ( type == VariableType::NodeVariable )
-        N = (nx+1)*(ny+1)*(nz+1);
+        N = ( nx + 1 ) * ( ny + 1 ) * ( nz + 1 );
-    else if ( type==VariableType::SurfaceVariable )
+    else if ( type == VariableType::SurfaceVariable )
-        N = (nx+1)*ny*nz + nx*(ny+1)*nz + nx*ny*(nz+1);
+        N = ( nx + 1 ) * ny * nz + nx * ( ny + 1 ) * nz + nx * ny * ( nz + 1 );
-    else if ( type==VariableType::VolumeVariable )
+    else if ( type == VariableType::VolumeVariable )
-        N = nx*ny*nz;
+        N = nx * ny * nz;
     return N;
 }
-std::pair<size_t,void*> DomainMesh::pack( int level ) const
+std::pair<size_t, void *> DomainMesh::pack( int level ) const
 {
-    std::pair<size_t,void*> data(0,NULL);
+    std::pair<size_t, void *> data( 0, NULL );
-    data.first = 7*sizeof(double);
+    data.first  = 7 * sizeof( double );
     data.second = new double[7];
-    memset(data.second,0,7*sizeof(double));
+    memset( data.second, 0, 7 * sizeof( double ) );
-    int *data_int = reinterpret_cast<int*>(data.second);
+    int *data_int       = reinterpret_cast<int *>( data.second );
-    double *data_double = &reinterpret_cast<double*>(data.second)[4];
+    double *data_double = &reinterpret_cast<double *>( data.second )[4];
-    data_int[0] = nprocx;
+    data_int[0]         = nprocx;
-    data_int[1] = nprocy;
+    data_int[1]         = nprocy;
-    data_int[2] = nprocz;
+    data_int[2]         = nprocz;
-    data_int[3] = rank;
+    data_int[3]         = rank;
-    data_int[4] = nx;
+    data_int[4]         = nx;
-    data_int[5] = ny;
+    data_int[5]         = ny;
-    data_int[6] = nz;
+    data_int[6]         = nz;
-    data_double[0] = Lx;
+    data_double[0]      = Lx;
-    data_double[1] = Ly;
+    data_double[1]      = Ly;
-    data_double[2] = Lz;
+    data_double[2]      = Lz;
     return data;
 }
-void DomainMesh::unpack( const std::pair<size_t,void*>& data )
+void DomainMesh::unpack( const std::pair<size_t, void *> &data )
 {
-    const int *data_int = reinterpret_cast<const int*>(data.second);
+    const int *data_int       = reinterpret_cast<const int *>( data.second );
-    const double *data_double = &reinterpret_cast<const double*>(data.second)[4];
+    const double *data_double = &reinterpret_cast<const double *>( data.second )[4];
-    nprocx = data_int[0];
+    nprocx                    = data_int[0];
-    nprocy = data_int[1];
+    nprocy                    = data_int[1];
-    nprocz = data_int[2];
+    nprocz                    = data_int[2];
-    rank = data_int[3];
+    rank                      = data_int[3];
-    nx = data_int[4];
+    nx                        = data_int[4];
-    ny = data_int[5];
+    ny                        = data_int[5];
-    nz = data_int[6];
+    nz                        = data_int[6];
-    Lx = data_double[0];
+    Lx                        = data_double[0];
-    Ly = data_double[1];
+    Ly                        = data_double[1];
-    Lz = data_double[2];
+    Lz                        = data_double[2];
 }
 
 
 /****************************************************
-* Converters                                        *
+ * Converters                                        *
-****************************************************/
+ ****************************************************/
 std::shared_ptr<PointList> getPointList( std::shared_ptr<Mesh> mesh )
 {
-    return std::dynamic_pointer_cast<PointList>(mesh);
+    return std::dynamic_pointer_cast<PointList>( mesh );
 }
 std::shared_ptr<TriMesh> getTriMesh( std::shared_ptr<Mesh> mesh )
 {
     std::shared_ptr<TriMesh> mesh2;
-    if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
+    if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
-        mesh2 = std::dynamic_pointer_cast<TriMesh>(mesh);
+        mesh2 = std::dynamic_pointer_cast<TriMesh>( mesh );
-    } else if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
+    } else if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
-        std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>(mesh);
+        std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>( mesh );
-        ASSERT(trilist.get()!=NULL);
+        ASSERT( trilist.get() != NULL );
-        mesh2.reset( new TriMesh(*trilist) );
+        mesh2.reset( new TriMesh( *trilist ) );
     }
     return mesh2;
 }
 std::shared_ptr<TriList> getTriList( std::shared_ptr<Mesh> mesh )
 {
     std::shared_ptr<TriList> mesh2;
-    if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
+    if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
-        mesh2 = std::dynamic_pointer_cast<TriList>(mesh);
+        mesh2 = std::dynamic_pointer_cast<TriList>( mesh );
-    } else if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
+    } else if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
-        std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>(mesh);
+        std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>( mesh );
-        ASSERT(trimesh.get()!=NULL);
+        ASSERT( trimesh.get() != NULL );
-        mesh2.reset( new TriList(*trimesh) );
+        mesh2.reset( new TriList( *trimesh ) );
     }
     return mesh2;
 }
 std::shared_ptr<const PointList> getPointList( std::shared_ptr<const Mesh> mesh )
 {
-    return getPointList( std::const_pointer_cast<Mesh>(mesh) );
+    return getPointList( std::const_pointer_cast<Mesh>( mesh ) );
 }
 std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh )
 {
-    return getTriMesh( std::const_pointer_cast<Mesh>(mesh) );
+    return getTriMesh( std::const_pointer_cast<Mesh>( mesh ) );
 }
 std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh )
 {
-    return getTriList( std::const_pointer_cast<Mesh>(mesh) );
+    return getTriList( std::const_pointer_cast<Mesh>( mesh ) );
 }
 
 
-} // IO namespace
+/****************************************************
+ * Convert enum values                               *
+ ****************************************************/
+std::string getString( VariableType type )
+{
+    if ( type == VariableType::NodeVariable )
+        return "node";
+    else if ( type == VariableType::EdgeVariable )
+        return "edge";
+    else if ( type == VariableType::SurfaceVariable )
+        return "face";
+    else if ( type == VariableType::VolumeVariable )
+        return "cell";
+    else if ( type == VariableType::NullVariable )
+        return "null";
+    else
+        ERROR( "Invalid type" );
+    return "";
+}
+VariableType getVariableType( const std::string &type_in )
+{
+    auto type = deblank( type_in );
+    if ( type == "node" )
+        return VariableType::NodeVariable;
+    else if ( type == "edge" || type == "1" )
+        return VariableType::EdgeVariable;
+    else if ( type == "face" )
+        return VariableType::SurfaceVariable;
+    else if ( type == "cell" || type == "3" )
+        return VariableType::VolumeVariable;
+    else if ( type == "null" )
+        return VariableType::NullVariable;
+    else
+        ERROR( "Invalid type: " + type );
+    return VariableType::NullVariable;
+}
+std::string getString( DataType type )
+{
+    if ( type == DataType::Double )
+        return "double";
+    else if ( type == DataType::Float )
+        return "float";
+    else if ( type == DataType::Int )
+        return "int";
+    else if ( type == DataType::Null )
+        return "null";
+    else
+        ERROR( "Invalid type" );
+    return "";
+}
+DataType getDataType( const std::string &type_in )
+{
+    auto type = deblank( type_in );
+    if ( type == "double" )
+        return DataType::Double;
+    else if ( type == "float" )
+        return DataType::Float;
+    else if ( type == "int" )
+        return DataType::Int;
+    else if ( type == "null" )
+        return DataType::Null;
+    else
+        ERROR( "Invalid type: " + type );
+    return DataType::Null;
+}
+std::string getString( MeshType type )
+{
+    if ( type == MeshType::PointMesh )
+        return "PointMesh";
+    else if ( type == MeshType::SurfaceMesh )
+        return "SurfaceMesh";
+    else if ( type == MeshType::VolumeMesh )
+        return "VolumeMesh";
+    else if ( type == MeshType::Unknown )
+        return "unknown";
+    else
+        ERROR( "Invalid type" );
+    return "";
+}
+MeshType getMeshType( const std::string &type_in )
+{
+    auto type = deblank( type_in );
+    if ( type == "PointMesh" || type == "1" )
+        return MeshType::PointMesh;
+    else if ( type == "SurfaceMesh" || type == "2" )
+        return MeshType::SurfaceMesh;
+    else if ( type == "VolumeMesh" || type == "3" )
+        return MeshType::VolumeMesh;
+    else if ( type == "unknown" || type == "-1" )
+        return MeshType::Unknown;
+    else
+        ERROR( "Invalid type: " + type );
+    return MeshType::Unknown;
+}
+std::string getString( FileFormat type )
+{
+    if ( type == FileFormat::OLD )
+        return "old";
+    else if ( type == FileFormat::NEW )
+        return "new";
+    else if ( type == FileFormat::NEW_SINGLE )
+        return "new(single)";
+    else if ( type == FileFormat::SILO )
+        return "silo";
+    else
+        ERROR( "Invalid type" );
+    return "";
+}
+FileFormat getFileFormat( const std::string &type_in )
+{
+    auto type = deblank( type_in );
+    if ( type == "old" || type == "1" )
+        return FileFormat::OLD;
+    else if ( type == "new" || type == "2" )
+        return FileFormat::NEW;
+    else if ( type == "new(single)" || type == "3" )
+        return FileFormat::NEW_SINGLE;
+    else if ( type == "silo" || type == "4" )
+        return FileFormat::SILO;
+    else
+        ERROR( "Invalid type: " + type );
+    return FileFormat::SILO;
+}
 
+
+} // namespace IO

IO/Mesh.h

@@ -6,17 +6,36 @@
 #include <string.h>
 #include <vector>
 
+#include "analysis/PointList.h"
 #include "common/Array.h"
 #include "common/Communication.h"
-#include "analysis/PointList.h"
 
 
 namespace IO {
 
 
-//! Possible variable types
+//! Enums to define types
-enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
+enum class VariableType {
-enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 };
+    NodeVariable,
+    EdgeVariable,
+    SurfaceVariable,
+    VolumeVariable,
+    NullVariable
+};
+enum class DataType { Double, Float, Int, Null };
+enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown };
+enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO };
+
+
+//! Convert enums to/from strings (more future-proof than static_cast<int>)
+std::string getString( VariableType );
+std::string getString( DataType );
+std::string getString( MeshType );
+std::string getString( FileFormat );
+VariableType getVariableType( const std::string & );
+DataType getDataType( const std::string & );
+MeshType getMeshType( const std::string & );
+FileFormat getFileFormat( const std::string & );
 
 
 /*! \class Mesh
@@ -32,21 +51,22 @@ public:
     //! Number of points for the given variable type
     virtual size_t numberPointsVar( VariableType type ) const = 0;
     //! Pack the data
-    virtual std::pair<size_t,void*> pack( int level ) const = 0;
+    virtual std::pair<size_t, void *> pack( int level ) const = 0;
     //! Unpack the data
-    virtual void unpack( const std::pair<size_t,void*>& data ) = 0;
+    virtual void unpack( const std::pair<size_t, void *> &data ) = 0;
+
 protected:
     //! Empty constructor
     Mesh();
-    Mesh(const Mesh&);
+    Mesh( const Mesh & );
-    Mesh& operator=(const Mesh&);
+    Mesh &operator=( const Mesh & );
 };
 
 
 /*! \class PointList
     \brief A class used to hold a list of verticies
 */
-class PointList: public Mesh
+class PointList : public Mesh
 {
 public:
     //! Empty constructor
@@ -60,13 +80,14 @@ public:
     //! Number of points for the given variable type
     virtual size_t numberPointsVar( VariableType type ) const;
     //! Pack the data
-    virtual std::pair<size_t,void*> pack( int level ) const;
+    virtual std::pair<size_t, void *> pack( int level ) const;
     //! Unpack the data
-    virtual void unpack( const std::pair<size_t,void*>& data );
+    virtual void unpack( const std::pair<size_t, void *> &data );
     //! Access the points
-    const std::vector<Point>& getPoints() const { return points; }
+    const std::vector<Point> &getPoints() const { return points; }
+
 public:
-    std::vector<Point>  points;  //!< List of points vertex
+    std::vector<Point> points; //!< List of points vertex
 };
 
 
@@ -74,7 +95,7 @@ public:
     \brief A class used to hold a list of triangles specified by their vertex coordinates
 */
 class TriMesh;
-class TriList: public Mesh
+class TriList : public Mesh
 {
 public:
     //! Empty constructor
@@ -82,7 +103,7 @@ public:
     //! Constructor for N triangles
     TriList( size_t N_tri );
     //! Constructor from TriMesh
-    TriList( const TriMesh& );
+    TriList( const TriMesh & );
     //! Destructor
     virtual ~TriList();
     //! Mesh class name
@@ -90,20 +111,22 @@ public:
     //! Number of points for the given variable type
     virtual size_t numberPointsVar( VariableType type ) const;
     //! Pack the data
-    virtual std::pair<size_t,void*> pack( int level ) const;
+    virtual std::pair<size_t, void *> pack( int level ) const;
     //! Unpack the data
-    virtual void unpack( const std::pair<size_t,void*>& data );
+    virtual void unpack( const std::pair<size_t, void *> &data );
+
 public:
-    std::vector<Point>  A;      //!< First vertex
+    std::vector<Point> A; //!< First vertex
-    std::vector<Point>  B;      //!< Second vertex
+    std::vector<Point> B; //!< Second vertex
-    std::vector<Point>  C;      //!< Third vertex
+    std::vector<Point> C; //!< Third vertex
 };
 
 
 /*! \class TriMesh
-    \brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes
+    \brief A class used to hold a list of trianges specified by their vertex number and list of
+   coordiantes
 */
-class TriMesh: public Mesh
+class TriMesh : public Mesh
 {
 public:
     //! TriMesh constructor
@@ -113,7 +136,7 @@ public:
     //! Constructor for Nt triangles and the given points
     TriMesh( size_t N_tri, std::shared_ptr<PointList> points );
     //! Constructor from TriList
-    TriMesh( const TriList& );
+    TriMesh( const TriList & );
     //! Destructor
     virtual ~TriMesh();
     //! Mesh class name
@@ -121,21 +144,22 @@ public:
     //! Number of points for the given variable type
     virtual size_t numberPointsVar( VariableType type ) const;
     //! Pack the data
-    virtual std::pair<size_t,void*> pack( int level ) const;
+    virtual std::pair<size_t, void *> pack( int level ) const;
     //! Unpack the data
-    virtual void unpack( const std::pair<size_t,void*>& data );
+    virtual void unpack( const std::pair<size_t, void *> &data );
+
 public:
-    std::shared_ptr<PointList> vertices;    //!< List of verticies
+    std::shared_ptr<PointList> vertices; //!< List of verticies
-    std::vector<int>    A;                  //!< First vertex
+    std::vector<int> A;                  //!< First vertex
-    std::vector<int>    B;                  //!< Second vertex
+    std::vector<int> B;                  //!< Second vertex
-    std::vector<int>    C;                  //!< Third vertex
+    std::vector<int> C;                  //!< Third vertex
 };
 
 
 /*! \class Domain
     \brief A class used to hold the domain
 */
-class DomainMesh: public Mesh
+class DomainMesh : public Mesh
 {
 public:
     //! Empty constructor
@@ -149,9 +173,10 @@ public:
     //! Number of points for the given variable type
     virtual size_t numberPointsVar( VariableType type ) const;
     //! Pack the data
-    virtual std::pair<size_t,void*> pack( int level ) const;
+    virtual std::pair<size_t, void *> pack( int level ) const;
     //! Unpack the data
-    virtual void unpack( const std::pair<size_t,void*>& data );
+    virtual void unpack( const std::pair<size_t, void *> &data );
+
 public:
     int nprocx, nprocy, nprocz, rank;
     int nx, ny, nz;
@@ -159,37 +184,40 @@ public:
 };
 
 
-
 /*! \class Variable
     \brief A base class for variables
 */
-struct Variable
+struct Variable {
-{
 public:
     // Internal variables
-    unsigned char dim;          //!< Number of points per grid point (1: scalar, 3: vector, ...)
+    unsigned char dim;  //!< Number of points per grid point (1: scalar, 3: vector, ...)
-    VariableType type;          //!< Variable type
+    VariableType type;  //!< Variable type
-    DataType precision;         //!< Variable precision to use for IO
+    DataType precision; //!< Variable precision to use for IO
-    std::string name;           //!< Variable name
+    std::string name;   //!< Variable name
-    Array<double> data;         //!< Variable data
+    Array<double> data; //!< Variable data
     //! Empty constructor
-    Variable(): dim(0), type(VariableType::NullVariable), precision(DataType::Double) {}
+    Variable() : dim( 0 ), type( VariableType::NullVariable ), precision( DataType::Double ) {}
     //! Constructor
-    Variable( int dim_, IO::VariableType type_, const std::string& name_ ):
+    Variable( int dim_, IO::VariableType type_, const std::string &name_ )
-        dim(dim_), type(type_), precision(DataType::Double), name(name_) {}
+        : dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ )
+    {
+    }
     //! Constructor
-    Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array<double>& data_ ):
+    Variable(
-        dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {}
+        int dim_, IO::VariableType type_, const std::string &name_, const Array<double> &data_ )
+        : dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ), data( data_ )
+    {
+    }
     //! Destructor
     virtual ~Variable() {}
+
 protected:
     //! Empty constructor
-    Variable(const Variable&);
+    Variable( const Variable & );
-    Variable& operator=(const Variable&);
+    Variable &operator=( const Variable & );
 };
 
 
-
 /*! \class MeshDataStruct
     \brief A class used to hold database info for saving a mesh
 */
@@ -197,11 +225,11 @@ struct MeshDataStruct {
     DataType precision;         //!< Precision to use for IO (mesh)
     std::string meshName;       //!< Mesh name
     std::shared_ptr<Mesh> mesh; //!< Mesh data
-    std::vector<std::shared_ptr<Variable> >  vars;
+    std::vector<std::shared_ptr<Variable>> vars;
     //! Empty constructor
-    MeshDataStruct(): precision(DataType::Double) {}
+    MeshDataStruct() : precision( DataType::Double ) {}
     //! Check the data
-    bool check() const;
+    bool check( bool abort = true ) const;
 };
 
 
@@ -214,7 +242,6 @@ std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh );
 std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh );
 
 
-} // IO namespace
+} // namespace IO
 
 #endif
-

IO/MeshDatabase.cpp

@@ -1,130 +1,144 @@
 #include "IO/MeshDatabase.h"
+#include "IO/IOHelpers.h"
 #include "IO/Mesh.h"
 #include "IO/PackData.h"
-#include "IO/IOHelpers.h"
 #include "common/MPI.h"
 #include "common/Utilities.h"
 
-#include <vector>
+#include <cstdio>
 #include <map>
 #include <set>
-#include <cstdio>
+#include <vector>
 
 #include <ProfilerApp.h>
 
 
+// Default pack/unpack
+// clang-format off
+#define INSTANTIATE_PACK( TYPE )                            \
+    template<>                                              \
+    size_t packsize<TYPE>( const TYPE &rhs )                \
+    {                                                       \
+        return sizeof( TYPE );                              \
+    }                                                       \
+    template<>                                              \
+    void pack<TYPE>( const TYPE &rhs, char *buffer )        \
+    {                                                       \
+        memcpy( buffer, &rhs, sizeof( IO::MeshType ) );     \
+    }                                                       \
+    template<>                                              \
+    void unpack<TYPE>( TYPE &data, const char *buffer )     \
+    {                                                       \
+        memcpy( &data, buffer, sizeof( IO::MeshType ) );    \
+    }
+INSTANTIATE_PACK( IO::VariableType )
+INSTANTIATE_PACK( IO::DataType )
+INSTANTIATE_PACK( IO::MeshType )
+INSTANTIATE_PACK( IO::FileFormat )
+// clang-format on
+
 
-// MeshType
-template<>
-size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
-{
-    return sizeof(IO::MeshType);
-}
-template<>
-void pack<IO::MeshType>( const IO::MeshType& rhs, char *buffer )
-{
-    memcpy(buffer,&rhs,sizeof(IO::MeshType));
-}
-template<>
-void unpack<IO::MeshType>( IO::MeshType& data, const char *buffer )
-{
-    memcpy(&data,buffer,sizeof(IO::MeshType));
-}
-// Variable::VariableType
-template<>
-size_t packsize<IO::VariableType>( const IO::VariableType& rhs )
-{
-    return sizeof(IO::VariableType);
-}
-template<>
-void pack<IO::VariableType>( const IO::VariableType& rhs, char *buffer )
-{
-    memcpy(buffer,&rhs,sizeof(IO::VariableType));
-}
-template<>
-void unpack<IO::VariableType>( IO::VariableType& data, const char *buffer )
-{
-    memcpy(&data,buffer,sizeof(IO::VariableType));
-}
 // DatabaseEntry
 template<>
-size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs )
+size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs )
 {
-    return packsize(rhs.name)+packsize(rhs.file)+packsize(rhs.offset);
+    return packsize( rhs.name ) + packsize( rhs.file ) + packsize( rhs.offset );
 }
 template<>
-void pack<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs, char *buffer )
+void pack<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs, char *buffer )
 {
-    size_t i=0;
+    size_t i = 0;
-    pack(rhs.name,&buffer[i]);      i+=packsize(rhs.name);
+    pack( rhs.name, &buffer[i] );
-    pack(rhs.file,&buffer[i]);      i+=packsize(rhs.file);
+    i += packsize( rhs.name );
-    pack(rhs.offset,&buffer[i]);    i+=packsize(rhs.offset);
+    pack( rhs.file, &buffer[i] );
+    i += packsize( rhs.file );
+    pack( rhs.offset, &buffer[i] );
+    i += packsize( rhs.offset );
 }
 template<>
-void unpack<IO::DatabaseEntry>( IO::DatabaseEntry& data, const char *buffer )
+void unpack<IO::DatabaseEntry>( IO::DatabaseEntry &data, const char *buffer )
 {
-    size_t i=0;
+    size_t i = 0;
-    unpack(data.name,&buffer[i]);   i+=packsize(data.name);
+    unpack( data.name, &buffer[i] );
-    unpack(data.file,&buffer[i]);   i+=packsize(data.file);
+    i += packsize( data.name );
-    unpack(data.offset,&buffer[i]); i+=packsize(data.offset);
+    unpack( data.file, &buffer[i] );
+    i += packsize( data.file );
+    unpack( data.offset, &buffer[i] );
+    i += packsize( data.offset );
 }
 // VariableDatabase
 template<>
-size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase& rhs )
+size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase &rhs )
 {
-    return packsize(rhs.name)+packsize(rhs.type)+packsize(rhs.dim);
+    return packsize( rhs.name ) + packsize( rhs.type ) + packsize( rhs.dim );
 }
 template<>
-void pack<IO::VariableDatabase>( const IO::VariableDatabase& rhs, char *buffer )
+void pack<IO::VariableDatabase>( const IO::VariableDatabase &rhs, char *buffer )
 {
-    size_t i=0;
+    size_t i = 0;
-    pack(rhs.name,&buffer[i]);      i+=packsize(rhs.name);
+    pack( rhs.name, &buffer[i] );
-    pack(rhs.type,&buffer[i]);      i+=packsize(rhs.type);
+    i += packsize( rhs.name );
-    pack(rhs.dim,&buffer[i]);       i+=packsize(rhs.dim);
+    pack( rhs.type, &buffer[i] );
+    i += packsize( rhs.type );
+    pack( rhs.dim, &buffer[i] );
+    i += packsize( rhs.dim );
 }
 template<>
-void unpack<IO::VariableDatabase>( IO::VariableDatabase& data, const char *buffer )
+void unpack<IO::VariableDatabase>( IO::VariableDatabase &data, const char *buffer )
 {
-    size_t i=0;
+    size_t i = 0;
-    unpack(data.name,&buffer[i]);   i+=packsize(data.name);
+    unpack( data.name, &buffer[i] );
-    unpack(data.type,&buffer[i]);   i+=packsize(data.type);
+    i += packsize( data.name );
-    unpack(data.dim,&buffer[i]);    i+=packsize(data.dim);
+    unpack( data.type, &buffer[i] );
+    i += packsize( data.type );
+    unpack( data.dim, &buffer[i] );
+    i += packsize( data.dim );
 }
 // MeshDatabase
 template<>
-size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase& data )
+size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase &data )
 {
-    return packsize(data.name) 
+    return packsize( data.name ) + packsize( data.type ) + packsize( data.meshClass ) +
-         + packsize(data.type)
+           packsize( data.format ) + packsize( data.domains ) + packsize( data.variables ) +
-         + packsize(data.meshClass)
+           packsize( data.variable_data );
-         + packsize(data.format)
-         + packsize(data.domains)
-         + packsize(data.variables)
-         + packsize(data.variable_data);
 }
 template<>
-void pack<IO::MeshDatabase>( const IO::MeshDatabase& rhs, char *buffer )
+void pack<IO::MeshDatabase>( const IO::MeshDatabase &rhs, char *buffer )
 {
     size_t i = 0;
-    pack(rhs.name,&buffer[i]);          i+=packsize(rhs.name);
+    pack( rhs.name, &buffer[i] );
-    pack(rhs.type,&buffer[i]);          i+=packsize(rhs.type);
+    i += packsize( rhs.name );
-    pack(rhs.meshClass,&buffer[i]);     i+=packsize(rhs.meshClass);
+    pack( rhs.type, &buffer[i] );
-    pack(rhs.format,&buffer[i]);        i+=packsize(rhs.format);
+    i += packsize( rhs.type );
-    pack(rhs.domains,&buffer[i]);       i+=packsize(rhs.domains);
+    pack( rhs.meshClass, &buffer[i] );
-    pack(rhs.variables,&buffer[i]);     i+=packsize(rhs.variables);
+    i += packsize( rhs.meshClass );
-    pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data);
+    pack( rhs.format, &buffer[i] );
+    i += packsize( rhs.format );
+    pack( rhs.domains, &buffer[i] );
+    i += packsize( rhs.domains );
+    pack( rhs.variables, &buffer[i] );
+    i += packsize( rhs.variables );
+    pack( rhs.variable_data, &buffer[i] );
+    i += packsize( rhs.variable_data );
 }
 template<>
-void unpack<IO::MeshDatabase>( IO::MeshDatabase& data, const char *buffer )
+void unpack<IO::MeshDatabase>( IO::MeshDatabase &data, const char *buffer )
 {
-    size_t i=0;
+    size_t i = 0;
-    unpack(data.name,&buffer[i]);       i+=packsize(data.name);
+    unpack( data.name, &buffer[i] );
-    unpack(data.type,&buffer[i]);       i+=packsize(data.type);
+    i += packsize( data.name );
-    unpack(data.meshClass,&buffer[i]);  i+=packsize(data.meshClass);
+    unpack( data.type, &buffer[i] );
-    unpack(data.format,&buffer[i]);     i+=packsize(data.format);
+    i += packsize( data.type );
-    unpack(data.domains,&buffer[i]);    i+=packsize(data.domains);
+    unpack( data.meshClass, &buffer[i] );
-    unpack(data.variables,&buffer[i]);  i+=packsize(data.variables);
+    i += packsize( data.meshClass );
-    unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data);
+    unpack( data.format, &buffer[i] );
+    i += packsize( data.format );
+    unpack( data.domains, &buffer[i] );
+    i += packsize( data.domains );
+    unpack( data.variables, &buffer[i] );
+    i += packsize( data.variables );
+    unpack( data.variable_data, &buffer[i] );
+    i += packsize( data.variable_data );
 }
 
 
@@ -132,79 +146,72 @@ namespace IO {
 
 
 /****************************************************
-* VariableDatabase                                  *
+ * VariableDatabase                                  *
-****************************************************/
+ ****************************************************/
-bool VariableDatabase::operator==(const VariableDatabase& rhs ) const
+bool VariableDatabase::operator==( const VariableDatabase &rhs ) const
 {
-    return type==rhs.type && dim==rhs.dim && name==rhs.name;
+    return type == rhs.type && dim == rhs.dim && name == rhs.name;
 }
-bool VariableDatabase::operator!=(const VariableDatabase& rhs ) const
+bool VariableDatabase::operator!=( const VariableDatabase &rhs ) const
 {
-    return type!=rhs.type || dim!=rhs.dim || name!=rhs.name;
+    return type != rhs.type || dim != rhs.dim || name != rhs.name;
 }
-bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const
+bool VariableDatabase::operator>=( const VariableDatabase &rhs ) const
 {
-    return operator>(rhs) || operator==(rhs);
+    return operator>( rhs ) || operator==( rhs );
 }
-bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const
+bool VariableDatabase::operator<=( const VariableDatabase &rhs ) const { return !operator>( rhs ); }
+bool VariableDatabase::operator>( const VariableDatabase &rhs ) const
 {
-    return !operator>(rhs);
+    if ( name > rhs.name )
-}
-bool VariableDatabase::operator>(const VariableDatabase& rhs ) const
-{
-    if ( name>rhs.name )
         return true;
-    else if ( name<rhs.name )
+    else if ( name < rhs.name )
         return false;
-    if ( type>rhs.type )
+    if ( type > rhs.type )
         return true;
-    else if ( type<rhs.type )
+    else if ( type < rhs.type )
         return false;
-    if ( dim>rhs.dim )
+    if ( dim > rhs.dim )
         return true;
-    else if ( dim<rhs.dim )
+    else if ( dim < rhs.dim )
         return false;
     return false;
 }
-bool VariableDatabase::operator<(const VariableDatabase& rhs ) const
+bool VariableDatabase::operator<( const VariableDatabase &rhs ) const
 {
-    return !operator>(rhs) && operator!=(rhs);
+    return !operator>( rhs ) && operator!=( rhs );
 }
 
 
 /****************************************************
-* MeshDatabase                                      *
+ * MeshDatabase                                      *
-****************************************************/
+ ****************************************************/
-MeshDatabase::MeshDatabase()
+MeshDatabase::MeshDatabase() {}
+MeshDatabase::~MeshDatabase() {}
+MeshDatabase::MeshDatabase( const MeshDatabase &rhs )
 {
-}
+    name          = rhs.name;
-MeshDatabase::~MeshDatabase()
+    type          = rhs.type;
-{
+    meshClass     = rhs.meshClass;
-}
+    format        = rhs.format;
-MeshDatabase::MeshDatabase(const MeshDatabase& rhs)
+    domains       = rhs.domains;
-{
+    variables     = rhs.variables;
-    name = rhs.name;
-    type = rhs.type;
-    meshClass = rhs.meshClass;
-    format = rhs.format;
-    domains = rhs.domains;
-    variables = rhs.variables;
     variable_data = rhs.variable_data;
 }
-MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs)
+MeshDatabase &MeshDatabase::operator=( const MeshDatabase &rhs )
 {
-    this->name = rhs.name;
+    this->name          = rhs.name;
-    this->type = rhs.type;
+    this->type          = rhs.type;
-    this->meshClass = rhs.meshClass;
+    this->meshClass     = rhs.meshClass;
-    this->format = rhs.format;
+    this->format        = rhs.format;
-    this->domains = rhs.domains;
+    this->domains       = rhs.domains;
-    this->variables = rhs.variables;
+    this->variables     = rhs.variables;
     this->variable_data = rhs.variable_data;
     return *this;
 }
-VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname ) const
+VariableDatabase MeshDatabase::getVariableDatabase( const std::string &varname ) const
 {
-    for (size_t i=0; i<variables.size(); i++) {
+    for ( size_t i = 0; i < variables.size(); i++ ) {
         if ( variables[i].name == varname )
             return variables[i];
     }
@@ -213,217 +220,219 @@ VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname )
 
 
 /****************************************************
-* DatabaseEntry                                     *
+ * DatabaseEntry                                     *
-****************************************************/
+ ****************************************************/
-std::string DatabaseEntry::write( ) const
+std::string DatabaseEntry::write() const
 {
     char tmp[1000];
-    sprintf(tmp,"%s; %s; %lu",name.c_str(),file.c_str(),offset);
+    sprintf( tmp, "%s; %s; %lu", name.c_str(), file.c_str(), offset );
-    return std::string(tmp);
+    return std::string( tmp );
 }
-DatabaseEntry::DatabaseEntry( const char* line )
+DatabaseEntry::DatabaseEntry( const char *line )
 {
-    std::vector<std::string> list = splitList(line,';');
+    auto list = splitList( line, ';' );
-    name = list[0];
+    name      = list[0];
-    file = list[1];
+    file      = list[1];
-    offset = atol(list[2].c_str());
+    offset    = atol( list[2].c_str() );
 }
-void DatabaseEntry::read( const char* line )
+void DatabaseEntry::read( const char *line )
 {
-    std::vector<std::string> list = splitList(line,';');
+    auto list = splitList( line, ';' );
-    name = list[0];
+    name      = list[0];
-    file = list[1];
+    file      = list[1];
-    offset = atol(list[2].c_str());
+    offset    = atol( list[2].c_str() );
 }
-void DatabaseEntry::read( const std::string& line )
+void DatabaseEntry::read( const std::string &line )
 {
-    std::vector<std::string> list = splitList(line.c_str(),';');
+    auto list = splitList( line.c_str(), ';' );
-    name = list[0];
+    name      = list[0];
-    file = list[1];
+    file      = list[1];
-    offset = atol(list[2].c_str());
+    offset    = atol( list[2].c_str() );
 }
 
 
 // Gather the mesh databases from all processors
-inline int tod( int N ) { return (N+7)/sizeof(double); }
+inline int tod( int N ) { return ( N + 7 ) / sizeof( double ); }
-std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm )
+std::vector<MeshDatabase> gatherAll(
+    const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm )
 {
     if ( comm.getSize() == 1 )
         return meshes;
-    PROFILE_START("gatherAll");
+    PROFILE_START( "gatherAll" );
-    PROFILE_START("gatherAll-pack",2);
+    PROFILE_START( "gatherAll-pack", 2 );
     int size = comm.getSize();
     // First pack the mesh data to local buffers
     int localsize = 0;
-    for (size_t i=0; i<meshes.size(); i++)
+    for ( size_t i = 0; i < meshes.size(); i++ )
-        localsize += tod(packsize(meshes[i]));
+        localsize += tod( packsize( meshes[i] ) );
     auto localbuf = new double[localsize];
-    int pos = 0;
+    int pos       = 0;
-    for (size_t i=0; i<meshes.size(); i++) {
+    for ( size_t i = 0; i < meshes.size(); i++ ) {
-        pack( meshes[i], (char*) &localbuf[pos] );
+        pack( meshes[i], (char *) &localbuf[pos] );
-        pos += tod(packsize(meshes[i]));
+        pos += tod( packsize( meshes[i] ) );
     }
-    PROFILE_STOP("gatherAll-pack",2);
+    PROFILE_STOP( "gatherAll-pack", 2 );
     // Get the number of bytes each processor will be sending/recieving
-    PROFILE_START("gatherAll-send1",2);
+    PROFILE_START( "gatherAll-send1", 2 );
-    auto recvsize = comm.allGather( localsize );
+    auto recvsize  = comm.allGather( localsize );
     int globalsize = recvsize[0];
-    auto disp = new int[size];
+    auto disp      = new int[size];
-    disp[0] = 0;
+    disp[0]        = 0;
-    for (int i=1; i<size; i++) {
+    for ( int i = 1; i < size; i++ ) {
-        disp[i] = disp[i-1] + recvsize[i];
+        disp[i] = disp[i - 1] + recvsize[i];
         globalsize += recvsize[i];
     }
-    PROFILE_STOP("gatherAll-send1",2);
+    PROFILE_STOP( "gatherAll-send1", 2 );
     // Send/recv the global data
-    PROFILE_START("gatherAll-send2",2);
+    PROFILE_START( "gatherAll-send2", 2 );
     auto globalbuf = new double[globalsize];
-    comm.allGather(localbuf,localsize,globalbuf,recvsize.data(),disp,true);
+    comm.allGather( localbuf, localsize, globalbuf, recvsize.data(), disp, true );
-    PROFILE_STOP("gatherAll-send2",2);
+    PROFILE_STOP( "gatherAll-send2", 2 );
     // Unpack the data
-    PROFILE_START("gatherAll-unpack",2);
+    PROFILE_START( "gatherAll-unpack", 2 );
-    std::map<std::string,MeshDatabase> data;
+    std::map<std::string, MeshDatabase> data;
     pos = 0;
     while ( pos < globalsize ) {
         MeshDatabase tmp;
-        unpack(tmp,(char*)&globalbuf[pos]);
+        unpack( tmp, (char *) &globalbuf[pos] );
-        pos += tod(packsize(tmp));
+        pos += tod( packsize( tmp ) );
-        std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
+        std::map<std::string, MeshDatabase>::iterator it = data.find( tmp.name );
-        if ( it==data.end() ) {
+        if ( it == data.end() ) {
             data[tmp.name] = tmp;
         } else {
-            for (size_t i=0; i<tmp.domains.size(); i++)
+            for ( size_t i = 0; i < tmp.domains.size(); i++ )
-                it->second.domains.push_back(tmp.domains[i]);
+                it->second.domains.push_back( tmp.domains[i] );
-            for (size_t i=0; i<tmp.variables.size(); i++)
+            for ( size_t i = 0; i < tmp.variables.size(); i++ )
-                it->second.variables.push_back(tmp.variables[i]);
+                it->second.variables.push_back( tmp.variables[i] );
-            it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
+            it->second.variable_data.insert( tmp.variable_data.begin(), tmp.variable_data.end() );
         }
     }
-    for (auto it=data.begin(); it!=data.end(); ++it) {
+    for ( auto it = data.begin(); it != data.end(); ++it ) {
         // Get the unique variables
-        std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
+        std::set<VariableDatabase> data2(
-        it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
+            it->second.variables.begin(), it->second.variables.end() );
+        it->second.variables = std::vector<VariableDatabase>( data2.begin(), data2.end() );
     }
     // Free temporary memory
-    delete [] localbuf;
+    delete[] localbuf;
-    delete [] disp;
+    delete[] disp;
-    delete [] globalbuf;
+    delete[] globalbuf;
     // Return the results
-    std::vector<MeshDatabase> data2(data.size());
+    std::vector<MeshDatabase> data2( data.size() );
-    size_t i=0; 
+    size_t i = 0;
-    for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
+    for ( auto it = data.begin(); it != data.end(); ++it, ++i )
         data2[i] = it->second;
-    PROFILE_STOP("gatherAll-unpack",2);
+    PROFILE_STOP( "gatherAll-unpack", 2 );
-    PROFILE_STOP("gatherAll");
+    PROFILE_STOP( "gatherAll" );
     return data2;
 }
 
 
 //! Write the mesh databases to a file
-void write( const std::vector<MeshDatabase>& meshes, const std::string& filename )
+void write( const std::vector<MeshDatabase> &meshes, const std::string &filename )
 {
-    PROFILE_START("write");
+    PROFILE_START( "write" );
-    FILE *fid = fopen(filename.c_str(),"wb");
+    FILE *fid = fopen( filename.c_str(), "wb" );
-    for (size_t i=0; i<meshes.size(); i++) {
+    for ( size_t i = 0; i < meshes.size(); i++ ) {
-        fprintf(fid,"%s\n",meshes[i].name.c_str());
+        fprintf( fid, "%s\n", meshes[i].name.c_str() );
-        fprintf(fid,"   type: %i\n",static_cast<int>(meshes[i].type));
+        fprintf( fid, "   type: %s\n", getString( meshes[i].type ).data() );
-        fprintf(fid,"   meshClass: %s\n",meshes[i].meshClass.c_str());
+        fprintf( fid, "   meshClass: %s\n", meshes[i].meshClass.c_str() );
-        fprintf(fid,"   format: %i\n",static_cast<int>(meshes[i].format));
+        fprintf( fid, "   format: %s\n", getString( meshes[i].format ).data() );
-        for (size_t j=0; j<meshes[i].domains.size(); j++)
+        for ( size_t j = 0; j < meshes[i].domains.size(); j++ )
-            fprintf(fid,"   domain: %s\n",meshes[i].domains[j].write().c_str());
+            fprintf( fid, "   domain: %s\n", meshes[i].domains[j].write().c_str() );
-        fprintf(fid,"   variables: ");
+        fprintf( fid, "   variables: " );
-        for (size_t j=0; j<meshes[i].variables.size(); j++) {
+        for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) {
-            const VariableDatabase& var = meshes[i].variables[j];
+            const VariableDatabase &var = meshes[i].variables[j];
-            fprintf(fid,"%s|%i|%i; ",var.name.c_str(),static_cast<int>(var.type),var.dim);
+            fprintf( fid, "%s|%s|%i; ", var.name.data(), getString( var.type ).data(), var.dim );
         }
-        fprintf(fid,"\n");
+        fprintf( fid, "\n" );
-        std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
+        for ( auto it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end();
-        for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) {
+              ++it ) {
-            const char* domain = it->first.first.c_str();
+            const char *domain   = it->first.first.c_str();
-            const char* variable = it->first.second.c_str();
+            const char *variable = it->first.second.c_str();
-            fprintf(fid,"   variable(%s,%s): %s\n",domain,variable,it->second.write().c_str());
+            fprintf(
+                fid, "   variable(%s,%s): %s\n", domain, variable, it->second.write().c_str() );
         }
     }
-    fclose(fid);
+    fclose( fid );
-    PROFILE_STOP("write");
+    PROFILE_STOP( "write" );
 }
 
 
 //! Read the mesh databases from a file
-std::vector<MeshDatabase> read( const std::string& filename )
+std::vector<MeshDatabase> read( const std::string &filename )
 {
     std::vector<MeshDatabase> meshes;
-    PROFILE_START("read");
+    PROFILE_START( "read" );
-    FILE *fid = fopen(filename.c_str(),"rb");
+    FILE *fid = fopen( filename.c_str(), "rb" );
-    if ( fid==NULL )
+    if ( fid == NULL )
-        ERROR("Error opening file");
+        ERROR( "Error opening file" );
     char *line = new char[10000];
-    while ( std::fgets(line,1000,fid) != NULL ) {
+    while ( std::fgets( line, 1000, fid ) != NULL ) {
-        if ( line[0]<32 ) {
+        if ( line[0] < 32 ) {
             // Empty line
             continue;
         } else if ( line[0] != ' ' ) {
-            meshes.resize(meshes.size()+1);
+            meshes.resize( meshes.size() + 1 );
-            std::string name(line);
+            std::string name( line );
-            name.resize(name.size()-1);
+            name.resize( name.size() - 1 );
             meshes.back().name = name;
-        } else if ( strncmp(line,"   format:",10)==0 ) {
+        } else if ( strncmp( line, "   format:", 10 ) == 0 ) {
-            meshes.back().format = static_cast<unsigned char>(atoi(&line[10]));
+            meshes.back().format = getFileFormat( &line[10] );
-        } else if ( strncmp(line,"   type:",8)==0 ) {
+        } else if ( strncmp( line, "   type:", 8 ) == 0 ) {
-            meshes.back().type = static_cast<MeshType>(atoi(&line[8]));
+            meshes.back().type = getMeshType( &line[8] );
-        } else if ( strncmp(line,"   meshClass:",13)==0 ) {
+        } else if ( strncmp( line, "   meshClass:", 13 ) == 0 ) {
-            meshes.back().meshClass = deblank(std::string(&line[13]));
+            meshes.back().meshClass = deblank( std::string( &line[13] ) );
-        } else if ( strncmp(line,"   domain:",10)==0 ) {
+        } else if ( strncmp( line, "   domain:", 10 ) == 0 ) {
-            DatabaseEntry data(&line[10]);
+            DatabaseEntry data( &line[10] );
-            meshes.back().domains.push_back(data);
+            meshes.back().domains.push_back( data );
-        } else if ( strncmp(line,"   variables:",13)==0 ) {
+        } else if ( strncmp( line, "   variables:", 13 ) == 0 ) {
-            MeshDatabase& mesh = meshes.back();
+            MeshDatabase &mesh                 = meshes.back();
-            std::vector<std::string> variables = splitList(&line[13],';');
+            std::vector<std::string> variables = splitList( &line[13], ';' );
-            mesh.variables.resize(variables.size());
+            mesh.variables.resize( variables.size() );
-            for (size_t i=0; i<variables.size(); i++) {
+            for ( size_t i = 0; i < variables.size(); i++ ) {
-                std::vector<std::string> tmp = splitList(variables[i].c_str(),'|');
+                std::vector<std::string> tmp = splitList( variables[i].c_str(), '|' );
-                ASSERT(tmp.size()==3);
+                ASSERT( tmp.size() == 3 );
                 mesh.variables[i].name = tmp[0];
-                mesh.variables[i].type = static_cast<VariableType>(atoi(tmp[1].c_str()));
+                mesh.variables[i].type = getVariableType( tmp[1] );
-                mesh.variables[i].dim = atoi(tmp[2].c_str());
+                mesh.variables[i].dim  = atoi( tmp[2].c_str() );
             }
-        } else if ( strncmp(line,"   variable(",12)==0 ) {
+        } else if ( strncmp( line, "   variable(", 12 ) == 0 ) {
-            size_t i1 = find(line,',');
+            size_t i1            = find( line, ',' );
-            size_t i2 = find(line,':');
+            size_t i2            = find( line, ':' );
-            std::string domain = deblank(std::string(line,12,i1-12));
+            std::string domain   = deblank( std::string( line, 12, i1 - 12 ) );
-            std::string variable = deblank(std::string(line,i1+1,i2-i1-2));
+            std::string variable = deblank( std::string( line, i1 + 1, i2 - i1 - 2 ) );
-            std::pair<std::string,std::string> key(domain,variable);
+            std::pair<std::string, std::string> key( domain, variable );
-            DatabaseEntry data(&line[i2+1]);
+            DatabaseEntry data( &line[i2 + 1] );
-            meshes.back().variable_data.insert( 
+            meshes.back().variable_data.insert(
-                std::pair<std::pair<std::string,std::string>,DatabaseEntry>(key,data) );
+                std::pair<std::pair<std::string, std::string>, DatabaseEntry>( key, data ) );
         } else {
-            ERROR("Error reading line");
+            ERROR( "Error reading line" );
         }
     }
-    fclose(fid);
+    fclose( fid );
-    delete [] line;
+    delete[] line;
-    PROFILE_STOP("read");
+    PROFILE_STOP( "read" );
     return meshes;
 }
 
 
 // Return the mesh type
-IO::MeshType meshType( const IO::Mesh& mesh )
+IO::MeshType meshType( const IO::Mesh &mesh )
 {
-    IO::MeshType type = IO::Unknown;
+    IO::MeshType type           = IO::MeshType::Unknown;
     const std::string meshClass = mesh.className();
-    if ( meshClass=="PointList" ) {
+    if ( meshClass == "PointList" ) {
-        type = IO::PointMesh;
+        type = IO::MeshType::PointMesh;
-    } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
+    } else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
-        type = IO::SurfaceMesh;
+        type = IO::MeshType::SurfaceMesh;
-    } else if ( meshClass=="DomainMesh" ) {
+    } else if ( meshClass == "DomainMesh" ) {
-        type = IO::VolumeMesh;
+        type = IO::MeshType::VolumeMesh;
     } else {
-        ERROR("Unknown mesh");
+        ERROR( "Unknown mesh" );
     }
     return type;
 }
 
 
-} // IO namespace
+} // namespace IO
-

IO/MeshDatabase.h

@@ -1,90 +1,85 @@
 #ifndef MeshDatabase_INC
 #define MeshDatabase_INC
 
-#include "IO/Mesh.h" 
+#include "IO/Mesh.h"
 #include "common/MPI.h"
 
 #include <iostream>
+#include <map>
 #include <memory>
 #include <string.h>
 #include <vector>
-#include <map>
 
 
 namespace IO {
 
-class Mesh;
-
-
-//! Enum to identify mesh type
-//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
-enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
-
 
 //! Helper struct for containing offsets for the mesh info
 struct DatabaseEntry {
-    std::string name;                   //!< Name of the entry
+    std::string name;                     //!< Name of the entry
-    std::string file;                   //!< Name of the file containing the entry
+    std::string file;                     //!< Name of the file containing the entry
-    size_t offset;                      //!< Offset in the file to start reading
+    size_t offset;                        //!< Offset in the file to start reading
-    std::string write( ) const;         //!< Convert the data to a string
+    std::string write() const;            //!< Convert the data to a string
-    void read( const char* line );      //!< Convert the string to data
+    void read( const char *line );        //!< Convert the string to data
-    void read( const std::string& line ); //!< Convert the string to data
+    void read( const std::string &line ); //!< Convert the string to data
-    DatabaseEntry( ) {}                 //!< Empty constructor
+    DatabaseEntry() {}                    //!< Empty constructor
-    DatabaseEntry( const char* line );  //!< Convert the string to data
+    DatabaseEntry( const char *line );    //!< Convert the string to data
-    ~DatabaseEntry() {}                 //!< Destructor
+    ~DatabaseEntry() {}                   //!< Destructor
 };
 
 
 //! Structure to hold the info about the variables
 struct VariableDatabase {
-    std::string name;                   //!< Name of the variable
+    std::string name;      //!< Name of the variable
-    IO::VariableType type;              //!< Variable
+    IO::VariableType type; //!< Variable
-    unsigned int dim;                   //!< Number of points per grid point (1: scalar, 3: vector, ...)
+    unsigned int dim;      //!< Number of points per grid point (1: scalar, 3: vector, ...)
     // Overload key operators
-    bool operator==(const VariableDatabase& rhs ) const;
+    bool operator==( const VariableDatabase &rhs ) const;
-    bool operator!=(const VariableDatabase& rhs ) const;
+    bool operator!=( const VariableDatabase &rhs ) const;
-    bool operator>=(const VariableDatabase& rhs ) const;
+    bool operator>=( const VariableDatabase &rhs ) const;
-    bool operator<=(const VariableDatabase& rhs ) const;
+    bool operator<=( const VariableDatabase &rhs ) const;
-    bool operator> (const VariableDatabase& rhs ) const;
+    bool operator>( const VariableDatabase &rhs ) const;
-    bool operator< (const VariableDatabase& rhs ) const;
+    bool operator<( const VariableDatabase &rhs ) const;
 };
 
 
 //! Structure to hold the info about the meshes
 struct MeshDatabase {
-    typedef  std::pair<std::string,std::string>  variable_id;
+    typedef std::pair<std::string, std::string> variable_id;
     std::string name;                   //!< Name of the mesh
     MeshType type;                      //!< Mesh type
     std::string meshClass;              //!< Mesh class
-    unsigned char format;               //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
+    FileFormat format;                  //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
     std::vector<DatabaseEntry> domains; //!< List of the domains
-    std::vector<VariableDatabase> variables; //!< List of the variables
+    std::vector<VariableDatabase> variables;            //!< List of the variables
-    std::map<variable_id,DatabaseEntry> variable_data; //!< Data for the variables
+    std::map<variable_id, DatabaseEntry> variable_data; //!< Data for the variables
-    VariableDatabase getVariableDatabase( const std::string& varname ) const;
+    VariableDatabase getVariableDatabase( const std::string &varname ) const;
+
 public:
     MeshDatabase();
     ~MeshDatabase();
-    MeshDatabase(const MeshDatabase&);
+    MeshDatabase( const MeshDatabase & );
-    MeshDatabase& operator=(const MeshDatabase&);
+    MeshDatabase &operator=( const MeshDatabase & );
 };
 
 
 //! Gather the mesh databases from all processors
-std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm );
+std::vector<MeshDatabase> gatherAll(
+    const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm );
 
 
 //! Write the mesh databases to a file
-void write( const std::vector<MeshDatabase>& meshes, const std::string& filename );
+void write( const std::vector<MeshDatabase> &meshes, const std::string &filename );
 
 
 //! Read the mesh databases from a file
-std::vector<MeshDatabase> read( const std::string& filename );
+std::vector<MeshDatabase> read( const std::string &filename );
 
 
 //! Return the mesh type
-IO::MeshType meshType( const IO::Mesh& mesh );
+IO::MeshType meshType( const IO::Mesh &mesh );
 
 
-} // IO namespace
+} // namespace IO
 
 #endif

IO/PIO.cpp

@@ -1,10 +1,10 @@
 #include "IO/PIO.h"
-#include "common/Utilities.h"
 #include "common/MPI.h"
+#include "common/Utilities.h"
 
+#include <cstring>
 #include <fstream>
 #include <string>
-#include <cstring>
 
 
 namespace IO {
@@ -15,19 +15,18 @@ static ParallelStreamBuffer perr_buffer;
 static ParallelStreamBuffer plog_buffer;
 
 
-std::ostream pout(&pout_buffer);
+std::ostream pout( &pout_buffer );
-std::ostream perr(&perr_buffer);
+std::ostream perr( &perr_buffer );
-std::ostream plog(&plog_buffer);
+std::ostream plog( &plog_buffer );
-
 
 
 /****************************************************************************
-*  Functions to control logging                                             *
+ *  Functions to control logging                                             *
-****************************************************************************/
+ ****************************************************************************/
-std::ofstream *global_filestream=NULL;
+std::ofstream *global_filestream = NULL;
-static void shutdownFilestream( )
+static void shutdownFilestream()
 {
-    if ( global_filestream!=NULL ) {
+    if ( global_filestream != NULL ) {
         global_filestream->flush();
         global_filestream->close();
         delete global_filestream;
@@ -37,16 +36,16 @@ static void shutdownFilestream( )
 void Utilities::logOnlyNodeZero( const std::string &filename )
 {
     int rank = 0;
-    #ifdef USE_MPI
+#ifdef USE_MPI
-        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-    #endif
+#endif
     if ( rank == 0 )
-        logAllNodes(filename,true);
+        logAllNodes( filename, true );
 }
 void Utilities::logAllNodes( const std::string &filename, bool singleStream )
 {
     if ( singleStream )
-        ERROR("Not implimented yet");
+        ERROR( "Not implimented yet" );
 
     // If the filestream was open, then close it and reset streams
     shutdownFilestream();
@@ -55,33 +54,33 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream )
     std::string full_filename = filename;
     if ( !singleStream ) {
         int rank = 0;
-        #ifdef USE_MPI
+#ifdef USE_MPI
-            MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank );
-        #endif
+#endif
         char tmp[100];
-        sprintf(tmp,".%04i",rank);
+        sprintf( tmp, ".%04i", rank );
-        full_filename += std::string(tmp);
+        full_filename += std::string( tmp );
     }
-    global_filestream = new std::ofstream(full_filename.c_str());
+    global_filestream = new std::ofstream( full_filename.c_str() );
 
-    if ( !(*global_filestream) ) {
+    if ( !( *global_filestream ) ) {
         delete global_filestream;
         global_filestream = NULL;
         perr << "PIO: Could not open log file ``" << full_filename << "''\n";
     } else {
-        pout_buffer.setOutputStream(global_filestream);
+        pout_buffer.setOutputStream( global_filestream );
-        pout_buffer.setOutputStream(&std::cout);
+        pout_buffer.setOutputStream( &std::cout );
-        perr_buffer.setOutputStream(global_filestream);
+        perr_buffer.setOutputStream( global_filestream );
-        perr_buffer.setOutputStream(&std::cerr);
+        perr_buffer.setOutputStream( &std::cerr );
-        plog_buffer.setOutputStream(global_filestream);
+        plog_buffer.setOutputStream( global_filestream );
     }
 }
 
 
 /****************************************************************************
-*  ParallelStreamBuffer class                                               *
+ *  ParallelStreamBuffer class                                               *
-****************************************************************************/
+ ****************************************************************************/
-void Utilities::stopLogging( )
+void Utilities::stopLogging()
 {
     pout_buffer.reset();
     perr_buffer.reset();
@@ -93,77 +92,71 @@ void Utilities::stopLogging( )
 
 
 /****************************************************************************
-*  ParallelStreamBuffer class                                               *
+ *  ParallelStreamBuffer class                                               *
-****************************************************************************/
+ ****************************************************************************/
-ParallelStreamBuffer::ParallelStreamBuffer( ):
+ParallelStreamBuffer::ParallelStreamBuffer()
-    d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL)
+    : d_rank( 0 ), d_size( 0 ), d_buffer_size( 0 ), d_buffer( NULL )
 {
 }
-ParallelStreamBuffer:: ~ParallelStreamBuffer()
+ParallelStreamBuffer::~ParallelStreamBuffer() { delete[] d_buffer; }
-{
+void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) { d_stream.push_back( stream ); }
-    delete [] d_buffer;
-}
-void ParallelStreamBuffer::setOutputStream( std::ostream *stream )
-{
-    d_stream.push_back( stream );
-}
 int ParallelStreamBuffer::sync()
 {
-    for (size_t i=0; i<d_stream.size(); i++) {
+    for ( size_t i = 0; i < d_stream.size(); i++ ) {
-        std::ostream& stream = *d_stream[i];
+        std::ostream &stream = *d_stream[i];
         stream << d_buffer;
     }
     d_size = 0;
-    memset(d_buffer,0,d_buffer_size);
+    memset( d_buffer, 0, d_buffer_size );
     return 0;
 }
 void ParallelStreamBuffer::reserve( size_t size )
 {
     if ( size > d_buffer_size ) {
-        if ( d_buffer_size==0 ) {
+        if ( d_buffer_size == 0 ) {
             d_buffer_size = 1024;
-            d_buffer = new char[d_buffer_size];
+            d_buffer      = new char[d_buffer_size];
-            memset(d_buffer,0,d_buffer_size);
+            memset( d_buffer, 0, d_buffer_size );
         }
         while ( size > d_buffer_size ) {
             char *tmp = d_buffer;
             d_buffer_size *= 2;
             d_buffer = new char[d_buffer_size];
-            memset(d_buffer,0,d_buffer_size);
+            memset( d_buffer, 0, d_buffer_size );
-            memcpy(d_buffer,tmp,d_size);
+            memcpy( d_buffer, tmp, d_size );
-            delete [] tmp;
+            delete[] tmp;
         }
     }
 }
-std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize n )
+std::streamsize ParallelStreamBuffer::xsputn( const char *text, std::streamsize n )
 {
-    reserve(d_size+n);
+    reserve( d_size + n );
-    memcpy(&d_buffer[d_size],text,n);
+    memcpy( &d_buffer[d_size], text, n );
     d_size += n;
-    if ( text[n-1]==0 || text[n-1]==10 ) { sync(); }
+    if ( text[n - 1] == 0 || text[n - 1] == 10 ) {
+        sync();
+    }
     return n;
 }
-int ParallelStreamBuffer::overflow(int ch)
+int ParallelStreamBuffer::overflow( int ch )
 {
-    reserve(d_size+1);
+    reserve( d_size + 1 );
     d_buffer[d_size] = ch;
     d_size++;
-    if ( ch==0 || ch==10 ) { sync(); }
+    if ( ch == 0 || ch == 10 ) {
-    return std::char_traits<char>::to_int_type(ch);
+        sync();
+    }
+    return std::char_traits<char>::to_int_type( ch );
 }
-int ParallelStreamBuffer::underflow() 
+int ParallelStreamBuffer::underflow() { return -1; }
-{
+void ParallelStreamBuffer::reset()
-    return -1;
-}
-void ParallelStreamBuffer::reset() 
 {
     sync();
     d_stream.clear();
-    delete [] d_buffer;
+    delete[] d_buffer;
-    d_buffer = NULL;
+    d_buffer      = NULL;
     d_buffer_size = 0;
 }
 
 
-} // IO namespace
+} // namespace IO
-

IO/PIO.h

@@ -17,7 +17,7 @@ extern std::ostream pout;
 
 /*!
  * Parallel output stream perr writes to the standard error from all nodes.
- * Output is prepended with the processor number.  
+ * Output is prepended with the processor number.
  */
 extern std::ostream perr;
 
@@ -45,12 +45,11 @@ inline int printp( const char *format, ... );
 class ParallelStreamBuffer : public std::streambuf
 {
 public:
-
     /*!
      * Create a parallel buffer class.  The object will require further
      * initialization to set up the I/O streams and prefix string.
      */
-    ParallelStreamBuffer( );
+    ParallelStreamBuffer();
 
     /*!
      * Set the output file stream (multiple output streams are supported)
@@ -60,26 +59,26 @@ public:
 
     /*!
      * The destructor simply deallocates any internal data
-    * buffers.  It does not modify the output streams.
+     * buffers.  It does not modify the output streams.
      */
     virtual ~ParallelStreamBuffer();
 
     /*!
      * Synchronize the parallel buffer (called from streambuf).
      */
-   virtual int sync();
+    virtual int sync();
 
     /**
      * Write the specified number of characters into the output stream (called
      * from streambuf).
-    */
+     */
-    virtual std::streamsize xsputn(const char* text, std::streamsize n);
+    virtual std::streamsize xsputn( const char *text, std::streamsize n );
 
     /*!
      * Write an overflow character into the parallel buffer (called from
      * streambuf).
      */
-    virtual int overflow(int ch);
+    virtual int overflow( int ch );
 
     /*!
      * Read an overflow character from the parallel buffer (called from
@@ -98,30 +97,30 @@ private:
     size_t d_size;
     size_t d_buffer_size;
     char *d_buffer;
-    std::vector<std::ostream*> d_stream;
+    std::vector<std::ostream *> d_stream;
     inline void reserve( size_t size );
 };
 
 
 namespace Utilities {
 
-    /*!
+/*!
-     * Log messages for node zero only to the specified filename.  All output
+ * Log messages for node zero only to the specified filename.  All output
-     * to pout, perr, and plog on node zero will go to the log file.
+ * to pout, perr, and plog on node zero will go to the log file.
-     */
+ */
-    void logOnlyNodeZero( const std::string &filename );
+void logOnlyNodeZero( const std::string &filename );
 
-    /*!
+/*!
-     * Log messages from all nodes.  The diagnostic data for processor XXXXX
+ * Log messages from all nodes.  The diagnostic data for processor XXXXX
-     * will be sent to a file with the name filename.XXXXX, where filename is
+ * will be sent to a file with the name filename.XXXXX, where filename is
-     * the function argument.
+ * the function argument.
-     */
+ */
-    void logAllNodes( const std::string &filename, bool singleStream=false );
+void logAllNodes( const std::string &filename, bool singleStream = false );
 
-    /*!
+/*!
-     * Stop logging messages, flush buffers, and reset memory.
+ * Stop logging messages, flush buffers, and reset memory.
-     */
+ */
-    void stopLogging( );
+void stopLogging();
 
 
 } // namespace Utilities

IO/PIO.hpp

@@ -3,9 +3,9 @@
 
 #include "IO/PIO.h"
 
+#include <cstdio>
 #include <iostream>
 #include <stdarg.h>
-#include <cstdio>
 
 
 namespace IO {
@@ -13,17 +13,17 @@ namespace IO {
 
 inline int printp( const char *format, ... )
 {
-    va_list ap; 
+    va_list ap;
-    va_start(ap,format); 
+    va_start( ap, format );
     char tmp[1024];
-    int n = vsprintf(tmp,format,ap);
+    int n = vsprintf( tmp, format, ap );
-    va_end(ap);
+    va_end( ap );
     pout << tmp;
     pout.flush();
     return n;
 }
 
 
-} // IO namespace
+} // namespace IO
 
 #endif

IO/PackData.cpp

@@ -4,102 +4,101 @@
 
 
 /********************************************************
-* Concrete implimentations for packing/unpacking        *
+ * Concrete implimentations for packing/unpacking        *
-********************************************************/
+ ********************************************************/
 // unsigned char
 template<>
-size_t packsize<unsigned char>( const unsigned char& rhs )
+size_t packsize<unsigned char>( const unsigned char &rhs )
 {
-    return sizeof(unsigned char);
+    return sizeof( unsigned char );
 }
 template<>
-void pack<unsigned char>( const unsigned char& rhs, char *buffer )
+void pack<unsigned char>( const unsigned char &rhs, char *buffer )
 {
-    memcpy(buffer,&rhs,sizeof(unsigned char));
+    memcpy( buffer, &rhs, sizeof( unsigned char ) );
 }
 template<>
-void unpack<unsigned char>( unsigned char& data, const char *buffer )
+void unpack<unsigned char>( unsigned char &data, const char *buffer )
 {
-    memcpy(&data,buffer,sizeof(unsigned char));
+    memcpy( &data, buffer, sizeof( unsigned char ) );
 }
 // char
 template<>
-size_t packsize<char>( const char& rhs )
+size_t packsize<char>( const char &rhs )
 {
-    return sizeof(char);
+    return sizeof( char );
 }
 template<>
-void pack<char>( const char& rhs, char *buffer )
+void pack<char>( const char &rhs, char *buffer )
 {
-    memcpy(buffer,&rhs,sizeof(char));
+    memcpy( buffer, &rhs, sizeof( char ) );
 }
 template<>
-void unpack<char>( char& data, const char *buffer )
+void unpack<char>( char &data, const char *buffer )
 {
-    memcpy(&data,buffer,sizeof(char));
+    memcpy( &data, buffer, sizeof( char ) );
 }
 // int
 template<>
-size_t packsize<int>( const int& rhs )
+size_t packsize<int>( const int &rhs )
 {
-    return sizeof(int);
+    return sizeof( int );
 }
 template<>
-void pack<int>( const int& rhs, char *buffer )
+void pack<int>( const int &rhs, char *buffer )
 {
-    memcpy(buffer,&rhs,sizeof(int));
+    memcpy( buffer, &rhs, sizeof( int ) );
 }
 template<>
-void unpack<int>( int& data, const char *buffer )
+void unpack<int>( int &data, const char *buffer )
 {
-    memcpy(&data,buffer,sizeof(int));
+    memcpy( &data, buffer, sizeof( int ) );
 }
 // unsigned int
 template<>
-size_t packsize<unsigned int>( const unsigned int& rhs )
+size_t packsize<unsigned int>( const unsigned int &rhs )
 {
-    return sizeof(unsigned int);
+    return sizeof( unsigned int );
 }
 template<>
-void pack<unsigned int>( const unsigned int& rhs, char *buffer )
+void pack<unsigned int>( const unsigned int &rhs, char *buffer )
 {
-    memcpy(buffer,&rhs,sizeof(int));
+    memcpy( buffer, &rhs, sizeof( int ) );
 }
 template<>
-void unpack<unsigned int>( unsigned int& data, const char *buffer )
+void unpack<unsigned int>( unsigned int &data, const char *buffer )
 {
-    memcpy(&data,buffer,sizeof(int));
+    memcpy( &data, buffer, sizeof( int ) );
 }
 // size_t
 template<>
-size_t packsize<size_t>( const size_t& rhs )
+size_t packsize<size_t>( const size_t &rhs )
 {
-    return sizeof(size_t);
+    return sizeof( size_t );
 }
 template<>
-void pack<size_t>( const size_t& rhs, char *buffer )
+void pack<size_t>( const size_t &rhs, char *buffer )
 {
-    memcpy(buffer,&rhs,sizeof(size_t));
+    memcpy( buffer, &rhs, sizeof( size_t ) );
 }
 template<>
-void unpack<size_t>( size_t& data, const char *buffer )
+void unpack<size_t>( size_t &data, const char *buffer )
 {
-    memcpy(&data,buffer,sizeof(size_t));
+    memcpy( &data, buffer, sizeof( size_t ) );
 }
 // std::string
 template<>
-size_t packsize<std::string>( const std::string& rhs )
+size_t packsize<std::string>( const std::string &rhs )
 {
-    return rhs.size()+1;
+    return rhs.size() + 1;
 }
 template<>
-void pack<std::string>( const std::string& rhs, char *buffer )
+void pack<std::string>( const std::string &rhs, char *buffer )
 {
-    memcpy(buffer,rhs.c_str(),rhs.size()+1);
+    memcpy( buffer, rhs.c_str(), rhs.size() + 1 );
 }
 template<>
-void unpack<std::string>( std::string& data, const char *buffer )
+void unpack<std::string>( std::string &data, const char *buffer )
 {
-    data = std::string(buffer);
+    data = std::string( buffer );
 }
-

IO/PackData.h

@@ -2,77 +2,76 @@
 #ifndef included_PackData
 #define included_PackData
 
-#include <vector>
-#include <set>
 #include <map>
+#include <set>
+#include <vector>
 
 
 //! Template function to return the buffer size required to pack a class
 template<class TYPE>
-size_t packsize( const TYPE& rhs );
+size_t packsize( const TYPE &rhs );
 
 //! Template function to pack a class to a buffer
 template<class TYPE>
-void pack( const TYPE& rhs, char *buffer );
+void pack( const TYPE &rhs, char *buffer );
 
 //! Template function to unpack a class from a buffer
 template<class TYPE>
-void unpack( TYPE& data, const char *buffer );
+void unpack( TYPE &data, const char *buffer );
 
 
 //! Template function to return the buffer size required to pack a std::vector
 template<class TYPE>
-size_t packsize( const std::vector<TYPE>& rhs );
+size_t packsize( const std::vector<TYPE> &rhs );
 
 //! Template function to pack a class to a buffer
 template<class TYPE>
-void pack( const std::vector<TYPE>& rhs, char *buffer );
+void pack( const std::vector<TYPE> &rhs, char *buffer );
 
 //! Template function to pack a class to a buffer
 template<class TYPE>
-void unpack( std::vector<TYPE>& data, const char *buffer );
+void unpack( std::vector<TYPE> &data, const char *buffer );
 
 
 //! Template function to return the buffer size required to pack a std::pair
 template<class TYPE1, class TYPE2>
-size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
+size_t packsize( const std::pair<TYPE1, TYPE2> &rhs );
 
 //! Template function to pack a class to a buffer
 template<class TYPE1, class TYPE2>
-void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
+void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer );
 
 //! Template function to pack a class to a buffer
 template<class TYPE1, class TYPE2>
-void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
+void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer );
 
 
 //! Template function to return the buffer size required to pack a std::map
 template<class TYPE1, class TYPE2>
-size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
+size_t packsize( const std::map<TYPE1, TYPE2> &rhs );
 
 //! Template function to pack a class to a buffer
 template<class TYPE1, class TYPE2>
-void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
+void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer );
 
 //! Template function to pack a class to a buffer
 template<class TYPE1, class TYPE2>
-void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
+void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer );
 
 
 //! Template function to return the buffer size required to pack a std::set
 template<class TYPE>
-size_t packsize( const std::set<TYPE>& rhs );
+size_t packsize( const std::set<TYPE> &rhs );
 
 //! Template function to pack a class to a buffer
 template<class TYPE>
-void pack( const std::set<TYPE>& rhs, char *buffer );
+void pack( const std::set<TYPE> &rhs, char *buffer );
 
 //! Template function to pack a class to a buffer
 template<class TYPE>
-void unpack( std::set<TYPE>& data, const char *buffer );
+void unpack( std::set<TYPE> &data, const char *buffer );
 
 
 #include "IO/PackData.hpp"
 
 #endif
-

IO/PackData.hpp

@@ -4,152 +4,156 @@
 
 #include "IO/PackData.h"
 
+#include <map>
+#include <set>
 #include <string.h>
 #include <vector>
-#include <set>
-#include <map>
-
 
 
 /********************************************************
-* Default instantiations for std::vector                *
+ * Default instantiations for std::vector                *
-********************************************************/
+ ********************************************************/
 template<class TYPE>
-size_t packsize( const std::vector<TYPE>& rhs )
+size_t packsize( const std::vector<TYPE> &rhs )
 {
-    size_t bytes = sizeof(size_t);
+    size_t bytes = sizeof( size_t );
-    for (size_t i=0; i<rhs.size(); i++)
+    for ( size_t i = 0; i < rhs.size(); i++ )
-        bytes += packsize(rhs[i]);
+        bytes += packsize( rhs[i] );
     return bytes;
 }
 template<class TYPE>
-void pack( const std::vector<TYPE>& rhs, char *buffer )
+void pack( const std::vector<TYPE> &rhs, char *buffer )
 {
     size_t size = rhs.size();
-    memcpy(buffer,&size,sizeof(size_t));
+    memcpy( buffer, &size, sizeof( size_t ) );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
-    for (size_t i=0; i<rhs.size(); i++) {
+    for ( size_t i = 0; i < rhs.size(); i++ ) {
-        pack(rhs[i],&buffer[pos]);
+        pack( rhs[i], &buffer[pos] );
-        pos += packsize(rhs[i]);
+        pos += packsize( rhs[i] );
     }
 }
 template<class TYPE>
-void unpack( std::vector<TYPE>& data, const char *buffer )
+void unpack( std::vector<TYPE> &data, const char *buffer )
 {
     size_t size;
-    memcpy(&size,buffer,sizeof(size_t));
+    memcpy( &size, buffer, sizeof( size_t ) );
     data.clear();
-    data.resize(size);
+    data.resize( size );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
-    for (size_t i=0; i<data.size(); i++) {
+    for ( size_t i = 0; i < data.size(); i++ ) {
-        unpack(data[i],&buffer[pos]);
+        unpack( data[i], &buffer[pos] );
-        pos += packsize(data[i]);
+        pos += packsize( data[i] );
     }
 }
 
 
 /********************************************************
-* Default instantiations for std::pair                  *
+ * Default instantiations for std::pair                  *
-********************************************************/
+ ********************************************************/
 template<class TYPE1, class TYPE2>
-size_t packsize( const std::pair<TYPE1,TYPE2>& rhs )
+size_t packsize( const std::pair<TYPE1, TYPE2> &rhs )
 {
-    return packsize(rhs.first)+packsize(rhs.second);
+    return packsize( rhs.first ) + packsize( rhs.second );
 }
 template<class TYPE1, class TYPE2>
-void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer )
+void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer )
 {
-    pack(rhs.first,buffer);
+    pack( rhs.first, buffer );
-    pack(rhs.second,&buffer[packsize(rhs.first)]);
+    pack( rhs.second, &buffer[packsize( rhs.first )] );
 }
 template<class TYPE1, class TYPE2>
-void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer )
+void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer )
 {
-    unpack(data.first,buffer);
+    unpack( data.first, buffer );
-    unpack(data.second,&buffer[packsize(data.first)]);
+    unpack( data.second, &buffer[packsize( data.first )] );
 }
 
 
 /********************************************************
-* Default instantiations for std::map                   *
+ * Default instantiations for std::map                   *
-********************************************************/
+ ********************************************************/
 template<class TYPE1, class TYPE2>
-size_t packsize( const std::map<TYPE1,TYPE2>& rhs )
+size_t packsize( const std::map<TYPE1, TYPE2> &rhs )
 {
-    size_t bytes = sizeof(size_t);
+    size_t bytes = sizeof( size_t );
-    typename std::map<TYPE1,TYPE2>::const_iterator it;
+    typename std::map<TYPE1, TYPE2>::const_iterator it;
-    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+    for ( it = rhs.begin(); it != rhs.end(); ++it ) {
-        bytes += packsize(it->first);
+        bytes += packsize( it->first );
-        bytes += packsize(it->second);
+        bytes += packsize( it->second );
     }
     return bytes;
 }
 template<class TYPE1, class TYPE2>
-void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer )
+void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer )
 {
     size_t N = rhs.size();
-    pack(N,buffer);
+    pack( N, buffer );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
-    typename std::map<TYPE1,TYPE2>::const_iterator it;
+    typename std::map<TYPE1, TYPE2>::const_iterator it;
-    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+    for ( it = rhs.begin(); it != rhs.end(); ++it ) {
-        pack(it->first,&buffer[pos]);   pos+=packsize(it->first);
+        pack( it->first, &buffer[pos] );
-        pack(it->second,&buffer[pos]);  pos+=packsize(it->second);
+        pos += packsize( it->first );
+        pack( it->second, &buffer[pos] );
+        pos += packsize( it->second );
     }
 }
 template<class TYPE1, class TYPE2>
-void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer )
+void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer )
 {
     size_t N = 0;
-    unpack(N,buffer);
+    unpack( N, buffer );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
     data.clear();
-    for (size_t i=0; i<N; i++) {
+    for ( size_t i = 0; i < N; i++ ) {
-        std::pair<TYPE1,TYPE2> tmp;
+        std::pair<TYPE1, TYPE2> tmp;
-        unpack(tmp.first,&buffer[pos]);   pos+=packsize(tmp.first);
+        unpack( tmp.first, &buffer[pos] );
-        unpack(tmp.second,&buffer[pos]);  pos+=packsize(tmp.second);
+        pos += packsize( tmp.first );
-        data.insert(tmp);
+        unpack( tmp.second, &buffer[pos] );
+        pos += packsize( tmp.second );
+        data.insert( tmp );
     }
 }
 
 
 /********************************************************
-* Default instantiations for std::set                   *
+ * Default instantiations for std::set                   *
-********************************************************/
+ ********************************************************/
 template<class TYPE>
-size_t packsize( const std::set<TYPE>& rhs )
+size_t packsize( const std::set<TYPE> &rhs )
 {
-    size_t bytes = sizeof(size_t);
+    size_t bytes = sizeof( size_t );
     typename std::set<TYPE>::const_iterator it;
-    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+    for ( it = rhs.begin(); it != rhs.end(); ++it ) {
-        bytes += packsize(*it);
+        bytes += packsize( *it );
     }
     return bytes;
 }
 template<class TYPE>
-void pack( const std::set<TYPE>& rhs, char *buffer )
+void pack( const std::set<TYPE> &rhs, char *buffer )
 {
     size_t N = rhs.size();
-    pack(N,buffer);
+    pack( N, buffer );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
     typename std::set<TYPE>::const_iterator it;
-    for (it=rhs.begin(); it!=rhs.end(); ++it) {
+    for ( it = rhs.begin(); it != rhs.end(); ++it ) {
-        pack(*it);   pos+=packsize(*it);
+        pack( *it );
+        pos += packsize( *it );
     }
 }
 template<class TYPE>
-void unpack( std::set<TYPE>& data, const char *buffer )
+void unpack( std::set<TYPE> &data, const char *buffer )
 {
     size_t N = 0;
-    unpack(N,buffer);
+    unpack( N, buffer );
-    size_t pos = sizeof(size_t);
+    size_t pos = sizeof( size_t );
     data.clear();
-    for (size_t i=0; i<N; i++) {
+    for ( size_t i = 0; i < N; i++ ) {
         TYPE tmp;
-        unpack(tmp,&buffer[pos]);   pos+=packsize(tmp);
+        unpack( tmp, &buffer[pos] );
-        data.insert(tmp);
+        pos += packsize( tmp );
+        data.insert( tmp );
     }
 }
 
 
 #endif
-

IO/Reader.cpp

@@ -1,7 +1,7 @@
 #include "IO/Reader.h"
+#include "IO/IOHelpers.h"
 #include "IO/Mesh.h"
 #include "IO/MeshDatabase.h"
-#include "IO/IOHelpers.h"
 #include "common/Utilities.h"
 
 #ifdef USE_SILO
@@ -10,60 +10,139 @@
 
 
 #include <ProfilerApp.h>
+#include <cstdio>
+#include <fstream>
 #include <iostream>
+#include <map>
 #include <string.h>
 #include <vector>
-#include <map>
-#include <cstdio>
 
 
 // Inline function to read line without a return argument
-static inline void fgetl( char * str, int num, FILE * stream )
+static inline void fgetl( char *str, int num, FILE *stream )
 {
-    char* ptr = fgets( str, num, stream );
+    char *ptr = fgets( str, num, stream );
-    if ( 0 ) {char *temp = (char *)&ptr; temp++;}
+    if ( 0 ) {
+        char *temp = (char *) &ptr;
+        temp++;
+    }
+}
+
+
+// Check if the file exists
+bool fileExists( const std::string &filename )
+{
+    std::ifstream ifile( filename.c_str() );
+    return ifile.good();
 }
 
 
 // Get the path to a file
-std::string IO::getPath( const std::string& filename )
+std::string IO::getPath( const std::string &filename )
 {
-    std::string file(filename);
+    std::string file( filename );
-    size_t k1 = file.rfind(47);
+    size_t k1 = file.rfind( 47 );
-    size_t k2 = file.rfind(92);
+    size_t k2 = file.rfind( 92 );
-    if ( k1==std::string::npos ) { k1=0; }
+    if ( k1 == std::string::npos ) {
-    if ( k2==std::string::npos ) { k2=0; }
+        k1 = 0;
-    return file.substr(0,std::max(k1,k2));
+    }
+    if ( k2 == std::string::npos ) {
+        k2 = 0;
+    }
+    return file.substr( 0, std::max( k1, k2 ) );
 }
 
 
-// List the timesteps in the given directors (dumps.LBPM)
+// List the timesteps in the given directory (dumps.LBPM)
-std::vector<std::string> IO::readTimesteps( const std::string& filename )
+std::vector<std::string> IO::readTimesteps( const std::string &path, const std::string &format )
 {
-    PROFILE_START("readTimesteps");
+    // Get the name of the summary filename
-    FILE *fid= fopen(filename.c_str(),"rb");
+    std::string filename = path + "/";
-    if ( fid==NULL )
+    if ( format == "old" || format == "new" ) {
-        ERROR("Error opening file");
+        filename += "summary.LBM";
+    } else if ( format == "silo" ) {
+        filename += "LBM.visit";
+    } else if ( format == "auto" ) {
+        bool test_old  = fileExists( path + "/summary.LBM" );
+        bool test_silo = fileExists( path + "/LBM.visit" );
+        if ( test_old && test_silo ) {
+            ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" );
+        } else if ( test_old ) {
+            filename += "summary.LBM";
+        } else if ( test_silo ) {
+            filename += "LBM.visit";
+        } else {
+            ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" );
+        }
+    } else {
+        ERROR( "Unknown format: " + format );
+    }
+    PROFILE_START( "readTimesteps" );
+    // Read the data
+    FILE *fid = fopen( filename.c_str(), "rb" );
+    if ( fid == NULL )
+        ERROR( "Error opening file" );
     std::vector<std::string> timesteps;
     char buf[1000];
-    while (fgets(buf,sizeof(buf),fid) != NULL) {
+    while ( fgets( buf, sizeof( buf ), fid ) != NULL ) {
-        std::string line(buf);
+        std::string line( buf );
-        line.resize(line.size()-1);
+        line.resize( line.size() - 1 );
         auto pos = line.find( "summary.silo" );
         if ( pos != std::string::npos )
-            line.resize(pos);
+            line.resize( pos );
         if ( line.empty() )
             continue;
-        timesteps.push_back(line);
+        timesteps.push_back( line );
     }
-    fclose(fid);
+    fclose( fid );
-    PROFILE_STOP("readTimesteps");
+    PROFILE_STOP( "readTimesteps" );
+    return timesteps;
     return timesteps;
 }
 
 
+// Get the maximum number of domains
+int IO::maxDomains( const std::string &path, const std::string &format, const Utilities::MPI &comm )
+{
+    int rank      = comm.getRank();
+    int n_domains = 0;
+    if ( rank == 0 ) {
+        // Get the timesteps
+        auto timesteps = IO::readTimesteps( path, format );
+        ASSERT( !timesteps.empty() );
+        // Get the database for the first domain
+        auto db = IO::getMeshList( path, timesteps[0] );
+        for ( size_t i = 0; i < db.size(); i++ )
+            n_domains = std::max<int>( n_domains, db[i].domains.size() );
+    }
+    return comm.bcast( n_domains, 0 );
+}
+
+
+// Read the data for the given timestep
+std::vector<IO::MeshDataStruct> IO::readData(
+    const std::string &path, const std::string &timestep, int rank )
+{
+    // Get the mesh databases
+    auto db = IO::getMeshList( path, timestep );
+    // Create the data
+    std::vector<IO::MeshDataStruct> data( db.size() );
+    for ( size_t i = 0; i < data.size(); i++ ) {
+        data[i].precision = IO::DataType::Double;
+        data[i].meshName  = db[i].name;
+        data[i].mesh      = getMesh( path, timestep, db[i], rank );
+        data[i].vars.resize( db[i].variables.size() );
+        for ( size_t j = 0; j < db[i].variables.size(); j++ )
+            data[i].vars[j] = getVariable( path, timestep, db[i], rank, db[i].variables[j].name );
+        INSIST( data[i].check(), "Failed check of " + data[i].meshName );
+    }
+    return data;
+}
+
+
 // Read the list of variables for the given timestep
-std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const std::string& timestep )
+std::vector<IO::MeshDatabase> IO::getMeshList(
+    const std::string &path, const std::string &timestep )
 {
     std::string filename = path + "/" + timestep + "/LBM.summary";
     return IO::read( filename );
@@ -71,270 +150,271 @@ std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const st
 
 
 // Read the given mesh domain
-std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::string& timestep, 
+std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::string &timestep,
-    const IO::MeshDatabase& meshDatabase, int domain )
+    const IO::MeshDatabase &meshDatabase, int domain )
 {
-    PROFILE_START("getMesh");
+    PROFILE_START( "getMesh" );
     std::shared_ptr<IO::Mesh> mesh;
-    if ( meshDatabase.format==1 ) {
+    if ( meshDatabase.format == FileFormat::OLD ) {
         // Old format (binary doubles)
         std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file;
-        FILE *fid = fopen(filename.c_str(),"rb");
+        FILE *fid            = fopen( filename.c_str(), "rb" );
-        INSIST(fid!=NULL,"Error opening file");
+        INSIST( fid != NULL, "Error opening file" );
         fseek( fid, 0, SEEK_END );
-        size_t bytes = ftell(fid);
+        size_t bytes = ftell( fid );
-        size_t N_max = bytes/sizeof(double)+1000;
+        size_t N_max = bytes / sizeof( double ) + 1000;
         double *data = new double[N_max];
-        fseek(fid,0,SEEK_SET);
+        fseek( fid, 0, SEEK_SET );
-        size_t count = fread(data,sizeof(double),N_max,fid);
+        size_t count = fread( data, sizeof( double ), N_max, fid );
-        fclose(fid);
+        fclose( fid );
-        if ( count%3 != 0 )
+        if ( count % 3 != 0 )
-            ERROR("Error reading file");
+            ERROR( "Error reading file" );
-        if ( meshDatabase.type==IO::PointMesh ) {
+        if ( meshDatabase.type == IO::MeshType::PointMesh ) {
-            size_t N = count/3;
+            size_t N = count / 3;
-            std::shared_ptr<PointList> pointlist( new PointList(N) );
+            std::shared_ptr<PointList> pointlist( new PointList( N ) );
-            std::vector<Point>& P = pointlist->points;
+            std::vector<Point> &P = pointlist->points;
-            for (size_t i=0; i<N; i++) {
+            for ( size_t i = 0; i < N; i++ ) {
-                P[i].x = data[3*i+0];
+                P[i].x = data[3 * i + 0];
-                P[i].y = data[3*i+1];
+                P[i].y = data[3 * i + 1];
-                P[i].z = data[3*i+2];
+                P[i].z = data[3 * i + 2];
             }
             mesh = pointlist;
-        } else if ( meshDatabase.type==IO::SurfaceMesh ) {
+        } else if ( meshDatabase.type == IO::MeshType::SurfaceMesh ) {
-            if ( count%9 != 0 )
+            if ( count % 9 != 0 )
-                ERROR("Error reading file (2)");
+                ERROR( "Error reading file (2)" );
-            size_t N_tri = count/9;
+            size_t N_tri = count / 9;
-            std::shared_ptr<TriList> trilist( new TriList(N_tri) );
+            std::shared_ptr<TriList> trilist( new TriList( N_tri ) );
-            std::vector<Point>& A = trilist->A;
+            std::vector<Point> &A = trilist->A;
-            std::vector<Point>& B = trilist->B;
+            std::vector<Point> &B = trilist->B;
-            std::vector<Point>& C = trilist->C;
+            std::vector<Point> &C = trilist->C;
-            for (size_t i=0; i<N_tri; i++) {
+            for ( size_t i = 0; i < N_tri; i++ ) {
-                A[i].x = data[9*i+0];
+                A[i].x = data[9 * i + 0];
-                A[i].y = data[9*i+1];
+                A[i].y = data[9 * i + 1];
-                A[i].z = data[9*i+2];
+                A[i].z = data[9 * i + 2];
-                B[i].x = data[9*i+3];
+                B[i].x = data[9 * i + 3];
-                B[i].y = data[9*i+4];
+                B[i].y = data[9 * i + 4];
-                B[i].z = data[9*i+5];
+                B[i].z = data[9 * i + 5];
-                C[i].x = data[9*i+6];
+                C[i].x = data[9 * i + 6];
-                C[i].y = data[9*i+7];
+                C[i].y = data[9 * i + 7];
-                C[i].z = data[9*i+8];
+                C[i].z = data[9 * i + 8];
             }
             mesh = trilist;
-        } else if ( meshDatabase.type==IO::VolumeMesh ) {
+        } else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) {
             // this was never supported in the old format
             mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
         } else {
-            ERROR("Unknown mesh type");
+            ERROR( "Unknown mesh type" );
         }
-        delete [] data;
+        delete[] data;
-    } else if ( meshDatabase.format==2 ) {
+    } else if ( meshDatabase.format == FileFormat::NEW ||
-        const DatabaseEntry& database = meshDatabase.domains[domain];
+                meshDatabase.format == FileFormat::NEW_SINGLE ) {
-        std::string filename = path + "/" + timestep + "/" + database.file;
+        const DatabaseEntry &database = meshDatabase.domains[domain];
-        FILE *fid = fopen(filename.c_str(),"rb");
+        std::string filename          = path + "/" + timestep + "/" + database.file;
-        fseek(fid,database.offset,SEEK_SET);
+        FILE *fid                     = fopen( filename.c_str(), "rb" );
+        fseek( fid, database.offset, SEEK_SET );
         char line[1000];
-        fgetl(line,1000,fid);
+        fgetl( line, 1000, fid );
-        size_t i1 = find(line,':');
+        size_t i1    = find( line, ':' );
-        size_t i2 = find(&line[i1+1],':')+i1+1;
+        size_t i2    = find( &line[i1 + 1], ':' ) + i1 + 1;
-        size_t bytes = atol(&line[i2+1]);
+        size_t bytes = atol( &line[i2 + 1] );
-        char *data = new char[bytes];
+        char *data   = new char[bytes];
-        size_t count = fread(data,1,bytes,fid);
+        size_t count = fread( data, 1, bytes, fid );
-        fclose(fid);
+        fclose( fid );
-        ASSERT(count==bytes);
+        ASSERT( count == bytes );
-        if ( meshDatabase.meshClass=="PointList" ) {
+        if ( meshDatabase.meshClass == "PointList" ) {
             mesh.reset( new IO::PointList() );
-        } else if ( meshDatabase.meshClass=="TriMesh" ) {
+        } else if ( meshDatabase.meshClass == "TriMesh" ) {
             mesh.reset( new IO::TriMesh() );
-        } else if ( meshDatabase.meshClass=="TriList" ) {
+        } else if ( meshDatabase.meshClass == "TriList" ) {
             mesh.reset( new IO::TriList() );
-        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+        } else if ( meshDatabase.meshClass == "DomainMesh" ) {
             mesh.reset( new IO::DomainMesh() );
         } else {
-            ERROR("Unknown mesh class");
+            ERROR( "Unknown mesh class" );
         }
-        mesh->unpack( std::pair<size_t,void*>(bytes,data) );
+        mesh->unpack( std::pair<size_t, void *>( bytes, data ) );
-        delete [] data;
+        delete[] data;
-    } else if ( meshDatabase.format==4 ) {
+    } else if ( meshDatabase.format == FileFormat::SILO ) {
         // Reading a silo file
 #ifdef USE_SILO
-        const DatabaseEntry& database = meshDatabase.domains[domain];
+        const DatabaseEntry &database = meshDatabase.domains[domain];
-        std::string filename = path + "/" + timestep + "/" + database.file;
+        std::string filename          = path + "/" + timestep + "/" + database.file;
-        auto fid = silo::open( filename, silo::READ );
+        auto fid                      = silo::open( filename, silo::READ );
-        if ( meshDatabase.meshClass=="PointList" ) {
+        if ( meshDatabase.meshClass == "PointList" ) {
             Array<double> coords = silo::readPointMesh<double>( fid, database.name );
-            ASSERT(coords.size(1)==3);
+            ASSERT( coords.size( 1 ) == 3 );
-            std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size(0) ) );
+            std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size( 0 ) ) );
-            for (size_t i=0; i<coords.size(1); i++) {
+            for ( size_t i = 0; i < coords.size( 1 ); i++ ) {
-                mesh2->points[i].x = coords(i,0);
+                mesh2->points[i].x = coords( i, 0 );
-                mesh2->points[i].y = coords(i,1);
+                mesh2->points[i].y = coords( i, 1 );
-                mesh2->points[i].z = coords(i,2);
+                mesh2->points[i].z = coords( i, 2 );
             }
             mesh = mesh2;
-        } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
+        } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
             Array<double> coords;
             Array<int> tri;
             silo::readTriMesh( fid, database.name, coords, tri );
-            ASSERT( tri.size(1)==3 && coords.size(1)==3 );
+            ASSERT( tri.size( 1 ) == 3 && coords.size( 1 ) == 3 );
-            int N_tri = tri.size(0);
+            int N_tri   = tri.size( 0 );
-            int N_point = coords.size(0);
+            int N_point = coords.size( 0 );
             std::shared_ptr<IO::TriMesh> mesh2( new IO::TriMesh( N_tri, N_point ) );
-            for (int i=0; i<N_point; i++) {
+            for ( int i = 0; i < N_point; i++ ) {
-                mesh2->vertices->points[i].x = coords(i,0);
+                mesh2->vertices->points[i].x = coords( i, 0 );
-                mesh2->vertices->points[i].y = coords(i,1);
+                mesh2->vertices->points[i].y = coords( i, 1 );
-                mesh2->vertices->points[i].z = coords(i,2);
+                mesh2->vertices->points[i].z = coords( i, 2 );
             }
-            for (int i=0; i<N_tri; i++) {
+            for ( int i = 0; i < N_tri; i++ ) {
-                mesh2->A[i] = tri(i,0);
+                mesh2->A[i] = tri( i, 0 );
-                mesh2->B[i] = tri(i,1);
+                mesh2->B[i] = tri( i, 1 );
-                mesh2->C[i] = tri(i,2);
+                mesh2->C[i] = tri( i, 2 );
             }
-            if ( meshDatabase.meshClass=="TriMesh" ) {
+            if ( meshDatabase.meshClass == "TriMesh" ) {
                 mesh = mesh2;
-            } else if ( meshDatabase.meshClass=="TriList" ) {
+            } else if ( meshDatabase.meshClass == "TriList" ) {
                 auto trilist = IO::getTriList( std::dynamic_pointer_cast<IO::Mesh>( mesh2 ) );
-                mesh = trilist;
+                mesh         = trilist;
             }
-        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+        } else if ( meshDatabase.meshClass == "DomainMesh" ) {
             std::vector<double> range;
             std::vector<int> N;
             silo::readUniformMesh( fid, database.name, range, N );
-            auto rankinfo = silo::read<int>( fid, database.name+"_rankinfo" );
+            auto rankinfo = silo::read<int>( fid, database.name + "_rankinfo" );
             RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
-            mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) );
+            mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0],
+                range[3] - range[2], range[5] - range[4] ) );
         } else {
-            ERROR("Unknown mesh class");
+            ERROR( "Unknown mesh class" );
         }
         silo::close( fid );
 #else
-        ERROR("Build without silo support");
+        ERROR( "Build without silo support" );
 #endif
     } else {
-        ERROR("Unknown format");
+        ERROR( "Unknown format" );
     }
-    PROFILE_STOP("getMesh");
+    PROFILE_STOP( "getMesh" );
     return mesh;
 }
 
 
 // Read the given variable for the given mesh domain
-std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const std::string& timestep, 
+std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const std::string &timestep,
-    const MeshDatabase& meshDatabase, int domain, const std::string& variable )
+    const MeshDatabase &meshDatabase, int domain, const std::string &variable )
 {
-    std::pair<std::string,std::string> key(meshDatabase.domains[domain].name,variable);
+    std::pair<std::string, std::string> key( meshDatabase.domains[domain].name, variable );
-    std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
+    auto it = meshDatabase.variable_data.find( key );
-    it = meshDatabase.variable_data.find(key);
+    if ( it == meshDatabase.variable_data.end() )
-    if ( it==meshDatabase.variable_data.end() )
         return std::shared_ptr<IO::Variable>();
     std::shared_ptr<IO::Variable> var;
-    if ( meshDatabase.format == 2 ) {
+    if ( meshDatabase.format == FileFormat::NEW || meshDatabase.format == FileFormat::NEW_SINGLE ) {
-        const DatabaseEntry& database = it->second;
+        const DatabaseEntry &database = it->second;
-        std::string filename = path + "/" + timestep + "/" + database.file;
+        std::string filename          = path + "/" + timestep + "/" + database.file;
-        FILE *fid = fopen(filename.c_str(),"rb");
+        FILE *fid                     = fopen( filename.c_str(), "rb" );
-        fseek(fid,database.offset,SEEK_SET);
+        fseek( fid, database.offset, SEEK_SET );
         char line[1000];
-        fgetl(line,1000,fid);
+        fgetl( line, 1000, fid );
-        size_t i1 = find(line,':');
+        size_t i1                       = find( line, ':' );
-        size_t i2 = find(&line[i1+1],':')+i1+1;
+        size_t i2                       = find( &line[i1 + 1], ':' ) + i1 + 1;
-        std::vector<std::string> values = splitList(&line[i2+1],',');
+        std::vector<std::string> values = splitList( &line[i2 + 1], ',' );
-        ASSERT(values.size()==5);
+        ASSERT( values.size() == 5 );
-        int dim = atoi(values[0].c_str());
+        int dim               = atoi( values[0].c_str() );
-        int type = atoi(values[1].c_str());
+        auto type             = values[1];
-        size_t N = atol(values[2].c_str());
+        size_t N              = atol( values[2].c_str() );
-        size_t bytes = atol(values[3].c_str());
+        size_t bytes          = atol( values[3].c_str() );
-        std::string precision  = values[4];
+        std::string precision = values[4];
-        var = std::shared_ptr<IO::Variable>( new IO::Variable() );
+        var                   = std::shared_ptr<IO::Variable>( new IO::Variable() );
-        var->dim = dim;
+        var->dim              = dim;
-        var->type = static_cast<IO::VariableType>(type);
+        var->type             = getVariableType( type );
-        var->name = variable;
+        var->name             = variable;
-        var->data.resize(N*dim);
+        var->data.resize( N, dim );
-        if ( precision=="double" ) {
+        if ( precision == "double" ) {
-            size_t count = fread(var->data.data(),sizeof(double),N*dim,fid);
+            size_t count = fread( var->data.data(), sizeof( double ), N * dim, fid );
-            ASSERT(count*sizeof(double)==bytes);
+            ASSERT( count * sizeof( double ) == bytes );
         } else {
-            ERROR("Format not implimented");
+            ERROR( "Format not implimented" );
         }
-        fclose(fid);
+        fclose( fid );
-    } else if ( meshDatabase.format == 4 ) {
+    } else if ( meshDatabase.format == FileFormat::SILO ) {
         // Reading a silo file
 #ifdef USE_SILO
-        const auto& database = meshDatabase.domains[domain];
+        const auto &database  = meshDatabase.domains[domain];
         auto variableDatabase = meshDatabase.getVariableDatabase( variable );
-        std::string filename = path + "/" + timestep + "/" + database.file;
+        std::string filename  = path + "/" + timestep + "/" + database.file;
-        auto fid = silo::open( filename, silo::READ );
+        auto fid              = silo::open( filename, silo::READ );
         var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
-        if ( meshDatabase.meshClass=="PointList" ) {
+        if ( meshDatabase.meshClass == "PointList" ) {
             var->data = silo::readPointMeshVariable<double>( fid, variable );
-        } else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
+        } else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
             var->data = silo::readTriMeshVariable<double>( fid, variable );
-        } else if ( meshDatabase.meshClass=="DomainMesh" ) {
+        } else if ( meshDatabase.meshClass == "DomainMesh" ) {
             var->data = silo::readUniformMeshVariable<double>( fid, variable );
         } else {
-            ERROR("Unknown mesh class");
+            ERROR( "Unknown mesh class" );
         }
         silo::close( fid );
 #else
-        ERROR("Build without silo support");
+        ERROR( "Build without silo support" );
 #endif
 
     } else {
-        ERROR("Unknown format");
+        ERROR( "Unknown format" );
     }
     return var;
 }
 
 
 /****************************************************
-* Reformat the variable to match the mesh           *
+ * Reformat the variable to match the mesh           *
-****************************************************/
+ ****************************************************/
-void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
+void IO::reformatVariable( const IO::Mesh &mesh, IO::Variable &var )
 {
     if ( mesh.className() == "DomainMesh" ) {
-        const IO::DomainMesh& mesh2 = dynamic_cast<const IO::DomainMesh&>( mesh );
+        const IO::DomainMesh &mesh2 = dynamic_cast<const IO::DomainMesh &>( mesh );
         if ( var.type == VariableType::NodeVariable ) {
-            size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1));
+            size_t N2 =
-            ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() );
+                var.data.length() / ( ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) );
-            var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } );
+            ASSERT(
+                ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) * N2 == var.data.length() );
+            var.data.reshape(
+                { (size_t) mesh2.nx + 1, (size_t) mesh2.ny + 1, (size_t) mesh2.nz + 1, N2 } );
         } else if ( var.type == VariableType::EdgeVariable ) {
-            ERROR("Not finished");
+            ERROR( "Not finished" );
         } else if ( var.type == VariableType::SurfaceVariable ) {
-            ERROR("Not finished");
+            ERROR( "Not finished" );
         } else if ( var.type == VariableType::VolumeVariable ) {
-            size_t N2 = var.data.length() / (mesh2.nx*mesh2.ny*mesh2.nz);
+            size_t N2 = var.data.length() / ( mesh2.nx * mesh2.ny * mesh2.nz );
-            ASSERT( mesh2.nx*mesh2.ny*mesh2.nz*N2 == var.data.length() );
+            ASSERT( mesh2.nx * mesh2.ny * mesh2.nz * N2 == var.data.length() );
             var.data.reshape( { (size_t) mesh2.nx, (size_t) mesh2.ny, (size_t) mesh2.nz, N2 } );
         } else {
-            ERROR("Invalid variable type");
+            ERROR( "Invalid variable type" );
         }
     } else if ( mesh.className() == "PointList" ) {
-        const IO::PointList& mesh2 = dynamic_cast<const IO::PointList&>( mesh );
+        const IO::PointList &mesh2 = dynamic_cast<const IO::PointList &>( mesh );
-        size_t N = mesh2.points.size();
+        size_t N                   = mesh2.points.size();
-        size_t N_var = var.data.length()/N;
+        size_t N_var               = var.data.length() / N;
-        ASSERT( N*N_var == var.data.length() );
+        ASSERT( N * N_var == var.data.length() );
         var.data.reshape( { N, N_var } );
-    } else if ( mesh.className()=="TriMesh" || mesh.className() == "TriList" ) {
+    } else if ( mesh.className() == "TriMesh" || mesh.className() == "TriList" ) {
-        std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh*>(&mesh), []( void* ) {} );
+        std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh *>( &mesh ), []( void * ) {} );
         std::shared_ptr<TriMesh> mesh2 = getTriMesh( mesh_ptr );
         if ( var.type == VariableType::NodeVariable ) {
-            size_t N = mesh2->vertices->points.size();
+            size_t N     = mesh2->vertices->points.size();
-            size_t N_var = var.data.length()/N;
+            size_t N_var = var.data.length() / N;
-            ASSERT( N*N_var == var.data.length() );
+            ASSERT( N * N_var == var.data.length() );
             var.data.reshape( { N, N_var } );
         } else if ( var.type == VariableType::EdgeVariable ) {
-            ERROR("Not finished");
+            ERROR( "Not finished" );
         } else if ( var.type == VariableType::SurfaceVariable ) {
-            ERROR("Not finished");
+            ERROR( "Not finished" );
         } else if ( var.type == VariableType::VolumeVariable ) {
-            size_t N = mesh2->A.size();
+            size_t N     = mesh2->A.size();
-            size_t N_var = var.data.length()/N;
+            size_t N_var = var.data.length() / N;
-            ASSERT( N*N_var == var.data.length() );
+            ASSERT( N * N_var == var.data.length() );
             var.data.reshape( { N, N_var } );
         } else {
-            ERROR("Invalid variable type");
+            ERROR( "Invalid variable type" );
         }
     } else {
-        ERROR("Unknown mesh type");
+        ERROR( "Unknown mesh type" );
     }
 }
-
-
-

IO/Reader.h

@@ -14,20 +14,59 @@ namespace IO {
 
 
 //! Get the path to a file
-std::string getPath( const std::string& filename );
+std::string getPath( const std::string &filename );
 
 
-//! List the timesteps in the given directors (dumps.LBPM)
+/*!
-std::vector<std::string> readTimesteps( const std::string& filename );
+ * @brief  Get the maximum number of domains written
+ * @details  This function reads the summary files to determine the maximum
+ *    number of domains in the output.
+ * @param[in] path          The path to use for reading
+ * @param[in] format        The data format to use:
+ *                              old - Old mesh format (provided for backward compatibility)
+ *                              new - New format, 1 file/process
+ *                              silo - Silo
+ *                              auto - Auto-determin the format
+ * @param[in] comm          Optional comm to use to reduce IO load by
+ *                          reading on rank 0 and then communicating the result
+ */
+int maxDomains( const std::string &path, const std::string &format = "auto",
+    const Utilities::MPI &comm = MPI_COMM_SELF );
+
+
+/*!
+ * @brief  Read the timestep list
+ * @details  This function reads the timestep list from the summary file.
+ * @param[in] path          The path to use for reading
+ * @param[in] format        The data format to use:
+ *                              old - Old mesh format (provided for backward compatibility)
+ *                              new - New format, 1 file/process
+ *                              silo - Silo
+ *                              auto - Auto-determin the format
+ * @return append           Append any existing data (default is false)
+ */
+std::vector<std::string> readTimesteps(
+    const std::string &path, const std::string &format = "auto" );
+
+
+/*!
+ * @brief  Read the data for the timestep
+ * @details  This function reads the mesh and variable data provided for the given timestep.
+ * @param[in] path          The path to use for reading
+ * @param[in] timestep      The timestep iteration
+ * @param[in] domain        The desired domain to read
+ */
+std::vector<IO::MeshDataStruct> readData(
+    const std::string &path, const std::string &timestep, int domain );
 
 
 //! Read the list of mesh databases for the given timestep
-std::vector<IO::MeshDatabase> getMeshList( const std::string& path, const std::string& timestep );
+std::vector<IO::MeshDatabase> getMeshList( const std::string &path, const std::string &timestep );
 
 
 //! Read the given mesh domain
-std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& timestep, 
+std::shared_ptr<IO::Mesh> getMesh( const std::string &path, const std::string &timestep,
-    const MeshDatabase& meshDatabase, int domain );
+    const MeshDatabase &meshDatabase, int domain );
 
 
 /*!
@@ -40,8 +79,8 @@ std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& t
  * @param[in] variable      The variable name to read
  * @return                  Returns the variable data as a linear array
  */
-std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::string& timestep, 
+std::shared_ptr<IO::Variable> getVariable( const std::string &path, const std::string &timestep,
-    const MeshDatabase& meshDatabase, int domain, const std::string& variable );
+    const MeshDatabase &meshDatabase, int domain, const std::string &variable );
 
 
 /*!
@@ -50,9 +89,9 @@ std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::s
  * @param[in] mesh          The underlying mesh
  * @param[in/out] variable  The variable name to read
  */
-void reformatVariable( const IO::Mesh& mesh, IO::Variable& var );
+void reformatVariable( const IO::Mesh &mesh, IO::Variable &var );
 
 
-} // IO namespace
+} // namespace IO
 
 #endif

IO/Writer.cpp

@@ -1,28 +1,69 @@
 #include "IO/Writer.h"
-#include "IO/MeshDatabase.h"
 #include "IO/IOHelpers.h"
+#include "IO/MeshDatabase.h"
 #include "IO/silo.h"
 #include "common/MPI.h"
 #include "common/Utilities.h"
 
-#include <sys/stat.h>
 #include <algorithm>
-#include <vector>
-#include <set>
 #include <memory>
+#include <set>
+#include <sys/stat.h>
+#include <vector>
 
 
-
+enum class Format { OLD, NEW, SILO, UNKNOWN };
-enum class Format { OLD, NEW, SILO, UNKNOWN }; 
-
 
 
 /****************************************************
-* Initialize the writer                             *
+ * Recursively create the subdirectory               *
-****************************************************/
+ ****************************************************/
+static void recursiveMkdir( const std::string &path, mode_t mode )
+{
+    // Iterate through the root directories until we create the desired path
+    for ( size_t pos = 0; pos < path.size(); ) {
+        // slide backwards in string until next slash found
+        pos++;
+        for ( ; pos < path.size(); pos++ ) {
+            if ( path[pos] == '/' || path[pos] == 92 )
+                break;
+        }
+        // Create the temporary path
+        auto path2 = path.substr( 0, pos );
+        // Check if the temporary path exists
+        struct stat status;
+        int result = stat( path2.data(), &status );
+        if ( result == 0 ) {
+            // if there is a part of the path that already exists make sure it is really a directory
+            if ( !S_ISDIR( status.st_mode ) ) {
+                ERROR(
+                    "Error in recursiveMkdir...\n"
+                    "    Cannot create directories in path = " +
+                    path +
+                    "\n    because some intermediate item in path exists and is NOT a directory" );
+            }
+            continue;
+        }
+        // Create the directory and test the result
+        result = mkdir( path2.data(), mode );
+        if ( result != 0 ) {
+            // Maybe another rank created the directory, check
+            int result = stat( path2.data(), &status );
+            if ( result != 0 && !S_ISDIR( status.st_mode ) )
+                ERROR( "Error in Utilities::recursiveMkdir...\n"
+                       "    Cannot create directory  = " +
+                       path2 );
+        }
+    }
+}
+
+
+/****************************************************
+ * Initialize the writer                             *
+ ****************************************************/
 static std::string global_IO_path;
 static Format global_IO_format = Format::UNKNOWN;
-void IO::initialize( const std::string& path, const std::string& format, bool append )
+void IO::initialize( const std::string &path, const std::string &format, bool append )
 {
     if ( path.empty() )
         global_IO_path = ".";
@@ -35,161 +76,168 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
     else if ( format == "silo" )
         global_IO_format = Format::SILO;
     else
-        ERROR("Unknown format");
+        ERROR( "Unknown format" );
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
+    int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
-    if ( !append && rank==0 ) {
+    if ( !append && rank == 0 ) {
-        mkdir(path.c_str(),S_IRWXU|S_IRGRP);
+        recursiveMkdir( path, S_IRWXU | S_IRGRP );
         std::string filename;
-        if ( global_IO_format==Format::OLD || global_IO_format==Format::NEW )
+        if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW )
             filename = global_IO_path + "/summary.LBM";
-        else if ( global_IO_format==Format::SILO )
+        else if ( global_IO_format == Format::SILO )
             filename = global_IO_path + "/LBM.visit";
         else
-            ERROR("Unknown format");
+            ERROR( "Unknown format" );
-        auto fid = fopen(filename.c_str(),"wb");
+        auto fid = fopen( filename.c_str(), "wb" );
-        fclose(fid);
+        fclose( fid );
     }
 }
 
 
 // Write the mesh data in the original format
-static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
+static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
+    const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, int rank )
 {
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
     std::vector<IO::MeshDatabase> meshes_written;
-    for (size_t i=0; i<meshData.size(); i++) {
+    for ( size_t i = 0; i < meshData.size(); i++ ) {
         char domainname[100], filename[100], fullpath[200];
-        sprintf(domainname,"%05i",rank);
+        sprintf( domainname, "%05i", rank );
-        sprintf(filename,"%s.%05i",meshData[i].meshName.c_str(),rank);
+        sprintf( filename, "%s.%05i", meshData[i].meshName.c_str(), rank );
-	    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+        sprintf( fullpath, "%s/%s", path.c_str(), filename );
-        FILE *fid = fopen(fullpath,"wb");
+        FILE *fid = fopen( fullpath, "wb" );
-        INSIST(fid!=NULL,std::string("Error opening file: ")+fullpath);
+        INSIST( fid != NULL, std::string( "Error opening file: " ) + fullpath );
         std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
         IO::MeshDatabase mesh_entry;
-        mesh_entry.name = meshData[i].meshName;
+        mesh_entry.name      = meshData[i].meshName;
-        mesh_entry.type = meshType(*mesh);
+        mesh_entry.type      = meshType( *mesh );
         mesh_entry.meshClass = meshData[i].mesh->className();
-        mesh_entry.format = 1;
+        mesh_entry.format    = IO::FileFormat::OLD;
         IO::DatabaseEntry domain;
-        domain.name = domainname;
+        domain.name   = domainname;
-        domain.file = filename;
+        domain.file   = filename;
         domain.offset = 0;
-        mesh_entry.domains.push_back(domain);
+        mesh_entry.domains.push_back( domain );
         if ( !meshData[i].vars.empty() ) {
-            printf("Warning: variables are not supported with this format\n");
+            printf( "Warning: variables are not supported with this format (original)\n" );
-            //for (size_t j=0; j<meshData[i].vars.size(); j++)
+            // for (size_t j=0; j<meshData[i].vars.size(); j++)
             //    mesh_entry.variables.push_back( meshData[i].vars[j]->name );
         }
         const std::string meshClass = mesh->className();
-        if ( meshClass=="PointList" ) {
+        if ( meshClass == "PointList" ) {
             // List of points
-            std::shared_ptr<IO::PointList> pointlist = std::dynamic_pointer_cast<IO::PointList>(mesh);
+            std::shared_ptr<IO::PointList> pointlist =
-            const std::vector<Point>& P = pointlist->points;
+                std::dynamic_pointer_cast<IO::PointList>( mesh );
-            for (size_t i=0; i<P.size(); i++) {
+            const std::vector<Point> &P = pointlist->points;
+            for ( size_t i = 0; i < P.size(); i++ ) {
                 double x[3];
-                x[0] = P[i].x;  x[1] = P[i].y;  x[2] = P[i].z;
+                x[0] = P[i].x;
-                fwrite(x,sizeof(double),3,fid);
+                x[1] = P[i].y;
+                x[2] = P[i].z;
+                fwrite( x, sizeof( double ), 3, fid );
             }
-        } else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
+        } else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
             // Triangle mesh
-            std::shared_ptr<IO::TriList> trilist = IO::getTriList(mesh);
+            std::shared_ptr<IO::TriList> trilist = IO::getTriList( mesh );
-            const std::vector<Point>& A = trilist->A;
+            const std::vector<Point> &A          = trilist->A;
-            const std::vector<Point>& B = trilist->B;
+            const std::vector<Point> &B          = trilist->B;
-            const std::vector<Point>& C = trilist->C;
+            const std::vector<Point> &C          = trilist->C;
-            for (size_t i=0; i<A.size(); i++) {
+            for ( size_t i = 0; i < A.size(); i++ ) {
                 double tri[9];
-                tri[0] = A[i].x;  tri[1] = A[i].y;  tri[2] = A[i].z;
+                tri[0] = A[i].x;
-                tri[3] = B[i].x;  tri[4] = B[i].y;  tri[5] = B[i].z;
+                tri[1] = A[i].y;
-                tri[6] = C[i].x;  tri[7] = C[i].y;  tri[8] = C[i].z;
+                tri[2] = A[i].z;
-                fwrite(tri,sizeof(double),9,fid);
+                tri[3] = B[i].x;
+                tri[4] = B[i].y;
+                tri[5] = B[i].z;
+                tri[6] = C[i].x;
+                tri[7] = C[i].y;
+                tri[8] = C[i].z;
+                fwrite( tri, sizeof( double ), 9, fid );
             }
-        } else if ( meshClass=="DomainMesh" ) {
+        } else if ( meshClass == "DomainMesh" ) {
             // This format was never supported with the old format
         } else {
-            ERROR("Unknown mesh");
+            ERROR( "Unknown mesh" );
         }
-        fclose(fid);
+        fclose( fid );
         std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() );
-        mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() );
+        mesh_entry.variables.erase(
-        meshes_written.push_back(mesh_entry);
+            std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ),
+            mesh_entry.variables.end() );
+        meshes_written.push_back( mesh_entry );
     }
     return meshes_written;
 }
 
 
 // Create the database entry for the mesh data
-static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
+static IO::MeshDatabase getDatabase(
+    const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
 {
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
     char domainname[100];
-    sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank);
+    sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank );
     // Create the MeshDatabase
     IO::MeshDatabase database;
-    database.name = mesh.meshName;
+    database.name      = mesh.meshName;
-    database.type = meshType(*(mesh.mesh));
+    database.type      = meshType( *( mesh.mesh ) );
     database.meshClass = mesh.mesh->className();
-    database.format = format;
+    database.format    = format;
     // Write the mesh
     IO::DatabaseEntry domain;
-    domain.name = domainname;
+    domain.name   = domainname;
-    domain.file = filename;
+    domain.file   = filename;
     domain.offset = -1;
-    database.domains.push_back(domain);
+    database.domains.push_back( domain );
     // Write the variables
-    for (size_t i=0; i<mesh.vars.size(); i++) {
+    for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
         // Add basic variable info
         IO::VariableDatabase info;
         info.name = mesh.vars[i]->name;
         info.type = mesh.vars[i]->type;
-        info.dim = mesh.vars[i]->dim;
+        info.dim  = mesh.vars[i]->dim;
-        database.variables.push_back(info);
+        database.variables.push_back( info );
         // Add domain variable info
         IO::DatabaseEntry variable;
-        variable.name = mesh.vars[i]->name;
+        variable.name   = mesh.vars[i]->name;
-        variable.file = filename;
+        variable.file   = filename;
         variable.offset = -1;
-        std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
+        std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
-        database.variable_data.insert( 
+        database.variable_data.insert(
-            std::pair<std::pair<std::string,std::string>,IO::DatabaseEntry>(key,variable) );
+            std::pair<std::pair<std::string, std::string>, IO::DatabaseEntry>( key, variable ) );
     }
     return database;
 }
 
 
 // Write a mesh (and variables) to a file
-static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
+static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
-    const IO::MeshDataStruct& mesh, int format )
+    const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
 {
     const int level = 0;
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
     // Create the MeshDatabase
-    IO::MeshDatabase database = getDatabase( filename, mesh, format );
+    IO::MeshDatabase database = getDatabase( filename, mesh, format, rank );
     // Write the mesh
-    IO::DatabaseEntry& domain = database.domains[0];
+    IO::DatabaseEntry &domain      = database.domains[0];
-    domain.offset = ftell(fid);
+    domain.offset                  = ftell( fid );
-    std::pair<size_t,void*> data = mesh.mesh->pack(level);
+    std::pair<size_t, void *> data = mesh.mesh->pack( level );
-    fprintf(fid,"Mesh: %s-%05i: %lu\n",mesh.meshName.c_str(),rank,data.first);
+    fprintf( fid, "Mesh: %s-%05i: %lu\n", mesh.meshName.c_str(), rank, data.first );
-    fwrite(data.second,1,data.first,fid);
+    fwrite( data.second, 1, data.first, fid );
-    fprintf(fid,"\n");
+    fprintf( fid, "\n" );
-    delete [] (char*) data.second;
+    delete[]( char * ) data.second;
     // Write the variables
-    for (size_t i=0; i<mesh.vars.size(); i++) {
+    for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
-        std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
+        ASSERT( mesh.vars[i]->type != IO::VariableType::NullVariable );
-        IO::DatabaseEntry& variable = database.variable_data[key];
+        std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
-        variable.offset = ftell(fid);
+        auto &variable  = database.variable_data[key];
-        int dim = mesh.vars[i]->dim;
+        variable.offset = ftell( fid );
-        int type = static_cast<int>(mesh.vars[i]->type);
+        int dim         = mesh.vars[i]->dim;
-        size_t N = mesh.vars[i]->data.length();
+        auto type       = getString( mesh.vars[i]->type );
-        if ( type == static_cast<int>(IO::VariableType::NullVariable) ) {
+        size_t N        = mesh.vars[i]->data.length();
-            ERROR("Variable type not set");
+        size_t N_mesh   = mesh.mesh->numberPointsVar( mesh.vars[i]->type );
-        }
+        ASSERT( N == dim * N_mesh );
-        size_t N_mesh = mesh.mesh->numberPointsVar(mesh.vars[i]->type);
+        fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank,
-        ASSERT(N==dim*N_mesh);
+            variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) );
-        fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n",
+        fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid );
-            database.name.c_str(), rank, variable.name.c_str(),
+        fprintf( fid, "\n" );
-            dim, type, N_mesh, N*sizeof(double) );
-        fwrite(mesh.vars[i]->data.data(),sizeof(double),N,fid);
-        fprintf(fid,"\n");
     }
     return database;
 }
@@ -198,72 +246,74 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
 #ifdef USE_SILO
 // Write a PointList mesh (and variables) to a file
 template<class TYPE>
-static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname )
+static void writeSiloPointMesh(
+    DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
 {
-    const auto& points = mesh.getPoints();
+    const auto &points = mesh.getPoints();
-    std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
+    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
-    for (size_t i=0; i<x.size(); i++) {
+    for ( size_t i = 0; i < x.size(); i++ ) {
         x[i] = points[i].x;
         y[i] = points[i].y;
         z[i] = points[i].z;
     }
     const TYPE *coords[] = { x.data(), y.data(), z.data() };
-    silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
+    IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
 }
-static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+static void writeSiloPointList(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
 {
-    const IO::PointList& mesh = dynamic_cast<IO::PointList&>( *meshData.mesh );
+    const IO::PointList &mesh  = dynamic_cast<IO::PointList &>( *meshData.mesh );
     const std::string meshname = database.domains[0].name;
     if ( meshData.precision == IO::DataType::Double ) {
         writeSiloPointMesh<double>( fid, mesh, meshname );
     } else if ( meshData.precision == IO::DataType::Float ) {
         writeSiloPointMesh<float>( fid, mesh, meshname );
     } else {
-        ERROR("Unsupported format");
+        ERROR( "Unsupported format" );
     }
-    const auto& points = mesh.getPoints();
+    const auto &points = mesh.getPoints();
-    std::vector<double> x(points.size()), y(points.size()), z(points.size());
+    std::vector<double> x( points.size() ), y( points.size() ), z( points.size() );
-    for (size_t i=0; i<x.size(); i++) {
+    for ( size_t i = 0; i < x.size(); i++ ) {
         x[i] = points[i].x;
         y[i] = points[i].y;
         z[i] = points[i].z;
     }
     const double *coords[] = { x.data(), y.data(), z.data() };
-    silo::writePointMesh( fid, meshname, 3, points.size(), coords );
+    IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
-    for (size_t i=0; i<meshData.vars.size(); i++) {
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
-        const IO::Variable& var = *meshData.vars[i];
+        const IO::Variable &var = *meshData.vars[i];
         if ( var.precision == IO::DataType::Double ) {
-            silo::writePointMeshVariable( fid, meshname, var.name, var.data );
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
         } else if ( var.precision == IO::DataType::Float ) {
             Array<float> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
         } else if ( var.precision == IO::DataType::Int ) {
             Array<int> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writePointMeshVariable( fid, meshname, var.name, data2 );
+            IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
         } else {
-            ERROR("Unsupported format");
+            ERROR( "Unsupported format" );
         }
     }
 }
 // Write a TriMesh mesh (and variables) to a file
 template<class TYPE>
-static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::string& meshname )
+static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname )
 {
-    const auto& points = mesh.vertices->getPoints();
+    const auto &points = mesh.vertices->getPoints();
-    std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
+    std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
-    for (size_t i=0; i<x.size(); i++) {
+    for ( size_t i = 0; i < x.size(); i++ ) {
         x[i] = points[i].x;
         y[i] = points[i].y;
         z[i] = points[i].z;
     }
     const TYPE *coords[] = { x.data(), y.data(), z.data() };
-    const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
+    const int *tri[]     = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
-    silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
+    IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
 }
-static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
+static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
-    const IO::TriMesh& mesh, IO::MeshDatabase database )
+    const IO::TriMesh &mesh, IO::MeshDatabase database )
 {
     const std::string meshname = database.domains[0].name;
     if ( meshData.precision == IO::DataType::Double ) {
@@ -271,238 +321,240 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
     } else if ( meshData.precision == IO::DataType::Float ) {
         writeSiloTriMesh<float>( fid, mesh, meshname );
     } else {
-        ERROR("Unsupported format");
+        ERROR( "Unsupported format" );
     }
-    for (size_t i=0; i<meshData.vars.size(); i++) {
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
-        const IO::Variable& var = *meshData.vars[i];
+        const IO::Variable &var = *meshData.vars[i];
-        auto type = static_cast<silo::VariableType>( var.type );
         if ( var.precision == IO::DataType::Double ) {
-            silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type );
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
         } else if ( var.precision == IO::DataType::Float ) {
             Array<float> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
         } else if ( var.precision == IO::DataType::Int ) {
             Array<int> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
+            IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
         } else {
-            ERROR("Unsupported format");
+            ERROR( "Unsupported format" );
         }
     }
 }
-static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+static void writeSiloTriMesh(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
 {
-    const IO::TriMesh& mesh = dynamic_cast<IO::TriMesh&>( *meshData.mesh );
+    const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
     writeSiloTriMesh2( fid, meshData, mesh, database );
 }
-static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+static void writeSiloTriList(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
 {
     auto mesh = getTriMesh( meshData.mesh );
     writeSiloTriMesh2( fid, meshData, *mesh, database );
 }
 // Write a DomainMesh mesh (and variables) to a file
-static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
+static void writeSiloDomainMesh(
+    DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
 {
-    const IO::DomainMesh& mesh = dynamic_cast<IO::DomainMesh&>( *meshData.mesh );
+    const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
     RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
-    std::array<double,6> range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx, 
+    std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
-                                   info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny,
+        ( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
-                                   info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz };
+        ( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
-    std::array<int,3> N = { mesh.nx, mesh.ny, mesh.nz };
+        ( info.kz + 1 ) * mesh.Lz / info.nz };
-    auto meshname = database.domains[0].name;
+    std::array<int, 3> N        = { mesh.nx, mesh.ny, mesh.nz };
-    silo::writeUniformMesh<3>( fid, meshname, range, N );
+    auto meshname               = database.domains[0].name;
-    silo::write<int>( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
+    IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
-    for (size_t i=0; i<meshData.vars.size(); i++) {
+    IO::silo::write<int>(
-        const auto& var = *meshData.vars[i];
+        fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
-        auto type = static_cast<silo::VariableType>( var.type );
+    for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
+        const auto &var = *meshData.vars[i];
         if ( var.precision == IO::DataType::Double ) {
-            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type );
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
         } else if ( var.precision == IO::DataType::Float ) {
             Array<float> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
         } else if ( var.precision == IO::DataType::Int ) {
             Array<int> data2( var.data.size() );
             data2.copy( var.data );
-            silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
+            IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
         } else {
-            ERROR("Unsupported format");
+            ERROR( "Unsupported format" );
         }
     }
 }
 // Write a mesh (and variables) to a file
-static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename,
+static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
-    const IO::MeshDataStruct& mesh, int format )
+    const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
 {
     // Create the MeshDatabase
-    auto database = getDatabase( filename, mesh, format );
+    auto database = getDatabase( filename, mesh, format, rank );
-    if ( database.meshClass=="PointList" ) {
+    if ( database.meshClass == "PointList" ) {
         writeSiloPointList( fid, mesh, database );
-    } else if ( database.meshClass=="TriMesh" ) {
+    } else if ( database.meshClass == "TriMesh" ) {
         writeSiloTriMesh( fid, mesh, database );
-    } else if ( database.meshClass=="TriList" ) {
+    } else if ( database.meshClass == "TriList" ) {
         writeSiloTriList( fid, mesh, database );
-    } else if ( database.meshClass=="DomainMesh" ) {
+    } else if ( database.meshClass == "DomainMesh" ) {
         writeSiloDomainMesh( fid, mesh, database );
     } else {
-        ERROR("Unknown mesh class");
+        ERROR( "Unknown mesh class" );
     }
     return database;
 }
 // Write the summary file for silo
-std::pair<int,int> getSiloMeshType( const std::string& meshClass )
+std::pair<int, int> getSiloMeshType( const std::string &meshClass )
 {
     int meshType = 0;
-    int varType = 0;
+    int varType  = 0;
-    if ( meshClass=="PointList" ) {
+    if ( meshClass == "PointList" ) {
         meshType = DB_POINTMESH;
         varType  = DB_POINTVAR;
-    } else if ( meshClass=="TriMesh" ) {
+    } else if ( meshClass == "TriMesh" ) {
         meshType = DB_UCDMESH;
         varType  = DB_UCDVAR;
-    } else if ( meshClass=="TriList" ) {
+    } else if ( meshClass == "TriList" ) {
         meshType = DB_UCDMESH;
         varType  = DB_UCDVAR;
-    } else if ( meshClass=="DomainMesh" ) {
+    } else if ( meshClass == "DomainMesh" ) {
         meshType = DB_QUAD_RECT;
         varType  = DB_QUADVAR;
     } else {
-        ERROR("Unknown mesh class");
+        ERROR( "Unknown mesh class" );
     }
     return std::make_pair( meshType, varType );
 }
-void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, const std::string& filename )
+void writeSiloSummary(
+    const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
 {
-    auto fid = silo::open( filename, silo::CREATE );
+    auto fid = IO::silo::open( filename, IO::silo::CREATE );
-    for ( const auto& data : meshes_written ) {
+    for ( const auto &data : meshes_written ) {
         auto type = getSiloMeshType( data.meshClass );
         std::vector<int> meshTypes( data.domains.size(), type.first );
         std::vector<int> varTypes( data.domains.size(), type.second );
         std::vector<std::string> meshNames;
-        for ( const auto& tmp : data.domains )
+        for ( const auto &tmp : data.domains )
             meshNames.push_back( tmp.file + ":" + tmp.name );
-        silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
+        IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
-        for (const auto& variable : data.variables ) {
+        for ( const auto &variable : data.variables ) {
             std::vector<std::string> varnames;
-            for ( const auto& tmp : data.domains )
+            for ( const auto &tmp : data.domains )
                 varnames.push_back( tmp.file + ":" + variable.name );
-            silo::writeMultiVar( fid, variable.name, varnames, varTypes );
+            IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
         }
     }
-    silo::close( fid );
+    IO::silo::close( fid );
 }
 #endif
 
 
 // Write the mesh data in the new format
-static std::vector<IO::MeshDatabase> writeMeshesNewFormat( 
+static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
-    const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
+    const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
+    int rank )
 {
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
     std::vector<IO::MeshDatabase> meshes_written;
     char filename[100], fullpath[200];
-    sprintf(filename,"%05i",rank);
+    sprintf( filename, "%05i", rank );
-    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+    sprintf( fullpath, "%s/%s", path.c_str(), filename );
-    FILE *fid = fopen(fullpath,"wb");
+    FILE *fid = fopen( fullpath, "wb" );
-    for (size_t i=0; i<meshData.size(); i++) {
+    for ( size_t i = 0; i < meshData.size(); i++ ) {
         std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
-        meshes_written.push_back( write_domain(fid,filename,meshData[i],format) );
+        meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) );
     }
-    fclose(fid);
+    fclose( fid );
     return meshes_written;
 }
 
 
 // Write the mesh data to silo
-static std::vector<IO::MeshDatabase> writeMeshesSilo( 
+static std::vector<IO::MeshDatabase> writeMeshesSilo(
-    const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
+    const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
+    int rank )
 {
 #ifdef USE_SILO
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
     std::vector<IO::MeshDatabase> meshes_written;
     char filename[100], fullpath[200];
-    sprintf(filename,"%05i.silo",rank);
+    sprintf( filename, "%05i.silo", rank );
-    sprintf(fullpath,"%s/%s",path.c_str(),filename);
+    sprintf( fullpath, "%s/%s", path.c_str(), filename );
-    auto fid = silo::open( fullpath, silo::CREATE );
+    auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
-    for (size_t i=0; i<meshData.size(); i++) {
+    for ( size_t i = 0; i < meshData.size(); i++ ) {
         auto mesh = meshData[i].mesh;
-        meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) );
+        meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
     }
-    silo::close( fid );
+    IO::silo::close( fid );
     return meshes_written;
 #else
-    ERROR("Application built without silo support");
+    NULL_USE( meshData );
+    NULL_USE( path );
+    NULL_USE( format );
+    NULL_USE( rank );
+    ERROR( "Application built without silo support" );
     return std::vector<IO::MeshDatabase>();
 #endif
-}        
+}
 
 
 /****************************************************
-* Write the mesh data                               *
+ * Write the mesh data                               *
-****************************************************/
+ ****************************************************/
-void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
+void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
+    const Utilities::MPI &comm )
 {
     if ( global_IO_path.empty() )
-        IO::initialize( );
+        IO::initialize();
-    PROFILE_START("writeData");
+    PROFILE_START( "writeData" );
-    int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
+    int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
     // Check the meshData before writing
-    for ( const auto& data : meshData ) {
+    for ( const auto &data : meshData )
-        if ( !data.check() )
+        ASSERT( data.check() );
-            ERROR("Error in meshData");
-    }
     // Create the output directory
     std::string path = global_IO_path + "/" + subdir;
-    if ( rank == 0 ) {
+    recursiveMkdir( path, S_IRWXU | S_IRGRP );
-        mkdir(path.c_str(),S_IRWXU|S_IRGRP);
-    }
-    comm.barrier();
     // Write the mesh files
     std::vector<IO::MeshDatabase> meshes_written;
     if ( global_IO_format == Format::OLD ) {
         // Write the original triangle format
-        meshes_written = writeMeshesOrigFormat( meshData, path );
+        meshes_written = writeMeshesOrigFormat( meshData, path, rank );
     } else if ( global_IO_format == Format::NEW ) {
         // Write the new format (double precision)
-        meshes_written = writeMeshesNewFormat( meshData, path, 2 );
+        meshes_written = writeMeshesNewFormat( meshData, path, IO::FileFormat::NEW, rank );
     } else if ( global_IO_format == Format::SILO ) {
         // Write silo
-        meshes_written = writeMeshesSilo( meshData, path, 4 );
+        meshes_written = writeMeshesSilo( meshData, path, IO::FileFormat::SILO, rank );
     } else {
-        ERROR("Unknown format");
+        ERROR( "Unknown format" );
     }
     // Gather a complete list of files on rank 0
-    meshes_written = gatherAll(meshes_written,comm);
+    meshes_written = gatherAll( meshes_written, comm );
     // Write the summary files
     if ( rank == 0 ) {
         // Write the summary file for the current timestep
         char filename[200];
-        sprintf(filename,"%s/LBM.summary",path.c_str());
+        sprintf( filename, "%s/LBM.summary", path.c_str() );
-        write(meshes_written,filename);
+        write( meshes_written, filename );
-        // Write summary silo file if needed
+// Write summary silo file if needed
-        #ifdef USE_SILO
+#ifdef USE_SILO
         if ( global_IO_format == Format::SILO ) {
-            sprintf(filename,"%s/summary.silo",path.c_str());
+            sprintf( filename, "%s/summary.silo", path.c_str() );
-            writeSiloSummary(meshes_written,filename);
+            writeSiloSummary( meshes_written, filename );
         }
-        #endif
+#endif
         // Add the timestep to the global summary file
         if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) {
-            auto filename = global_IO_path+"/summary.LBM";
+            auto filename = global_IO_path + "/summary.LBM";
-            FILE *fid = fopen(filename.c_str(),"ab");
+            FILE *fid     = fopen( filename.c_str(), "ab" );
-            fprintf(fid,"%s/\n",subdir.c_str());
+            fprintf( fid, "%s/\n", subdir.c_str() );
-            fclose(fid);
+            fclose( fid );
         } else if ( global_IO_format == Format::SILO ) {
-            auto filename = global_IO_path+"/LBM.visit";
+            auto filename = global_IO_path + "/LBM.visit";
-            FILE *fid = fopen(filename.c_str(),"ab");
+            FILE *fid     = fopen( filename.c_str(), "ab" );
-            fprintf(fid,"%s/summary.silo\n",subdir.c_str());
+            fprintf( fid, "%s/summary.silo\n", subdir.c_str() );
-            fclose(fid);
+            fclose( fid );
         } else {
-            ERROR("Unknown format");
+            ERROR( "Unknown format" );
         }
     }
-    PROFILE_STOP("writeData");
+    PROFILE_STOP( "writeData" );
 }
-
-

IO/Writer.h

@@ -14,17 +14,18 @@ namespace IO {
 
 /*!
  * @brief  Initialize the writer
- * @details  This function initializes the writer to the given path.  All subsequent
+ * @details  This function initializes the writer to the given path.
- *    writes will occur in this directory.  If this is not called, then it will default
+ *    All subsequent writes will occur in this directory.
- *    to the current path.
+ *    If this is not called, then it will default to the current path.
  * @param[in] path          The path to use for writes
  * @param[in] format        The data format to use:
- *                          old - Old mesh format (provided for backward compatibility, cannot write variables)
+ *                              old - Old mesh format
- *                          new - New format, 1 file/process
+ *                                    (provided for backward compatibility, cannot write variables)
- *                          silo - Silo
+ *                              new - New format, 1 file/process silo - Silo
  * @param[in] append        Append any existing data (default is false)
  */
-void initialize( const std::string& path="", const std::string& format="silo", bool append=false );
+void initialize(
+    const std::string &path = "", const std::string &format = "silo", bool append = false );
 
 
 /*!
@@ -34,7 +35,8 @@ void initialize( const std::string& path="", const std::string& format="silo", b
  * @param[in] meshData      The data to write
  * @param[in] comm          The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
  */
-void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm );
+void writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
+    const Utilities::MPI &comm );
 
 
 /*!
@@ -44,14 +46,15 @@ void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>
  * @param[in] meshData      The data to write
  * @param[in] comm          The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
  */
-inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
+inline void writeData(
+    int timestep, const std::vector<IO::MeshDataStruct> &meshData, const Utilities::MPI &comm )
 {
     char subdir[100];
-    sprintf(subdir,"vis%03i",timestep);
+    sprintf( subdir, "vis%03i", timestep );
     writeData( subdir, meshData, comm );
 }
 
 
-} // IO namespace
+} // namespace IO
 
 #endif

IO/netcdf.cpp

@@ -1,6 +1,6 @@
 #include "IO/netcdf.h"
-#include "common/Utilities.h"
 #include "common/MPI.h"
+#include "common/Utilities.h"
 
 #include "ProfilerApp.h"
 
@@ -12,14 +12,14 @@
 #include <netcdf_par.h>
 
 
-#define CHECK_NC_ERR( ERR )             \
+#define CHECK_NC_ERR( ERR )                                     \
-    do {                                \
+    do {                                                        \
-        if ( ERR != NC_NOERR ) {        \
+        if ( ERR != NC_NOERR ) {                                \
             std::string msg = "Error calling netcdf routine: "; \
-            msg += nc_strerror( ERR );  \
+            msg += nc_strerror( ERR );                          \
-            ERROR( msg );               \
+            ERROR( msg );                                       \
-        }                               \
+        }                                                       \
-    } while (0)
+    } while ( 0 )
 
 
 namespace netcdf {
@@ -50,43 +50,64 @@ static inline VariableType convertType( nc_type type )
     else if ( type == NC_DOUBLE )
         type2 = DOUBLE;
     else
-        ERROR("Unknown type");
+        ERROR( "Unknown type" );
     return type2;
 }
 
 
 // Get nc_type from the template
-template<class T> inline nc_type getType();
+template<class T>
-template<> inline nc_type getType<char>()   { return NC_CHAR; }
+inline nc_type getType();
-template<> inline nc_type getType<short>()  { return NC_SHORT; }
+template<>
-template<> inline nc_type getType<int>()    { return NC_INT; }
+inline nc_type getType<char>()
-template<> inline nc_type getType<float>()  { return NC_FLOAT; }
+{
-template<> inline nc_type getType<double>() { return NC_DOUBLE; }
+    return NC_CHAR;
+}
+template<>
+inline nc_type getType<short>()
+{
+    return NC_SHORT;
+}
+template<>
+inline nc_type getType<int>()
+{
+    return NC_INT;
+}
+template<>
+inline nc_type getType<float>()
+{
+    return NC_FLOAT;
+}
+template<>
+inline nc_type getType<double>()
+{
+    return NC_DOUBLE;
+}
 
 
 // Function to reverse an array
 template<class TYPE>
-inline std::vector<TYPE> reverse( const std::vector<TYPE>& x )
+inline std::vector<TYPE> reverse( const std::vector<TYPE> &x )
 {
-    std::vector<TYPE> y(x.size());
+    std::vector<TYPE> y( x.size() );
-    for (size_t i=0; i<x.size(); i++)
+    for ( size_t i = 0; i < x.size(); i++ )
-        y[i] = x[x.size()-i-1];
+        y[i] = x[x.size() - i - 1];
     return y;
 }
 // Function to reverse an array
 template<class TYPE1, class TYPE2>
-inline std::vector<TYPE2> convert( const std::vector<TYPE1>& x )
+inline std::vector<TYPE2> convert( const std::vector<TYPE1> &x )
 {
-    std::vector<TYPE2> y(x.size());
+    std::vector<TYPE2> y( x.size() );
-    for (size_t i=0; i<x.size(); i++)
+    for ( size_t i = 0; i < x.size(); i++ )
-        y[i] = static_cast<TYPE2>(x[i]);
+        y[i] = static_cast<TYPE2>( x[i] );
     return y;
 }
 
 
 /****************************************************
-* Convert the VariableType to a string              *
+ * Convert the VariableType to a string              *
-****************************************************/
+ ****************************************************/
 std::string VariableTypeName( VariableType type )
 {
     if ( type == BYTE )
@@ -114,9 +135,9 @@ std::string VariableTypeName( VariableType type )
 
 
 /****************************************************
-* Open/close a file                                 *
+ * Open/close a file                                 *
-****************************************************/
+ ****************************************************/
-int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm )
+int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm )
 {
     int fid = 0;
     if ( comm.isNull() ) {
@@ -127,23 +148,26 @@ int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm
             int err = nc_open( filename.c_str(), NC_WRITE, &fid );
             CHECK_NC_ERR( err );
         } else if ( mode == CREATE ) {
-            int err = nc_create( filename.c_str(), NC_SHARE|NC_64BIT_OFFSET, &fid );
+            int err = nc_create( filename.c_str(), NC_SHARE | NC_64BIT_OFFSET, &fid );
             CHECK_NC_ERR( err );
         } else {
-            ERROR("Unknown file mode");
+            ERROR( "Unknown file mode" );
         }
     } else {
         if ( mode == READ ) {
-            int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
+            int err = nc_open_par(
+                filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
             CHECK_NC_ERR( err );
         } else if ( mode == WRITE ) {
-            int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
+            int err = nc_open_par( filename.c_str(), NC_WRITE | NC_MPIPOSIX, comm.getCommunicator(),
+                MPI_INFO_NULL, &fid );
             CHECK_NC_ERR( err );
         } else if ( mode == CREATE ) {
-            int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid );
+            int err = nc_create_par( filename.c_str(), NC_NETCDF4 | NC_MPIIO,
+                comm.getCommunicator(), MPI_INFO_NULL, &fid );
             CHECK_NC_ERR( err );
         } else {
-            ERROR("Unknown file mode");
+            ERROR( "Unknown file mode" );
         }
     }
     return fid;
@@ -152,42 +176,42 @@ void close( int fid )
 {
     int err = nc_close( fid );
     if ( err != NC_NOERR )
-        ERROR("Error closing file");
+        ERROR( "Error closing file" );
 }
 
 
 /****************************************************
-* Query basic properties                            *
+ * Query basic properties                            *
-****************************************************/
+ ****************************************************/
 static std::vector<size_t> getDimVar( int fid, int varid )
 {
     int ndim = 0;
-    int err = nc_inq_varndims( fid, varid, &ndim );
+    int err  = nc_inq_varndims( fid, varid, &ndim );
     CHECK_NC_ERR( err );
-    std::vector<size_t> dims(ndim,0);
+    std::vector<size_t> dims( ndim, 0 );
-    int dimid[64] = {-1};
+    int dimid[64] = { -1 };
-    err = nc_inq_vardimid( fid, varid, dimid );
+    err           = nc_inq_vardimid( fid, varid, dimid );
     CHECK_NC_ERR( err );
-    for (int i=0; i<ndim; i++) {
+    for ( int i = 0; i < ndim; i++ ) {
         err = nc_inq_dimlen( fid, dimid[i], &dims[i] );
         CHECK_NC_ERR( err );
     }
     return dims;
 }
-static int getVarID( int fid, const std::string& var )
+static int getVarID( int fid, const std::string &var )
 {
-    int id = -1;
+    int id  = -1;
     int err = nc_inq_varid( fid, var.c_str(), &id );
     CHECK_NC_ERR( err );
     return id;
 }
-std::vector<size_t> getVarDim( int fid, const std::string& var )
+std::vector<size_t> getVarDim( int fid, const std::string &var )
 {
     return getDimVar( fid, getVarID( fid, var ) );
 }
-std::vector<size_t> getAttDim( int fid, const std::string& att )
+std::vector<size_t> getAttDim( int fid, const std::string &att )
 {
-    std::vector<size_t> dim(1,0);
+    std::vector<size_t> dim( 1, 0 );
     int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
     CHECK_NC_ERR( err );
     return dim;
@@ -197,9 +221,9 @@ std::vector<std::string> getVarNames( int fid )
     int nvar;
     int err = nc_inq( fid, NULL, &nvar, NULL, NULL );
     CHECK_NC_ERR( err );
-    std::vector<std::string> vars(nvar);
+    std::vector<std::string> vars( nvar );
-    for (int i=0; i<nvar; i++) {
+    for ( int i = 0; i < nvar; i++ ) {
-        char name[NC_MAX_NAME+1];
+        char name[NC_MAX_NAME + 1];
         err = nc_inq_varname( fid, i, name );
         CHECK_NC_ERR( err );
         vars[i] = name;
@@ -211,262 +235,269 @@ std::vector<std::string> getAttNames( int fid )
     int natt;
     int err = nc_inq( fid, NULL, NULL, &natt, NULL );
     CHECK_NC_ERR( err );
-    std::vector<std::string> att(natt);
+    std::vector<std::string> att( natt );
-    for (int i=0; i<natt; i++) {
+    for ( int i = 0; i < natt; i++ ) {
-        char name[NC_MAX_NAME+1];
+        char name[NC_MAX_NAME + 1];
-        err = nc_inq_attname( fid,  NC_GLOBAL, i, name );
+        err = nc_inq_attname( fid, NC_GLOBAL, i, name );
         CHECK_NC_ERR( err );
         att[i] = name;
     }
     return att;
 }
-VariableType getVarType( int fid, const std::string& var )
+VariableType getVarType( int fid, const std::string &var )
 {
     int varid = -1;
-    int err = nc_inq_varid( fid, var.c_str(), &varid );
+    int err   = nc_inq_varid( fid, var.c_str(), &varid );
     CHECK_NC_ERR( err );
-    nc_type type=0;
+    nc_type type = 0;
-    err = nc_inq_vartype( fid, varid, &type );
+    err          = nc_inq_vartype( fid, varid, &type );
     CHECK_NC_ERR( err );
-    return convertType(type);
+    return convertType( type );
 }
-VariableType getAttType( int fid, const std::string& att )
+VariableType getAttType( int fid, const std::string &att )
 {
-    nc_type type=0;
+    nc_type type = 0;
-    int err = nc_inq_atttype( fid,  NC_GLOBAL, att.c_str(), &type );
+    int err      = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type );
     CHECK_NC_ERR( err );
-    return convertType(type);
+    return convertType( type );
 }
 
 
-
 /****************************************************
-* Read a variable                                   *
+ * Read a variable                                   *
-****************************************************/
+ ****************************************************/
 template<>
-Array<unsigned short> getVar<unsigned short>( int fid, const std::string& var )
+Array<unsigned short> getVar<unsigned short>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<unsigned short>");
+    PROFILE_START( "getVar<unsigned short>" );
-    Array<unsigned short> x( reverse(getVarDim(fid,var)) );
+    Array<unsigned short> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_ushort( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_ushort( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<unsigned short>");
+    PROFILE_STOP( "getVar<unsigned short>" );
     return x.reverseDim();
 }
 template<>
-Array<short> getVar<short>( int fid, const std::string& var )
+Array<short> getVar<short>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<short>");
+    PROFILE_START( "getVar<short>" );
-    Array<short> x( reverse(getVarDim(fid,var)) );
+    Array<short> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_short( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_short( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<short>");
+    PROFILE_STOP( "getVar<short>" );
     return x.reverseDim();
 }
 template<>
-Array<unsigned int> getVar<unsigned int>( int fid, const std::string& var )
+Array<unsigned int> getVar<unsigned int>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<unsigned int>");
+    PROFILE_START( "getVar<unsigned int>" );
-    Array<unsigned int> x( reverse(getVarDim(fid,var)) );
+    Array<unsigned int> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_uint( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_uint( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<unsigned int>");
+    PROFILE_STOP( "getVar<unsigned int>" );
     return x.reverseDim();
 }
 template<>
-Array<int> getVar<int>( int fid, const std::string& var )
+Array<int> getVar<int>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<int>");
+    PROFILE_START( "getVar<int>" );
-    Array<int> x( reverse(getVarDim(fid,var)) );
+    Array<int> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_int( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_int( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<int>");
+    PROFILE_STOP( "getVar<int>" );
     return x.reverseDim();
 }
 template<>
-Array<float> getVar<float>( int fid, const std::string& var )
+Array<float> getVar<float>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<float>");
+    PROFILE_START( "getVar<float>" );
-    Array<float> x( reverse(getVarDim(fid,var)) );
+    Array<float> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_float( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_float( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<float>");
+    PROFILE_STOP( "getVar<float>" );
     return x.reverseDim();
 }
 template<>
-Array<double> getVar<double>( int fid, const std::string& var )
+Array<double> getVar<double>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<double>");
+    PROFILE_START( "getVar<double>" );
-    Array<double> x( reverse(getVarDim(fid,var)) );
+    Array<double> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_double( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_double( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<double>");
+    PROFILE_STOP( "getVar<double>" );
     return x.reverseDim();
 }
 template<>
-Array<char> getVar<char>( int fid, const std::string& var )
+Array<char> getVar<char>( int fid, const std::string &var )
-{ 
+{
-    PROFILE_START("getVar<char>");
+    PROFILE_START( "getVar<char>" );
-    Array<char> x( reverse(getVarDim(fid,var)) );
+    Array<char> x( reverse( getVarDim( fid, var ) ) );
-    int err = nc_get_var_text( fid, getVarID(fid,var), x.data() );
+    int err = nc_get_var_text( fid, getVarID( fid, var ), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<char>");
+    PROFILE_STOP( "getVar<char>" );
     return x.reverseDim();
 }
 template<>
-Array<std::string> getVar<std::string>( int fid, const std::string& var )
+Array<std::string> getVar<std::string>( int fid, const std::string &var )
 {
-    PROFILE_START("getVar<std::string>");
+    PROFILE_START( "getVar<std::string>" );
-    Array<char> tmp = getVar<char>( fid, var );
+    Array<char> tmp         = getVar<char>( fid, var );
-    std::vector<size_t> dim = {tmp.size(0), tmp.size(1), tmp.size(2) };
+    std::vector<size_t> dim = { tmp.size( 0 ), tmp.size( 1 ), tmp.size( 2 ) };
     if ( dim.size() == 1 )
         dim[0] = 1;
     else
         dim.erase( dim.begin() );
-    Array<std::string> text(dim);
+    Array<std::string> text( dim );
-    for (size_t i=0; i<text.length(); i++)
+    for ( size_t i = 0; i < text.length(); i++ )
-        text(i) = &(tmp(0,i));
+        text( i ) = &( tmp( 0, i ) );
-    PROFILE_STOP("getVar<std::string>");
+    PROFILE_STOP( "getVar<std::string>" );
     return text;
 }
-static inline void get_stride_args( const std::vector<int>& start,
+static inline void get_stride_args( const std::vector<int> &start, const std::vector<int> &count,
-    const std::vector<int>& count, const std::vector<int>& stride,
+    const std::vector<int> &stride, size_t *startp, size_t *countp, ptrdiff_t *stridep )
-    size_t *startp, size_t *countp, ptrdiff_t *stridep )
 {
-    for (size_t i=0; i<start.size(); i++)
+    for ( size_t i = 0; i < start.size(); i++ )
         startp[i] = start[i];
-    for (size_t i=0; i<count.size(); i++)
+    for ( size_t i = 0; i < count.size(); i++ )
         countp[i] = count[i];
-    for (size_t i=0; i<stride.size(); i++)
+    for ( size_t i = 0; i < stride.size(); i++ )
         stridep[i] = stride[i];
 }
 template<class TYPE>
-int nc_get_vars_TYPE( int fid, int varid, const size_t start[],
+int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[],
-    const size_t count[], const ptrdiff_t stride[], TYPE *ptr );
+    const ptrdiff_t stride[], TYPE *ptr );
 template<>
-int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[],
+int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[], const size_t count[],
-    const size_t count[], const ptrdiff_t stride[], short *ptr )
+    const ptrdiff_t stride[], short *ptr )
 {
     return nc_get_vars_short( fid, varid, start, count, stride, ptr );
 }
 template<>
-int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[],
+int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[], const size_t count[],
-    const size_t count[], const ptrdiff_t stride[], int *ptr )
+    const ptrdiff_t stride[], int *ptr )
 {
     return nc_get_vars_int( fid, varid, start, count, stride, ptr );
 }
 template<>
-int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[],
+int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[], const size_t count[],
-    const size_t count[], const ptrdiff_t stride[], float *ptr )
+    const ptrdiff_t stride[], float *ptr )
 {
     return nc_get_vars_float( fid, varid, start, count, stride, ptr );
 }
 template<>
-int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[],
+int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[], const size_t count[],
-    const size_t count[], const ptrdiff_t stride[], double *ptr )
+    const ptrdiff_t stride[], double *ptr )
 {
     return nc_get_vars_double( fid, varid, start, count, stride, ptr );
 }
 template<class TYPE>
-Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
+Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
-    const std::vector<int>& count, const std::vector<int>& stride )
+    const std::vector<int> &count, const std::vector<int> &stride )
 {
-    PROFILE_START("getVar<> (strided)");
+    PROFILE_START( "getVar<> (strided)" );
     std::vector<size_t> var_size = getVarDim( fid, var );
-    for (int d=0; d<(int)var_size.size(); d++) {
+    for ( int d = 0; d < (int) var_size.size(); d++ ) {
-        if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) {
+        if ( start[d] < 0 || start[d] + stride[d] * ( count[d] - 1 ) > (int) var_size[d] ) {
-            int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
+            int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
             char tmp[1000];
-            sprintf(tmp,"%i: Range exceeded array dimension:\n"
+            sprintf( tmp,
+                "%i: Range exceeded array dimension:\n"
                 "   start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",
-                rank,d,start[d],d,count[d],d,stride[d],d,(int)var_size[d]);
+                rank, d, start[d], d, count[d], d, stride[d], d, (int) var_size[d] );
-            ERROR(tmp);
+            ERROR( tmp );
         }
     }
-    Array<TYPE> x( reverse(convert<int,size_t>(count)) );
+    Array<TYPE> x( reverse( convert<int, size_t>( count ) ) );
     size_t startp[10], countp[10];
     ptrdiff_t stridep[10];
     get_stride_args( start, count, stride, startp, countp, stridep );
-    int err = nc_get_vars_TYPE<TYPE>( fid, getVarID(fid,var), startp, countp, stridep, x.data() );
+    int err =
+        nc_get_vars_TYPE<TYPE>( fid, getVarID( fid, var ), startp, countp, stridep, x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getVar<> (strided)");
+    PROFILE_STOP( "getVar<> (strided)" );
     return x.reverseDim();
 }
-template Array<short>  getVar<short>(  int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+template Array<short> getVar<short>( int, const std::string &, const std::vector<int> &,
-template Array<int>    getVar<int>(    int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+    const std::vector<int> &, const std::vector<int> & );
-template Array<float>  getVar<float>(  int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+template Array<int> getVar<int>( int, const std::string &, const std::vector<int> &,
-template Array<double> getVar<double>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
+    const std::vector<int> &, const std::vector<int> & );
+template Array<float> getVar<float>( int, const std::string &, const std::vector<int> &,
+    const std::vector<int> &, const std::vector<int> & );
+template Array<double> getVar<double>( int, const std::string &, const std::vector<int> &,
+    const std::vector<int> &, const std::vector<int> & );
 
 
 /****************************************************
-* Read an attribute                                 *
+ * Read an attribute                                 *
-****************************************************/
+ ****************************************************/
 template<>
-Array<double> getAtt<double>( int fid, const std::string& att )
+Array<double> getAtt<double>( int fid, const std::string &att )
 {
-    PROFILE_START("getAtt<double>");
+    PROFILE_START( "getAtt<double>" );
-    Array<double> x( getAttDim(fid,att) );
+    Array<double> x( getAttDim( fid, att ) );
     int err = nc_get_att_double( fid, NC_GLOBAL, att.c_str(), x.data() );
     CHECK_NC_ERR( err );
-    PROFILE_STOP("getAtt<double>");
+    PROFILE_STOP( "getAtt<double>" );
     return x;
 }
 template<>
-Array<std::string> getAtt<std::string>( int fid, const std::string& att )
+Array<std::string> getAtt<std::string>( int fid, const std::string &att )
 {
-    PROFILE_START("getAtt<std::string>");
+    PROFILE_START( "getAtt<std::string>" );
-    char *tmp = new char[getAttDim(fid,att)[0]];
+    char *tmp = new char[getAttDim( fid, att )[0]];
-    Array<std::string> x(1);
+    Array<std::string> x( 1 );
-    x(0) = tmp;
+    x( 0 ) = tmp;
-    delete [] tmp;
+    delete[] tmp;
-    PROFILE_STOP("getAtt<std::string>");
+    PROFILE_STOP( "getAtt<std::string>" );
     return x;
 }
 
 
 /****************************************************
-* Write an array to a file                          *
+ * Write an array to a file                          *
-****************************************************/
+ ****************************************************/
-std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims )
+std::vector<int> defDim(
+    int fid, const std::vector<std::string> &names, const std::vector<int> &dims )
 {
-    std::vector<int> dimid(names.size(),0);
+    std::vector<int> dimid( names.size(), 0 );
-    for (size_t i=0; i<names.size(); i++) {
+    for ( size_t i = 0; i < names.size(); i++ ) {
-        int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i]);
+        int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i] );
         CHECK_NC_ERR( err );
     }
     return dimid;
 }
 template<class TYPE>
-void write( int fid, const std::string& var, const std::vector<int>& dimids,
+void write( int fid, const std::string &var, const std::vector<int> &dimids,
-    const Array<TYPE>& data, const RankInfoStruct& info )
+    const Array<TYPE> &data, const RankInfoStruct &info )
 {
     // Define the variable
     int varid = 0;
-    int err = nc_def_var( fid, var.c_str(), getType<TYPE>(), data.ndim(), dimids.data(), &varid );
+    int err   = nc_def_var( fid, var.c_str(), getType<TYPE>(), data.ndim(), dimids.data(), &varid );
     CHECK_NC_ERR( err );
-    // exit define mode 
+    // exit define mode
     err = nc_enddef( fid );
     CHECK_NC_ERR( err );
-    // set the access method to use MPI/PnetCDF collective I/O 
+    // set the access method to use MPI/PnetCDF collective I/O
     err = nc_var_par_access( fid, varid, NC_INDEPENDENT );
     CHECK_NC_ERR( err );
     // parallel write: each process writes its subarray to the file
-    auto x = data.reverseDim();
+    auto x                    = data.reverseDim();
-    std::vector<size_t> count = { data.size(0), data.size(1), data.size(2) };
+    std::vector<size_t> count = { data.size( 0 ), data.size( 1 ), data.size( 2 ) };
-    std::vector<size_t> start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) };
+    std::vector<size_t> start = { info.ix * data.size( 0 ), info.jy * data.size( 1 ),
+        info.kz * data.size( 2 ) };
     nc_put_vara( fid, varid, start.data(), count.data(), x.data() );
 }
-template void write<short>(  int fid, const std::string& var, const std::vector<int>& dimids, const Array<short>& data,  const RankInfoStruct& info );
+template void write<short>( int fid, const std::string &var, const std::vector<int> &dimids,
-template void write<int>(    int fid, const std::string& var, const std::vector<int>& dimids, const Array<int>& data,    const RankInfoStruct& info );
+    const Array<short> &data, const RankInfoStruct &info );
-template void write<float>(  int fid, const std::string& var, const std::vector<int>& dimids, const Array<float>& data,  const RankInfoStruct& info );
+template void write<int>( int fid, const std::string &var, const std::vector<int> &dimids,
-template void write<double>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<double>& data, const RankInfoStruct& info );
+    const Array<int> &data, const RankInfoStruct &info );
+template void write<float>( int fid, const std::string &var, const std::vector<int> &dimids,
+    const Array<float> &data, const RankInfoStruct &info );
+template void write<double>( int fid, const std::string &var, const std::vector<int> &dimids,
+    const Array<double> &data, const RankInfoStruct &info );
 
 
-
+}; // namespace netcdf
-}; // netcdf namespace
 
 #else
 
 #endif
-
-

IO/netcdf.h

@@ -5,9 +5,8 @@
 #include <vector>
 
 #include "common/Array.h"
-#include "common/MPI.h"
 #include "common/Communication.h"
-
+#include "common/MPI.h"
 
 
 namespace netcdf {
@@ -31,15 +30,15 @@ std::string VariableTypeName( VariableType type );
  * @param filename      File to open
  * @param mode          Open the file for reading or writing
  * @param comm          MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf)
-*/
+ */
-int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm=MPI_COMM_NULL );
+int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm = MPI_COMM_NULL );
 
 
 /*!
  * @brief  Close netcdf file
  * @details  This function closes a netcdf file
  * @param fid           Handle to the open file
-*/
+ */
 void close( int fid );
 
 
@@ -47,7 +46,7 @@ void close( int fid );
  * @brief  Read the variable names
  * @details  This function reads a list of the variable names in the file
  * @param fid           Handle to the open file
-*/
+ */
 std::vector<std::string> getVarNames( int fid );
 
 
@@ -55,7 +54,7 @@ std::vector<std::string> getVarNames( int fid );
  * @brief  Read the attribute names
  * @details  This function reads a list of the attribute names in the file
  * @param fid           Handle to the open file
-*/
+ */
 std::vector<std::string> getAttNames( int fid );
 
 
@@ -64,8 +63,8 @@ std::vector<std::string> getAttNames( int fid );
  * @details  This function returns the type for a variable
  * @param fid           Handle to the open file
  * @param var           Variable to read
-*/
+ */
-VariableType getVarType( int fid, const std::string& var );
+VariableType getVarType( int fid, const std::string &var );
 
 
 /*!
@@ -73,8 +72,8 @@ VariableType getVarType( int fid, const std::string& var );
  * @details  This function returns the type for an attribute
  * @param fid           Handle to the open file
  * @param att           Attribute to read
-*/
+ */
-VariableType getAttType( int fid, const std::string& att );
+VariableType getAttType( int fid, const std::string &att );
 
 
 /*!
@@ -82,8 +81,8 @@ VariableType getAttType( int fid, const std::string& att );
  * @details  This function returns the die for a variable
  * @param fid           Handle to the open file
  * @param var           Variable to read
-*/
+ */
-std::vector<size_t> getVarDim( int fid, const std::string& var );
+std::vector<size_t> getVarDim( int fid, const std::string &var );
 
 
 /*!
@@ -91,9 +90,9 @@ std::vector<size_t> getVarDim( int fid, const std::string& var );
  * @details  This function reads a variable with the given name from the file
  * @param fid           Handle to the open file
  * @param var           Variable to read
-*/
+ */
 template<class TYPE>
-Array<TYPE> getVar( int fid, const std::string& var );
+Array<TYPE> getVar( int fid, const std::string &var );
 
 
 /*!
@@ -104,10 +103,10 @@ Array<TYPE> getVar( int fid, const std::string& var );
  * @param start         Starting corner for the read
  * @param count         Number of elements to read
  * @param stride        Stride size for the read
-*/
+ */
 template<class TYPE>
-Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
+Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
-    const std::vector<int>& count, const std::vector<int>& stride );
+    const std::vector<int> &count, const std::vector<int> &stride );
 
 
 /*!
@@ -115,27 +114,29 @@ Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& sta
  * @details  This function reads an attribute with the given name from the file
  * @param fid           Handle to the open file
  * @param att           Attribute to read
-*/
+ */
 template<class TYPE>
-Array<TYPE> getAtt( int fid, const std::string& att );
+Array<TYPE> getAtt( int fid, const std::string &att );
 
 
 /*!
  * @brief  Write the dimensions
- * @details  This function writes the grid dimensions to netcdf. 
+ * @details  This function writes the grid dimensions to netcdf.
  * @param fid           Handle to the open file
-*/
+ */
-std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims );
+std::vector<int> defDim(
+    int fid, const std::vector<std::string> &names, const std::vector<int> &dims );
 
 
 /*!
  * @brief  Write a variable
- * @details  This function writes a variable to netcdf. 
+ * @details  This function writes a variable to netcdf.
  * @param fid           Handle to the open file
-*/
+ */
 template<class TYPE>
-void write( int fid, const std::string& var, const std::vector<int>& dimids, const Array<TYPE>& data, const RankInfoStruct& rank_info );
+void write( int fid, const std::string &var, const std::vector<int> &dimids,
+    const Array<TYPE> &data, const RankInfoStruct &rank_info );
 
 
-}; // netcdf namespace
+}; // namespace netcdf
 #endif

IO/silo.cpp

@@ -1,6 +1,6 @@
 #include "IO/silo.h"
-#include "common/Utilities.h"
 #include "common/MPI.h"
+#include "common/Utilities.h"
 
 #include "ProfilerApp.h"
 
@@ -10,14 +10,13 @@
 #include <silo.h>
 
 
-
+namespace IO::silo {
-namespace silo {
 
 
 /****************************************************
-* Open/close a file                                 *
+ * Open/close a file                                 *
-****************************************************/
+ ****************************************************/
-DBfile* open( const std::string& filename, FileMode mode )
+DBfile *open( const std::string &filename, FileMode mode )
 {
     DBfile *fid = nullptr;
     if ( mode == CREATE ) {
@@ -29,82 +28,78 @@ DBfile* open( const std::string& filename, FileMode mode )
     }
     return fid;
 }
-void close( DBfile* fid )
+void close( DBfile *fid ) { DBClose( fid ); }
-{
-    DBClose( fid );
-}
 
 
 /****************************************************
-* Helper functions                                  *
+ * Helper functions                                  *
-****************************************************/
+ ****************************************************/
-VariableDataType varDataType( DBfile *fid, const std::string& name )
+DataType varDataType( DBfile *fid, const std::string &name )
 {
-    auto type = DBGetVarType( fid, name.c_str() );
+    auto type      = DBGetVarType( fid, name.c_str() );
-    VariableDataType type2 = VariableDataType::UNKNOWN;
+    DataType type2 = DataType::Null;
     if ( type == DB_DOUBLE )
-        type2 = VariableDataType::DOUBLE;
+        type2 = DataType::Double;
     else if ( type == DB_FLOAT )
-        type2 = VariableDataType::FLOAT;
+        type2 = DataType::Float;
     else if ( type == DB_INT )
-        type2 = VariableDataType::INT;
+        type2 = DataType::Int;
     return type2;
 }
 
 
 /****************************************************
-* Write/read a uniform mesh to silo                 *
+ * Write/read a uniform mesh to silo                 *
-****************************************************/
+ ****************************************************/
-void readUniformMesh( DBfile* fid, const std::string& meshname,
+void readUniformMesh(
-    std::vector<double>& range, std::vector<int>& N )
+    DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N )
 {
-    DBquadmesh* mesh = DBGetQuadmesh( fid, meshname.c_str() );
+    DBquadmesh *mesh = DBGetQuadmesh( fid, meshname.c_str() );
-    int ndim = mesh->ndims;
+    int ndim         = mesh->ndims;
-    range.resize(2*ndim);
+    range.resize( 2 * ndim );
-    N.resize(ndim);
+    N.resize( ndim );
-    for (int d=0; d<ndim; d++) {
+    for ( int d = 0; d < ndim; d++ ) {
-        N[d] = mesh->dims[d]-1;
+        N[d]             = mesh->dims[d] - 1;
-        range[2*d+0] = mesh->min_extents[d];
+        range[2 * d + 0] = mesh->min_extents[d];
-        range[2*d+1] = mesh->max_extents[d];
+        range[2 * d + 1] = mesh->max_extents[d];
     }
     DBFreeQuadmesh( mesh );
 }
 
 
 /****************************************************
-* Write a multimesh                                 *
+ * Write a multimesh                                 *
-****************************************************/
+ ****************************************************/
-void writeMultiMesh( DBfile* fid, const std::string& meshname,
+void writeMultiMesh( DBfile *fid, const std::string &meshname,
-    const std::vector<std::string>& meshNames,
+    const std::vector<std::string> &meshNames, const std::vector<int> &meshTypes )
-    const std::vector<int>& meshTypes )
 {
-    std::vector<char*> meshnames(meshNames.size());
+    std::vector<char *> meshnames( meshNames.size() );
     for ( size_t i = 0; i < meshNames.size(); ++i )
         meshnames[i] = (char *) meshNames[i].c_str();
     std::string tree_name = meshname + "_tree";
     DBoptlist *optList    = DBMakeOptlist( 1 );
     DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() );
-    DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr );
+    DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(),
+        (int *) meshTypes.data(), nullptr );
     DBFreeOptlist( optList );
 }
 
 
 /****************************************************
-* Write a multivariable                             *
+ * Write a multivariable                             *
-****************************************************/
+ ****************************************************/
-void writeMultiVar( DBfile* fid, const std::string& varname,
+void writeMultiVar( DBfile *fid, const std::string &varname,
-    const std::vector<std::string>& varNames,
+    const std::vector<std::string> &varNames, const std::vector<int> &varTypes )
-    const std::vector<int>& varTypes )
 {
-    std::vector<char*> varnames(varNames.size(),nullptr);
+    std::vector<char *> varnames( varNames.size(), nullptr );
-    for (size_t j=0; j<varNames.size(); j++)
+    for ( size_t j = 0; j < varNames.size(); j++ )
-        varnames[j] = const_cast<char*>(varNames[j].c_str());
+        varnames[j] = const_cast<char *>( varNames[j].c_str() );
-    DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr );
+    DBPutMultivar(
+        fid, varname.c_str(), varNames.size(), varnames.data(), (int *) varTypes.data(), nullptr );
 }
 
 
-
+}; // namespace IO::silo
-}; // silo namespace
 
 
 #else

IO/silo.h

@@ -1,32 +1,28 @@
 #ifndef SILO_INTERFACE
 #define SILO_INTERFACE
 
+#include <array>
 #include <string>
 #include <vector>
-#include <array>
 
+#include "IO/Mesh.h"
 #include "common/Array.h"
-#include "common/MPI.h"
 #include "common/Communication.h"
+#include "common/MPI.h"
 
 
 #ifdef USE_SILO
-    #include <silo.h>
+#include <silo.h>
 #else
-    typedef int DBfile;
+typedef int DBfile;
 #endif
 
 
-
+namespace IO::silo {
-namespace silo {
 
 
 enum FileMode { READ, WRITE, CREATE };
 
-enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
-
-enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
-
 
 /*!
  * @brief  Open silo file
@@ -34,16 +30,16 @@ enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
  * @param[in] filename      File to open
  * @param[in] mode          Open the file for reading or writing
  * @return This function returns a handle to the file
-*/
+ */
-DBfile* open( const std::string& filename, FileMode mode );
+DBfile *open( const std::string &filename, FileMode mode );
 
 
 /*!
  * @brief  Close silo file
  * @details  This function closes a silo file
  * @param[in] fid           Handle to the open file
-*/
+ */
-void close( DBfile* fid );
+void close( DBfile *fid );
 
 
 /*!
@@ -51,8 +47,8 @@ void close( DBfile* fid );
  * @details  This function returns the type of variable data
  * @param[in] fid           Handle to the open file
  * @param[in] name          Name of variable
-*/
+ */
-VariableDataType varDataType( DBfile *dbfile, const std::string& name );
+DataType varDataType( DBfile *dbfile, const std::string &name );
 
 
 /*!
@@ -61,9 +57,9 @@ VariableDataType varDataType( DBfile *dbfile, const std::string& name );
  * @param[in] fid           Handle to the open file
  * @param[in] varname       Variable name
  * @param[in] data          Data to write
-*/
+ */
 template<class TYPE>
-void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data );
+void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data );
 
 
 /*!
@@ -72,9 +68,9 @@ void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& da
  * @param[in] fid           Handle to the open file
  * @param[in] varname       Variable name
  * @return                  Data read
-*/
+ */
 template<class TYPE>
-std::vector<TYPE> read( DBfile* fid, const std::string& varname );
+std::vector<TYPE> read( DBfile *fid, const std::string &varname );
 
 
 /*!
@@ -84,10 +80,10 @@ std::vector<TYPE> read( DBfile* fid, const std::string& varname );
  * @param[in] meshname      Mesh name
  * @param[in] range         Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
  * @param[in] N             Number of cells in each direction
-*/
+ */
 template<int NDIM>
-void writeUniformMesh( DBfile* fid, const std::string& meshname,
+void writeUniformMesh( DBfile *fid, const std::string &meshname,
-    const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N );
+    const std::array<double, 2 * NDIM> &range, const std::array<int, NDIM> &N );
 
 
 /*!
@@ -97,9 +93,9 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname,
  * @param[in] meshname      Mesh name
  * @param[out] range         Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
  * @param[out] N             Number of cells in each direction
-*/
+ */
-void readUniformMesh( DBfile* fid, const std::string& meshname,
+void readUniformMesh(
-    std::vector<double>& range, std::vector<int>& N );
+    DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N );
 
 
 /*!
@@ -111,10 +107,11 @@ void readUniformMesh( DBfile* fid, const std::string& meshname,
  * @param[in] varname       Variable name
  * @param[in] data          Variable data
  * @param[in] type          Variable type
-*/
+ */
-template< int NDIM, class TYPE >
+template<int NDIM, class TYPE>
-void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
+void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
-    const std::string& varname, const Array<TYPE>& data, VariableType type );
+    const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
+    VariableType type );
 
 
 /*!
@@ -123,9 +120,9 @@ void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const s
  * @param[in] fid           Handle to the open file
  * @param[in] varname       Variable name
  * @return                  Variable data
-*/
+ */
 template<class TYPE>
-Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
+Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname );
 
 
 /*!
@@ -136,10 +133,10 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
  * @param[in] ndim          Number of dimensions
  * @param[in] N             Number of points
  * @param[in] coords        Coordinates of the points
-*/
+ */
 template<class TYPE>
-void writePointMesh( DBfile* fid, const std::string& meshname,
+void writePointMesh(
-    int ndim, int N, const TYPE *coords[] );
+    DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] );
 
 
 /*!
@@ -147,10 +144,10 @@ void writePointMesh( DBfile* fid, const std::string& meshname,
  * @details  This function reads a pointmesh from silo
  * @param[in] fid           Handle to the open file
  * @param[in] meshname      Mesh name
- * @return                  Returns the coordinates as a N x ndim array 
+ * @return                  Returns the coordinates as a N x ndim array
-*/
+ */
 template<class TYPE>
-Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
+Array<TYPE> readPointMesh( DBfile *fid, const std::string &meshname );
 
 
 /*!
@@ -160,10 +157,10 @@ Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
  * @param[in] meshname      Mesh name
  * @param[in] varname       Variable name
  * @param[in] data          Variable data
-*/
+ */
 template<class TYPE>
-void writePointMeshVariable( DBfile* fid, const std::string& meshname,
+void writePointMeshVariable(
-    const std::string& varname, const Array<TYPE>& data );
+    DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data );
 
 
 /*!
@@ -172,9 +169,9 @@ void writePointMeshVariable( DBfile* fid, const std::string& meshname,
  * @param[in] fid           Handle to the open file
  * @param[in] varname       Variable name
  * @return                  Variable data
-*/
+ */
 template<class TYPE>
-Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
+Array<TYPE> readPointMeshVariable( DBfile *fid, const std::string &varname );
 
 
 /*!
@@ -188,10 +185,10 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
  * @param[in] coords        Coordinates of the points
  * @param[in] N_tri         Number of triangles
  * @param[in] tri           Coordinates of the points
-*/
+ */
 template<class TYPE>
-void writeTriMesh( DBfile* fid, const std::string& meshname,
+void writeTriMesh( DBfile *fid, const std::string &meshname, int ndim, int ndim_tri, int N,
-    int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] );
+    const TYPE *coords[], int N_tri, const int *tri[] );
 
 
 /*!
@@ -201,9 +198,9 @@ void writeTriMesh( DBfile* fid, const std::string& meshname,
  * @param[in] meshname      Mesh name
  * @param[in] coords        Coordinates of the points
  * @param[in] tri           Coordinates of the points
-*/
+ */
 template<class TYPE>
-void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri );
+void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri );
 
 
 /*!
@@ -215,10 +212,10 @@ void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords,
  * @param[in] varname       Variable name
  * @param[in] data          Variable data
  * @param[in] type          Variable type
-*/
+ */
 template<class TYPE>
-void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
+void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname,
-    const std::string& varname, const Array<TYPE>& data, VariableType type );
+    const std::string &varname, const Array<TYPE> &data, VariableType type );
 
 
 /*!
@@ -227,9 +224,9 @@ void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
  * @param[in] fid           Handle to the open file
  * @param[in] varname       Variable name
  * @return                  Variable data
-*/
+ */
 template<class TYPE>
-Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
+Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname );
 
 
 /*!
@@ -239,10 +236,9 @@ Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
  * @param[in] meshname      Mesh name
  * @param[in] subMeshNames  Names of the sub meshes in the form "filename:meshname"
  * @param[in] subMeshTypes  Type of each submesh
-*/
+ */
-void writeMultiMesh( DBfile* fid, const std::string& meshname,
+void writeMultiMesh( DBfile *fid, const std::string &meshname,
-    const std::vector<std::string>& subMeshNames,
+    const std::vector<std::string> &subMeshNames, const std::vector<int> &subMeshTypes );
-    const std::vector<int>& subMeshTypes );
 
 
 /*!
@@ -255,14 +251,12 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
  * @param[in] subVarTypes   Type of each submesh
  * @param[in] ndim          Dimension of variable (used to determine suffix)
  * @param[in] nvar          Number of subvariables (used to determine suffix)
-*/
+ */
-void writeMultiVar( DBfile* fid, const std::string& varname,
+void writeMultiVar( DBfile *fid, const std::string &varname,
-    const std::vector<std::string>& subVarNames,
+    const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
-    const std::vector<int>& subVarTypes );
 
 
-}; // silo namespace
+}; // namespace IO::silo
 #endif
 
 #include "IO/silo.hpp"
-

IO/silo.hpp

@@ -13,7 +13,7 @@
 #include <silo.h>
 
 
-namespace silo {
+namespace IO::silo {
 
 
 /****************************************************
@@ -413,7 +413,7 @@ Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
 }
 
 
-}; // namespace silo
+}; // namespace IO::silo
 
 
 #endif

README.titan

@@ -26,7 +26,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=CC        \
     -D CFLAGS="-DCBUB"                   \
     -D CXXFLAGS="-DCBUB"                 \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=aprun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Debug     \

analysis/FreeEnergy.cpp

@@ -0,0 +1,181 @@
+#include "analysis/FreeEnergy.h"
+
+FreeEnergyAnalyzer::FreeEnergyAnalyzer(std::shared_ptr <Domain> dm):
+	Dm(dm)
+{
+	
+	Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
+	Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
+	
+	ChemicalPotential.resize(Nx,Ny,Nz);       ChemicalPotential.fill(0);
+	Phi.resize(Nx,Ny,Nz);      		Phi.fill(0);
+	Pressure.resize(Nx,Ny,Nz);     	Pressure.fill(0);
+	Rho.resize(Nx,Ny,Nz);       	Rho.fill(0);
+	Vel_x.resize(Nx,Ny,Nz);         Vel_x.fill(0);	    // Gradient of the phase indicator field
+	Vel_y.resize(Nx,Ny,Nz);         Vel_y.fill(0);
+	Vel_z.resize(Nx,Ny,Nz);         Vel_z.fill(0);
+	SDs.resize(Nx,Ny,Nz);         	SDs.fill(0);
+	
+	if (Dm->rank()==0){
+		bool WriteHeader=false;
+		TIMELOG = fopen("free.csv","r");
+		if (TIMELOG != NULL)
+			fclose(TIMELOG);
+		else
+			WriteHeader=true;
+
+		TIMELOG = fopen("free.csv","a+");
+		if (WriteHeader)
+		{
+			// If timelog is empty, write a short header to list the averages
+			//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
+			fprintf(TIMELOG,"timestep\n");				
+		}
+	}
+
+}
+
+FreeEnergyAnalyzer::~FreeEnergyAnalyzer(){
+	if (Dm->rank()==0){
+		fclose(TIMELOG);
+	}
+}
+
+void FreeEnergyAnalyzer::SetParams(){
+	
+}
+
+void FreeEnergyAnalyzer::Basic(ScaLBL_FreeLeeModel &LeeModel, int timestep){
+
+	int i,j,k;
+
+	if (Dm->rank()==0){	
+		fprintf(TIMELOG,"%i ",timestep); 
+		/*for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
+		}
+		*/
+		fprintf(TIMELOG,"\n");
+		fflush(TIMELOG);
+	}
+/*	else{
+		fprintf(TIMELOG,"%i ",timestep); 
+		for (int ion=0; ion<Ion.number_ion_species; ion++){
+			fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
+			fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
+		}
+		fflush(TIMELOG);
+	} */
+}
+
+void FreeEnergyAnalyzer::WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep){
+	
+	auto vis_db =  input_db->getDatabase( "Visualization" );
+    char VisName[40];	
+    
+    std::vector<IO::MeshDataStruct> visData;
+	fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
+
+    IO::initialize("","silo","false");
+    // Create the MeshDataStruct    
+    visData.resize(1);
+
+    visData[0].meshName = "domain";
+    visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
+    auto VisPhase = std::make_shared<IO::Variable>();
+    auto VisPressure = std::make_shared<IO::Variable>();
+    auto VisChemicalPotential = std::make_shared<IO::Variable>();
+    auto VxVar = std::make_shared<IO::Variable>();
+    auto VyVar = std::make_shared<IO::Variable>();
+    auto VzVar = std::make_shared<IO::Variable>();
+    
+
+    if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
+    	VisPhase->name = "Phase";
+    	VisPhase->type = IO::VariableType::VolumeVariable;
+    	VisPhase->dim = 1;
+    	VisPhase->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VisPhase);
+    }    
+    
+	if (vis_db->getWithDefault<bool>( "save_potential", true )){
+		
+    	VisPressure->name = "Pressure";
+    	VisPressure->type = IO::VariableType::VolumeVariable;
+    	VisPressure->dim = 1;
+    	VisPressure->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VisPressure);
+        
+    	VisChemicalPotential->name = "ChemicalPotential";
+    	VisChemicalPotential->type = IO::VariableType::VolumeVariable;
+    	VisChemicalPotential->dim = 1;
+    	VisChemicalPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VisChemicalPotential);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim = 1;
+        VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VxVar);
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim = 1;
+        VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VyVar);
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim = 1;
+        VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
+        visData[0].vars.push_back(VzVar);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "save_phase", true )){
+    	ASSERT(visData[0].vars[0]->name=="Phase");
+    	LeeModel.getPhase(Phi);
+    	Array<double>& PhaseData = visData[0].vars[0]->data;
+    	fillData.copy(Phi,PhaseData);
+    }
+
+    if (vis_db->getWithDefault<bool>( "save_potential", true )){
+    	ASSERT(visData[0].vars[1]->name=="Pressure");
+    	LeeModel.getPotential(Pressure, ChemicalPotential);
+    	Array<double>& PressureData = visData[0].vars[1]->data;
+    	fillData.copy(Pressure,PressureData);
+    	
+    	ASSERT(visData[0].vars[2]->name=="ChemicalPotential");
+    	Array<double>& ChemicalPotentialData = visData[0].vars[2]->data;
+    	fillData.copy(ChemicalPotential,ChemicalPotentialData);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "save_velocity", false )){
+    	ASSERT(visData[0].vars[3]->name=="Velocity_x");
+    	ASSERT(visData[0].vars[4]->name=="Velocity_y");
+    	ASSERT(visData[0].vars[5]->name=="Velocity_z");
+    	LeeModel.getVelocity(Vel_x,Vel_y,Vel_z);
+    	Array<double>& VelxData = visData[0].vars[3]->data;
+    	Array<double>& VelyData = visData[0].vars[4]->data;
+    	Array<double>& VelzData = visData[0].vars[5]->data;
+    	fillData.copy(Vel_x,VelxData);
+    	fillData.copy(Vel_y,VelyData);
+    	fillData.copy(Vel_z,VelzData);
+    }
+    
+    if (vis_db->getWithDefault<bool>( "write_silo", true ))
+    	IO::writeData( timestep, visData, Dm->Comm );
+
+/*    if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
+    	char CurrentIDFilename[40];
+    	sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
+    	Averages.AggregateLabels(CurrentIDFilename);
+    }
+*/	
+}

analysis/FreeEnergy.h

@@ -0,0 +1,54 @@
+/*
+ *  averaging tools for electrochemistry
+ */
+
+#ifndef FreeEnergyAnalyzer_INC
+#define FreeEnergyAnalyzer_INC
+
+#include <vector>
+#include "common/Domain.h"
+#include "common/Utilities.h"
+#include "common/MPI.h"
+#include "common/Communication.h"
+#include "analysis/analysis.h"
+#include "analysis/distance.h"
+#include "analysis/Minkowski.h"
+#include "analysis/SubPhase.h"
+#include "IO/MeshDatabase.h"
+#include "IO/Reader.h"
+#include "IO/Writer.h"
+#include "models/FreeLeeModel.h"
+
+class FreeEnergyAnalyzer{
+public:
+	std::shared_ptr <Domain> Dm;
+	double Volume;
+	// input variables
+	double rho_n, rho_w;
+	double nu_n, nu_w;
+	double gamma_wn, beta;
+	double Fx, Fy, Fz;
+
+    //...........................................................................
+    int Nx,Ny,Nz;
+	DoubleArray Rho;	           
+	DoubleArray Phi;			
+	DoubleArray ChemicalPotential;	    
+	DoubleArray Pressure; 	
+	DoubleArray Vel_x;		
+	DoubleArray Vel_y;
+	DoubleArray Vel_z;
+	DoubleArray SDs;
+
+	FreeEnergyAnalyzer(std::shared_ptr <Domain> Dm);
+	~FreeEnergyAnalyzer();
+	
+	void SetParams();
+	void Basic( ScaLBL_FreeLeeModel &LeeModel, int timestep);
+	void WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep);
+
+private:
+	FILE *TIMELOG;
+};
+#endif
+

analysis/morphology.cpp

@@ -702,12 +702,14 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
 
 		if (rank == 0) printf("     delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT);
 		// Now adjust morph_delta
-		double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
+		if (fabs(GrowthEstimate - GrowthPrevious) > 0.0) {
-		GrowthPrevious = GrowthEstimate;
+			double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
-		morph_delta_previous = morph_delta;
+			GrowthPrevious = GrowthEstimate;
-		morph_delta += step_size;
+			morph_delta_previous = morph_delta;
+			morph_delta += step_size;
+		}
 		if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0;
-				
+
 		//MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25);
 		
 		if (morph_delta > 0.0 ){

analysis/runAnalysis.cpp

@@ -706,6 +706,139 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStru
     }
 
 
+    // Initialize the comms
+    for ( int i = 0; i < 1024; i++ )
+        d_comm_used[i] = false;
+    // Initialize the threads
+    int N_threads = db->getWithDefault<int>( "N_threads", 4 );
+    auto method   = db->getWithDefault<std::string>( "load_balance", "default" );
+    createThreads( method, N_threads );
+}
+
+runAnalysis::runAnalysis(  ScaLBL_ColorModel &ColorModel)
+/*	std::shared_ptr<Database> input_db, const RankInfoStruct &rank_info,
+    std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> Dm, int Np,
+    bool Regular, IntArray Map )
+    : d_Np( Np ),
+      d_regular( Regular ),
+      d_rank_info( rank_info ),
+      d_Map( Map ),
+      d_comm( Dm->Comm.dup() ),
+      d_ScaLBL_Comm( ScaLBL_Comm )*/
+{
+	
+	d_comm = ColorModel.Dm->Comm.dup();
+	d_Np = ColorModel.Np;
+	bool Regular = false;
+	
+	auto input_db = ColorModel.db;
+    auto db     = input_db->getDatabase( "Analysis" );
+    auto vis_db = input_db->getDatabase( "Visualization" );
+
+    // Ids of work items to use for dependencies
+    ThreadPool::thread_id_t d_wait_blobID;
+    ThreadPool::thread_id_t d_wait_analysis;
+    ThreadPool::thread_id_t d_wait_vis;
+    ThreadPool::thread_id_t d_wait_restart;
+    ThreadPool::thread_id_t d_wait_subphase;
+
+    char rankString[20];
+    sprintf( rankString, "%05d", ColorModel.Dm->rank() );
+    d_n[0] = ColorModel.Dm->Nx - 2;
+    d_n[1] = ColorModel.Dm->Ny - 2;
+    d_n[2] = ColorModel.Dm->Nz - 2;
+    d_N[0] = ColorModel.Dm->Nx;
+    d_N[1] = ColorModel.Dm->Ny;
+    d_N[2] = ColorModel.Dm->Nz;
+
+    d_restart_interval           = db->getScalar<int>( "restart_interval" );
+    d_analysis_interval          = db->getScalar<int>( "analysis_interval" );
+    d_subphase_analysis_interval = INT_MAX;
+    d_visualization_interval     = INT_MAX;
+    d_blobid_interval            = INT_MAX;
+    if ( db->keyExists( "blobid_interval" ) ) {
+        d_blobid_interval = db->getScalar<int>( "blobid_interval" );
+    }
+    if ( db->keyExists( "visualization_interval" ) ) {
+        d_visualization_interval = db->getScalar<int>( "visualization_interval" );
+    }
+    if ( db->keyExists( "subphase_analysis_interval" ) ) {
+        d_subphase_analysis_interval = db->getScalar<int>( "subphase_analysis_interval" );
+    }
+
+    auto restart_file = db->getScalar<std::string>( "restart_file" );
+    d_restartFile     = restart_file + "." + rankString;
+
+
+    d_rank = d_comm.getRank();
+    writeIDMap( ID_map_struct(), 0, id_map_filename );
+    // Initialize IO for silo
+    IO::initialize( "", "silo", "false" );
+    // Create the MeshDataStruct
+    d_meshData.resize( 1 );
+
+    d_meshData[0].meshName = "domain";
+    d_meshData[0].mesh     = std::make_shared<IO::DomainMesh>(
+        d_rank_info, d_n[0], d_n[1], d_n[2], ColorModel.Dm->Lx, ColorModel.Dm->Ly, ColorModel.Dm->Lz );
+    auto PhaseVar    = std::make_shared<IO::Variable>();
+    auto PressVar    = std::make_shared<IO::Variable>();
+    auto VxVar       = std::make_shared<IO::Variable>();
+    auto VyVar       = std::make_shared<IO::Variable>();
+    auto VzVar       = std::make_shared<IO::Variable>();
+    auto SignDistVar = std::make_shared<IO::Variable>();
+    auto BlobIDVar   = std::make_shared<IO::Variable>();
+
+    if ( vis_db->getWithDefault<bool>( "save_phase_field", true ) ) {
+        PhaseVar->name = "phase";
+        PhaseVar->type = IO::VariableType::VolumeVariable;
+        PhaseVar->dim  = 1;
+        PhaseVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( PhaseVar );
+    }
+
+    if ( vis_db->getWithDefault<bool>( "save_pressure", false ) ) {
+        PressVar->name = "Pressure";
+        PressVar->type = IO::VariableType::VolumeVariable;
+        PressVar->dim  = 1;
+        PressVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( PressVar );
+    }
+
+    if ( vis_db->getWithDefault<bool>( "save_velocity", false ) ) {
+        VxVar->name = "Velocity_x";
+        VxVar->type = IO::VariableType::VolumeVariable;
+        VxVar->dim  = 1;
+        VxVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( VxVar );
+        VyVar->name = "Velocity_y";
+        VyVar->type = IO::VariableType::VolumeVariable;
+        VyVar->dim  = 1;
+        VyVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( VyVar );
+        VzVar->name = "Velocity_z";
+        VzVar->type = IO::VariableType::VolumeVariable;
+        VzVar->dim  = 1;
+        VzVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( VzVar );
+    }
+
+    if ( vis_db->getWithDefault<bool>( "save_distance", false ) ) {
+        SignDistVar->name = "SignDist";
+        SignDistVar->type = IO::VariableType::VolumeVariable;
+        SignDistVar->dim  = 1;
+        SignDistVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( SignDistVar );
+    }
+
+    if ( vis_db->getWithDefault<bool>( "save_connected_components", false ) ) {
+        BlobIDVar->name = "BlobID";
+        BlobIDVar->type = IO::VariableType::VolumeVariable;
+        BlobIDVar->dim  = 1;
+        BlobIDVar->data.resize( d_n[0], d_n[1], d_n[2] );
+        d_meshData[0].vars.push_back( BlobIDVar );
+    }
+
+
     // Initialize the comms
     for ( int i = 0; i < 1024; i++ )
         d_comm_used[i] = false;

analysis/runAnalysis.h

@@ -7,6 +7,7 @@
 #include "common/Communication.h"
 #include "common/ScaLBL.h"
 #include "threadpool/thread_pool.h"
+#include "models/ColorModel.h"
 #include <limits.h>
 
 
@@ -31,6 +32,8 @@ public:
     runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct &rank_info,
         std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> dm, int Np,
         bool Regular, IntArray Map );
+    
+    runAnalysis( ScaLBL_ColorModel &ColorModel);
 
     //! Destructor
     ~runAnalysis();

cmake/FindMPI.cmake

@@ -1,362 +0,0 @@
-# - Message Passing Interface (MPI) module.
-#
-# The Message Passing Interface (MPI) is a library used to write
-# high-performance parallel applications that use message passing, and
-# is typically deployed on a cluster. MPI is a standard interface
-# (defined by the MPI forum) for which many implementations are
-# available. All of these implementations have somewhat different
-# compilation approaches (different include paths, libraries to link
-# against, etc.), and this module tries to smooth out those differences.
-#
-# This module will set the following variables:
-#   MPI_FOUND                  TRUE if we have found MPI
-#   MPI_COMPILE_FLAGS          Compilation flags for MPI programs
-#   MPI_INCLUDE_PATH           Include path(s) for MPI header
-#   MPI_LINK_FLAGS             Linking flags for MPI programs
-#   MPI_LIBRARY                First MPI library to link against (cached)
-#   MPI_EXTRA_LIBRARY          Extra MPI libraries to link against (cached)
-#   MPI_LIBRARIES              All libraries to link MPI programs against
-#   MPIEXEC                    Executable for running MPI programs
-#   MPIEXEC_NUMPROC_FLAG       Flag to pass to MPIEXEC before giving it the
-#                              number of processors to run on
-#   MPIEXEC_PREFLAGS           Flags to pass to MPIEXEC directly before the
-#                              executable to run.
-#   MPIEXEC_POSTFLAGS          Flags to pass to MPIEXEC after all other flags.
-#
-# This module will attempt to auto-detect these settings, first by
-# looking for a MPI compiler, which many MPI implementations provide
-# as a pass-through to the native compiler to simplify the compilation
-# of MPI programs. The MPI compiler is stored in the cache variable
-# MPI_COMPILER, and will attempt to look for commonly-named drivers
-# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
-# recognized, it will be used to set all of the module variables. To
-# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
-# the CMake cache.
-#
-# If no compiler driver is found or the compiler driver is not
-# recognized, this module will then search for common include paths
-# and library names to try to detect MPI.
-#
-# If CMake initially finds a different MPI than was intended, and you
-# want to use the MPI compiler auto-detection for a different MPI
-# implementation, set MPI_COMPILER to the MPI compiler driver you want
-# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
-# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
-# will run again with the newly-specified MPI_COMPILER.
-#
-# When using MPIEXEC to execute MPI applications, you should typically
-# use all of the MPIEXEC flags as follows:
-#   ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
-#     ${MPIEXEC_POSTFLAGS} ARGS
-# where PROCS is the number of processors on which to execute the program,
-# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
-# MPI program.
-
-#=============================================================================
-# Copyright 2001-2009 Kitware, Inc.
-#
-# Distributed under the OSI-approved BSD License (the "License");
-# see accompanying file Copyright.txt for details.
-#
-# This software is distributed WITHOUT ANY WARRANTY; without even the
-# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the License for more information.
-#=============================================================================
-# (To distribute this file outside of CMake, substitute the full
-#  License text for the above reference.)
-
-# This module is maintained by David Partyka <dave.partyka@kitware.com>.
-
-# A set of directories to search through in addition to the standard system paths
-# that find_program will search through.
-# Microsoft HPC SDK is automatically added to the system path
-# Argonne National Labs MPICH2 sets a registry key that we can use.
-
-set(_MPI_PACKAGE_DIR
-  mpi
-  mpich
-  openmpi
-  lib/mpi
-  lib/mpich
-  lib/openmpi
-  "MPICH/SDK"
-  "Microsoft Compute Cluster Pack"
-  "Microsoft HPC Pack 2008 R2"
-  )
-
-set(_MPI_PREFIX_PATH)
-if(WIN32)
-  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
-  list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
-endif()
-
-foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
-  foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
-    if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
-      list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
-    endif()
-  endforeach(MpiPackageDir)
-endforeach(SystemPrefixDir)
-
-# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
-find_program(MPIEXEC
-  NAMES mpiexec mpirun lamexec
-  PATHS ${_MPI_PREFIX_PATH}
-  PATH_SUFFIXES bin
-  DOC "Executable for running MPI programs."
-  )
-
-# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
-# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
-get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
-get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
-
-# If there is an mpi compiler find it and interogate (farther below) it for the include
-# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
-find_program(MPI_COMPILER
-  NAMES mpic++ mpicxx mpiCC mpicc
-  HINTS "${_MPI_BASE_DIR}"
-  PATH_SUFFIXES bin
-  DOC "MPI compiler. Used only to detect MPI compilation flags.")
-mark_as_advanced(MPI_COMPILER)
-
-set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
-set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
-set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
-set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
-mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
-  MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
-
-if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
-  # the cache, and we don't want to override those settings.
-elseif (MPI_COMPILER)
-  # Check whether the -showme:compile option works. This indicates
-  # that we have either Open MPI or a newer version of LAM-MPI, and
-  # implies that -showme:link will also work.
-  # Note that Windows distros do not have an mpi compiler to interogate.
-  exec_program(${MPI_COMPILER}
-    ARGS -showme:compile
-    OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
-    RETURN_VALUE MPI_COMPILER_RETURN)
-
-  if (MPI_COMPILER_RETURN EQUAL 0)
-    # If we appear to have -showme:compile, then we should also have
-    # -showme:link. Try it.
-    exec_program(${MPI_COMPILER}
-      ARGS -showme:link
-      OUTPUT_VARIABLE MPI_LINK_CMDLINE
-      RETURN_VALUE MPI_COMPILER_RETURN)
-
-    # Note that we probably have -showme:incdirs and -showme:libdirs
-    # as well.
-    set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
-  endif (MPI_COMPILER_RETURN EQUAL 0)
-
-  if (MPI_COMPILER_RETURN EQUAL 0)
-    # Do nothing: we have our command lines now
-  else (MPI_COMPILER_RETURN EQUAL 0)
-    # Older versions of LAM-MPI have "-showme". Try it.
-    exec_program(${MPI_COMPILER}
-      ARGS -showme
-      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
-      RETURN_VALUE MPI_COMPILER_RETURN)
-  endif (MPI_COMPILER_RETURN EQUAL 0)
-
-  if (MPI_COMPILER_RETURN EQUAL 0)
-    # Do nothing: we have our command lines now
-  else (MPI_COMPILER_RETURN EQUAL 0)
-    # MPICH uses "-show". Try it.
-    exec_program(${MPI_COMPILER}
-      ARGS -show
-      OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
-      RETURN_VALUE MPI_COMPILER_RETURN)
-  endif (MPI_COMPILER_RETURN EQUAL 0)
-
-  if (MPI_COMPILER_RETURN EQUAL 0)
-    # We have our command lines, but we might need to copy
-    # MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
-    if (NOT MPI_LINK_CMDLINE)
-      SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
-    endif (NOT MPI_LINK_CMDLINE)
-  else (MPI_COMPILER_RETURN EQUAL 0)
-    message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
-  endif (MPI_COMPILER_RETURN EQUAL 0)
-endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-
-if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-  # Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
-  # the cache, and we don't want to override those settings.
-elseif (MPI_COMPILE_CMDLINE)
-  # Extract compile flags from the compile command line.
-  string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
-  set(MPI_COMPILE_FLAGS_WORK)
-  foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
-    if (MPI_COMPILE_FLAGS_WORK)
-      set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
-    else(MPI_COMPILE_FLAGS_WORK)
-      set(MPI_COMPILE_FLAGS_WORK ${FLAG})
-    endif(MPI_COMPILE_FLAGS_WORK)
-  endforeach(FLAG)
-
-  # Extract include paths from compile command line
-  string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
-  set(MPI_INCLUDE_PATH_WORK)
-  foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
-    string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
-    string(REGEX REPLACE "//" "/" IPATH ${IPATH})
-    list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
-  endforeach(IPATH)
-
-  if (NOT MPI_INCLUDE_PATH_WORK)
-    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
-      # The compile command line didn't have any include paths on it,
-      # but we may have -showme:incdirs. Use it.
-      exec_program(${MPI_COMPILER}
-        ARGS -showme:incdirs
-        OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
-        RETURN_VALUE MPI_COMPILER_RETURN)
-      separate_arguments(MPI_INCLUDE_PATH_WORK)
-    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
-  endif (NOT MPI_INCLUDE_PATH_WORK)
-
-  if (NOT MPI_INCLUDE_PATH_WORK)
-    # If all else fails, just search for mpi.h in the normal include
-    # paths.
-    find_path(MPI_INCLUDE_PATH mpi.h
-  HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
-  PATH_SUFFIXES include
-    )
-    set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
-  endif (NOT MPI_INCLUDE_PATH_WORK)
-
-  # Extract linker paths from the link command line
-  string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
-  set(MPI_LINK_PATH)
-  foreach(LPATH ${MPI_ALL_LINK_PATHS})
-    string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
-    string(REGEX REPLACE "//" "/" LPATH ${LPATH})
-    list(APPEND MPI_LINK_PATH ${LPATH})
-  endforeach(LPATH)
-
-  if (NOT MPI_LINK_PATH)
-    if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
-      # The compile command line didn't have any linking paths on it,
-      # but we may have -showme:libdirs. Use it.
-      exec_program(${MPI_COMPILER}
-        ARGS -showme:libdirs
-        OUTPUT_VARIABLE MPI_LINK_PATH
-        RETURN_VALUE MPI_COMPILER_RETURN)
-      separate_arguments(MPI_LINK_PATH)
-    endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
-  endif (NOT MPI_LINK_PATH)
-
-  # Extract linker flags from the link command line
-  string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
-  set(MPI_LINK_FLAGS_WORK)
-  foreach(FLAG ${MPI_ALL_LINK_FLAGS})
-    if (MPI_LINK_FLAGS_WORK)
-      set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
-    else(MPI_LINK_FLAGS_WORK)
-      set(MPI_LINK_FLAGS_WORK ${FLAG})
-    endif(MPI_LINK_FLAGS_WORK)
-  endforeach(FLAG)
-  if ( MPI_LINK_FLAGS_WORK )
-    string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
-  endif ()
-
-  # Extract the set of libraries to link against from the link command
-  # line
-  string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
-
-  # Determine full path names for all of the libraries that one needs
-  # to link against in an MPI program
-  set(MPI_LIBRARIES)
-  foreach(LIB ${MPI_LIBNAMES})
-    string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
-    set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
-    find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
-    if (MPI_LIB)
-      list(APPEND MPI_LIBRARIES ${MPI_LIB})
-    elseif (NOT MPI_FIND_QUIETLY)
-      message(WARNING "Unable to find MPI library ${LIB}")
-    endif ()
-  endforeach(LIB)
-  set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
-
-  # Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
-  # MPI_EXTRA_LIBRARY.
-  list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
-  list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
-  if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
-    list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
-    set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
-  else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
-    set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
-  endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
-  if (MPI_NUMLIBS GREATER 1)
-    set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
-    list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
-    set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
-  else (MPI_NUMLIBS GREATER 1)
-    set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
-  endif (MPI_NUMLIBS GREATER 1)
-
-  # Set up all of the appropriate cache entries
-  set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
-  set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
-  set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
-else (MPI_COMPILE_CMDLINE)
-# No MPI compiler to interogate so attempt to find everything with find functions.
-  find_path(MPI_INCLUDE_PATH mpi.h
-    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
-    PATH_SUFFIXES include Inc
-    )
-
-  # Decide between 32-bit and 64-bit libraries for Microsoft's MPI
-  if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
-    set(MS_MPI_ARCH_DIR amd64)
-  else()
-    set(MS_MPI_ARCH_DIR i386)
-  endif()
-
-  find_library(MPI_LIBRARY
-    NAMES mpi mpich msmpi
-    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
-    PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
-    )
-
-  find_library(MPI_EXTRA_LIBRARY
-    NAMES mpi++
-    HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
-    PATH_SUFFIXES lib
-    DOC "Extra MPI libraries to link against.")
-
-  set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
-  set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
-endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-
-# Set up extra variables to conform to
-if (MPI_EXTRA_LIBRARY)
-  set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
-else (MPI_EXTRA_LIBRARY)
-  set(MPI_LIBRARIES ${MPI_LIBRARY})
-endif (MPI_EXTRA_LIBRARY)
-
-if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-  set(MPI_FOUND TRUE)
-else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-  set(MPI_FOUND FALSE)
-endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
-
-#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
-# handle the QUIETLY and REQUIRED arguments
-#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
-
-mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
-  MPI_EXTRA_LIBRARY)
-
-# unset to cleanup namespace
-unset(_MPI_PACKAGE_DIR)
-unset(_MPI_PREFIX_PATH)
-unset(_MPI_BASE_DIR)

cmake/Find_TIMER.cmake

@@ -4,7 +4,7 @@
 #    CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR )
 # This function assumes that USE_TIMER is set to indicate if the timer should be used
 # If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer
-# If USE_TIMER is not set we will create a summy timer that does nothing.
+# If USE_TIMER is not set we will create a dummy timer that does nothing.
 # The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default.
 # The input argument NULL_TIMER_DIR specifies the location to install the dummy timer.  
 #    If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used.
@@ -13,7 +13,7 @@
 #    TIMER_CXXFLAGS - C++ flags for the timer library
 #    TIMER_LDFLAGS  - Linker flags to link the timer library
 #    TIMER_LDLIBS   - Linker libraries to link the timer library
-FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
+FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR QUIET )
     # Determine if we want to use the timer utility
     CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} )
     SET( TIMER_INCLUDE )
@@ -33,20 +33,23 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
             FIND_LIBRARY( TIMER_LIBS  NAMES timerutility  PATHS ${TIMER_DIRECTORY}/lib  NO_DEFAULT_PATH )
             SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include )
             SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" )
-            SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib )
+            SET( TIMER_LDFLAGS )
-            SET( TIMER_LDLIBS -ltimerutility )
+            SET( TIMER_LDLIBS "${TIMER_LIBS}" )
         ELSE()
             MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported.  Use -D TIMER_DIRECTORY=" )
         ENDIF()
-        SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
+        SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
         INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
         ADD_DEFINITIONS( -DUSE_TIMER )
-        MESSAGE( "Using timer utility" )
+        IF ( NOT QUIET )
-        MESSAGE( "  TIMER_LIBRARIES = ${TIMER_LIBS}" )
+            MESSAGE( STATUS "Using timer utility" )
+            MESSAGE( STATUS "  TIMER_LIBRARIES = ${TIMER_LIBS}" )
+        ENDIF()
     ELSE()
         IF ( "${NULL_TIMER_DIR}" STREQUAL "" )
             SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" )
         ENDIF()
+        # Write ProfilerApp.h
         FILE(WRITE  "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...)       do {} while(0)\n" )
         FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...)        do {} while(0)\n" )
         FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...)      do {} while(0)\n" )
@@ -61,9 +64,25 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
         FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE()  do {} while(0)\n" )
         FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY()  do {} while(0)\n" )
         FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" )
+        # Write MemoryApp.h
+        FILE(WRITE  "${NULL_TIMER_DIR}/MemoryApp.h" "#include <cstring>\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "class MemoryApp final {\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "public:\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   struct MemoryStats {\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "       size_t bytes_new, bytes_delete, N_new, N_delete, tot_bytes_used, system_memory, stack_used, stack_size;\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "       MemoryStats() { memset(this,0,sizeof(MemoryStats)); }\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   };\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   static inline void print( std::ostream& ) {}\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   static inline size_t getMemoryUsage() { return 0; }\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   static inline size_t getTotalMemoryUsage() { return 0; }\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   static inline size_t getSystemMemory() { return 0; }\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "   static inline MemoryStats getMemoryStats() { return MemoryStats(); }\n" )
+        FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "};\n" )
         SET( TIMER_INCLUDE  "${NULL_TIMER_DIR}" )
         INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
-        MESSAGE( "Disabling timer utility" )
+        IF ( NOT QUIET )
+            MESSAGE( STATUS "Disabling timer utility" )
+        ENDIF()
     ENDIF()
     SET( TIMER_INCLUDE  "${TIMER_INCLUDE}"  PARENT_SCOPE )
     SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE )
@@ -88,12 +107,12 @@ MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT )
         SET( ${FLAG} ${DEFAULT} )
     ELSEIF( ${FLAG}  STREQUAL "" )
         SET( ${FLAG} ${DEFAULT} )
-    ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
+    ELSEIF( ( ${${FLAG}} STREQUAL "FALSE" ) OR ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
         SET( ${FLAG} 0 )
-    ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
+    ELSEIF( ( ${${FLAG}} STREQUAL "TRUE" ) OR ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
         SET( ${FLAG} 1 )
     ELSE()
-        MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
+        MESSAGE( FATAL_ERROR "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
     ENDIF ()
 ENDMACRO()
 

cmake/ctest_script.cmake

@@ -32,7 +32,6 @@ SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} )
 SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} )
 SET( LDLIBS             $ENV{LDLIBS}            )
 SET( LDFLAGS            $ENV{LDFLAGS}           )
-SET( MPI_COMPILER       $ENV{MPI_COMPILER}      )
 SET( MPI_DIRECTORY      $ENV{MPI_DIRECTORY}     )
 SET( MPI_INCLUDE        $ENV{MPI_INCLUDE}       )
 SET( MPI_LINK_FLAGS     $ENV{MPI_LINK_FLAGS}    )
@@ -198,7 +197,7 @@ SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLA
 SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" )
 SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
 IF ( USE_MPI )
-    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}")
+    SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}")
     IF ( NOT USE_VALGRIND )
         SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true")
     ENDIF()

cmake/libraries.cmake

@@ -41,93 +41,61 @@ ENDMACRO()
 # Macro to find and configure the MPI libraries
 MACRO( CONFIGURE_MPI )
     # Determine if we want to use MPI
-    CHECK_ENABLE_FLAG(USE_MPI 1 )
+    CHECK_ENABLE_FLAG( USE_MPI 1 )
     IF ( USE_MPI )
-        # Check if we specified the MPI directory
+        MESSAGE( "Configuring MPI" )
-        IF ( MPI_DIRECTORY )
+        IF ( MPIEXEC )
-            # Check the provided MPI directory for include files
+            SET( MPIEXEC_EXECUTABLE ${MPIEXEC} )
-            VERIFY_PATH( "${MPI_DIRECTORY}" )
+        ENDIF()
-            IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" )
+        IF ( NOT MPI_SKIP_SEARCH )
-                SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" )
+            FIND_PACKAGE( MPI )
-            ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" )
-                SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" )
-            ELSE()
-                MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" )
-            ENDIF ()
-            INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
-            SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
-            # Set MPI libraries
-            IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
-                FIND_LIBRARY( MSMPI_LIB     NAMES msmpi     PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
-                FIND_LIBRARY( MSMPI_LIB     NAMES msmpi     PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
-                FIND_LIBRARY( MSMPIFEC_LIB  NAMES msmpifec  PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
-                FIND_LIBRARY( MSMPIFEC_LIB  NAMES msmpifec  PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
-                FIND_LIBRARY( MSMPIFMC_LIB  NAMES msmpifmc  PATHS "${MPI_DIRECTORY}/Lib/x64"    NO_DEFAULT_PATH )
-                FIND_LIBRARY( MSMPIFMC_LIB  NAMES msmpifmc  PATHS "${MPI_DIRECTORY}/Lib/amd64"  NO_DEFAULT_PATH )
-                SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} )
-            ENDIF()
-            # Set the mpi executable
-            IF ( MPIEXEC ) 
-                # User specified the MPI command directly, use as is
-            ELSEIF ( MPIEXEC_CMD )
-                # User specified the name of the MPI executable
-                SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
-                IF ( NOT EXISTS ${MPIEXEC} )
-                    MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
-                ENDIF ()
-            ELSE ()
-                # Search for the MPI executable in the current directory
-                FIND_PROGRAM( MPIEXEC  NAMES mpiexec mpirun lamexec  PATHS ${MPI_DIRECTORY}/bin  NO_DEFAULT_PATH )
-                IF ( NOT MPIEXEC )
-                    MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
-                ENDIF()
-            ENDIF ()
-            # Set MPI flags
-            IF ( NOT MPIEXEC_NUMPROC_FLAG )
-                SET( MPIEXEC_NUMPROC_FLAG "-np" )
-            ENDIF()
-        ELSEIF ( MPI_COMPILER )
-            # The mpi compiler should take care of everything
-            IF ( MPI_INCLUDE )
-                INCLUDE_DIRECTORIES( ${MPI_INCLUDE} )
-            ENDIF()
         ELSE()
-            # Perform the default search for MPI
+            # Write mpi test
-            INCLUDE ( FindMPI )
+            SET( MPI_TEST_SRC "${CMAKE_CURRENT_BINARY_DIR}/test_mpi.cpp" )
-            IF ( NOT MPI_FOUND )
+            FILE(WRITE  ${MPI_TEST_SRC} "#include <mpi.h>\n" )
-                MESSAGE( "  MPI_INCLUDE = ${MPI_INCLUDE}" )
+            FILE(APPEND ${MPI_TEST_SRC} "int main(int argc, char** argv) {\n" )
-                MESSAGE( "  MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
+            FILE(APPEND ${MPI_TEST_SRC} "    MPI_Init(NULL, NULL);\n")
-                MESSAGE( "  MPI_LIBRARIES = ${MPI_LIBRARIES}" )
+            FILE(APPEND ${MPI_TEST_SRC} "    MPI_Finalize();\n" )
-                MESSAGE( FATAL_ERROR "Did not find MPI" )
+            FILE(APPEND ${MPI_TEST_SRC} "}\n" )
-            ENDIF ()
+            # Test the compile
-            INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" )
+            IF ( CMAKE_CXX_COMPILER )
-            SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" )
+                SET( TMP_FLAGS -DINCLUDE_DIRECTORIES=${MPI_CXX_INCLUDE_PATH} )
+                TRY_COMPILE( MPI_TEST_CXX ${CMAKE_CURRENT_BINARY_DIR} ${MPI_TEST_SRC}
+                    CMAKE_FLAGS ${TMP_FLAGS}
+                    LINK_OPTIONS ${MPI_CXX_LINK_FLAGS}
+                    LINK_LIBRARIES ${MPI_CXX_LIBRARIES}
+                    OUTPUT_VARIABLE OUT_TXT)
+                IF ( NOT ${MPI_TEST} )
+                    MESSAGE( FATAL_ERROR "Skipping MPI search and default compile fails:\n${OUT_TXT}" )
+                ENDIF()
+                SET( MPI_C_FOUND TRUE )
+                SET( MPI_CXX_FOUND TRUE )
+                SET( MPI_Fortran_FOUND TRUE )
+            ENDIF()
         ENDIF()
-        # Check if we need to use MPI for serial tests
+        STRING( STRIP "${MPI_CXX_COMPILE_FLAGS}" MPI_CXX_COMPILE_FLAGS )
-        CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
+        STRING( STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS )
-        # Set defaults if they have not been set
+        STRING( STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES )
-        IF ( NOT MPIEXEC )
+        MESSAGE( "   MPI_CXX_FOUND = ${MPI_CXX_FOUND}" )
-            SET( MPIEXEC mpirun )
+        MESSAGE( "   MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}" )
+        MESSAGE( "   MPI_CXX_COMPILE_FLAGS = ${MPI_CXX_COMPILE_FLAGS}" )
+        MESSAGE( "   MPI_CXX_INCLUDE_PATH = ${MPI_CXX_INCLUDE_PATH}" )
+        MESSAGE( "   MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}" )
+        MESSAGE( "   MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}" )
+        MESSAGE( "   MPIEXEC = ${MPIEXEC}" )
+        MESSAGE( "   MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
+        MESSAGE( "   MPIEXEC_PREFLAGS = ${MPIEXEC_PREFLAGS}" )
+        MESSAGE( "   MPIEXEC_POSTFLAGS = ${MPIEXEC_POSTFLAGS}" )
+        ADD_DEFINITIONS( -DUSE_MPI )
+        INCLUDE_DIRECTORIES( ${MPI_CXX_INCLUDE_PATH} )
+        SET( MPI_LIBRARIES ${MPI_CXX_LIBRARIES} )
+        SET( MPI_LINK_FLAGS ${MPI_CXX_LINK_FLAGS} )
+        IF ( NOT MPI_CXX_FOUND )
+            MESSAGE( FATAL_ERROR "MPI not found" )
         ENDIF()
-        IF ( NOT MPIEXEC_NUMPROC_FLAG )
+        IF ( USE_MPI AND NOT MPIEXEC )
-            SET( MPIEXEC_NUMPROC_FLAG "-np" )
+            MESSAGE( FATAL_ERROR "Unable to find MPIEXEC, please set it before continuing" )
         ENDIF()
-        # Set the definitions
-        ADD_DEFINITIONS( "-DUSE_MPI" )  
-        MESSAGE( "Using MPI" )
-        MESSAGE( "  MPIEXEC = ${MPIEXEC}" )
-        MESSAGE( "  MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
-        MESSAGE( "  MPI_INCLUDE = ${MPI_INCLUDE}" )
-        MESSAGE( "  MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
-        MESSAGE( "  MPI_LIBRARIES = ${MPI_LIBRARIES}" )
-    ELSE()
-        SET( USE_MPI_FOR_SERIAL_TESTS 0 )
-        SET( MPIEXEC "" )
-        SET( MPIEXEC_NUMPROC_FLAG "" )
-        SET( MPI_INCLUDE "" )
-        SET( MPI_LINK_FLAGS "" )
-        SET( MPI_LIBRARIES "" )
-        MESSAGE( "Not using MPI, all parallel tests will be disabled" )
     ENDIF()
 ENDMACRO()
 

cmake/macros.cmake

@@ -681,8 +681,8 @@ MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
     FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
         TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
     ENDFOREACH()
-    FOREACH ( MPI_LIBRARIES )
+    FOREACH ( tmp ${MPI_LIBRARIES} )
-        TARGET_LINK_LIBRARIES( ${EXE} ${ARGN} ${tmp} )
+        TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
     ENDFOREACH()
     FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
         ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )

common/ScaLBL.cpp

@@ -516,9 +516,9 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
 				n = k*Nx*Ny+j*Nx+i;
 				if (id[n] > 0){
 					// Counts for the six faces
-					if (i>0 && i<=width)        Map(n)=idx++;
+					if (i>0 && i<=width)              Map(n)=idx++;
-					else if (j>0 && j<=width)  Map(n)=idx++;
+					else if (j>0 && j<=width)         Map(n)=idx++;
-					else if (k>0 && k<=width)  Map(n)=idx++;
+					else if (k>0 && k<=width) 		  Map(n)=idx++;
 					else if (i>Nx-width-2 && i<Nx-1)  Map(n)=idx++;
 					else if (j>Ny-width-2 && j<Ny-1)  Map(n)=idx++;
 					else if (k>Nz-width-2 && k<Nz-1)  Map(n)=idx++;
@@ -1623,35 +1623,31 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){
 	// Pack the distributions
 	//...Packing for x face(2,8,10,12,14)................................
 	ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N);
-
-	//...Packing for X face(1,7,9,11,13)................................
-	ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
-
-	//...Packing for y face(4,8,9,16,18).................................
-	ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
-
-	//...Packing for Y face(3,7,10,15,17).................................
-	ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
-
-	//...Packing for z face(6,12,13,16,17)................................
-	ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
-
-	//...Packing for Z face(5,11,14,15,18)................................
-	ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
-
-	//...................................................................................
-	// Send all the distributions
-	//...................................................................................
 	req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
 	req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
+	
+	//...Packing for X face(1,7,9,11,13)................................
+	ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
 	req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
 	req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
+	
+	//...Packing for y face(4,8,9,16,18).................................
+	ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
 	req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
 	req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
+	
+	//...Packing for Y face(3,7,10,15,17).................................
+	ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
 	req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
 	req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
+	
+	//...Packing for z face(6,12,13,16,17)................................
+	ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
 	req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
 	req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
+	
+	//...Packing for Z face(5,11,14,15,18)................................
+	ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
 	req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
 	req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
 }

common/ScaLBL.h

@@ -168,6 +168,12 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double
 extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi, 
 			int start, int finish, int Np);
 
+extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
+
 extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz);
 
 extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz);
@@ -200,18 +206,25 @@ extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double
 extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, 
                                                     double rhonA, double rhoB, double tauM, double W, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, 
+//extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, 
-                                                          double rhoA, double rhoB, int start, int finish, int Np);
+ //                                                         double rhoA, double rhoB, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, 
+//extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, 
-			                                               double rhoA, double rhoB, int start, int finish, int Np);
+//			                                               double rhoA, double rhoB, int start, int finish, int Np);
+extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
+                                                          double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *hq, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
+extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
-                                                double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
+		double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map,  double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np);
+
+extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, 
+                                                double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np);
 
-extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *hq, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
+extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
-                                                double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
+                                                double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np);
 
 extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,  
@@ -220,6 +233,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis
 extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, 
                                                                  double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
 
+extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np);
 
 // BOUNDARY CONDITION ROUTINES
 

common/WideHalo.cpp

@@ -234,59 +234,59 @@ void ScaLBLWideHalo_Communicator::Send(double *data){
 	//...................................................................................
 	// Send / Recv all the phase indcator field values
 	//...................................................................................
-	req1[0] = MPI_COMM_SCALBL.Isend(&sendCount_x,1,rank_x,sendtag+0);
+	req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x,sendCount_x,rank_x,sendtag+0);
-	req2[0] = MPI_COMM_SCALBL.Irecv(&recvCount_X,1,rank_X,recvtag+0);
+	req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag+0);
-	req1[1] = MPI_COMM_SCALBL.Isend(&sendCount_X,1,rank_X,sendtag+1);
+	req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X,sendCount_X,rank_X,sendtag+1);
-	req2[1] = MPI_COMM_SCALBL.Irecv(&recvCount_x,1,rank_x,recvtag+1);
+	req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag+1);
-	req1[2] = MPI_COMM_SCALBL.Isend(&sendCount_y,1,rank_y,sendtag+2);
+	req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y,sendCount_y,rank_y,sendtag+2);
-	req2[2] = MPI_COMM_SCALBL.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
+	req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag+2);
-	req1[3] = MPI_COMM_SCALBL.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
+	req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag+3);
-	req2[3] = MPI_COMM_SCALBL.Irecv(&recvCount_y,1,rank_y,recvtag+3);
+	req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag+3);
-	req1[4] = MPI_COMM_SCALBL.Isend(&sendCount_z,1,rank_z,sendtag+4);
+	req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z,sendCount_z,rank_z,sendtag+4);
-	req2[4] = MPI_COMM_SCALBL.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
+	req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag+4);
-	req1[5] = MPI_COMM_SCALBL.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
+	req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag+5);
-	req2[5] = MPI_COMM_SCALBL.Irecv(&recvCount_z,1,rank_z,recvtag+5);
+	req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag+5);
-	req1[6] = MPI_COMM_SCALBL.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
+	req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag+6);
-	req2[6] = MPI_COMM_SCALBL.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
+	req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag+6);
-	req1[7] = MPI_COMM_SCALBL.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
+	req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag+7);
-	req2[7] = MPI_COMM_SCALBL.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
+	req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag+7);
-	req1[8] = MPI_COMM_SCALBL.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
+	req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag+8);
-	req2[8] = MPI_COMM_SCALBL.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
+	req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag+8);
-	req1[9] = MPI_COMM_SCALBL.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
+	req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag+9);
-	req2[9] = MPI_COMM_SCALBL.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
+	req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag+9);
-	req1[10] = MPI_COMM_SCALBL.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
+	req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag+10);
-	req2[10] = MPI_COMM_SCALBL.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
+	req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag+10);
-	req1[11] = MPI_COMM_SCALBL.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
+	req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag+11);
-	req2[11] = MPI_COMM_SCALBL.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
+	req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag+11);
-	req1[12] = MPI_COMM_SCALBL.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
+	req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag+12);
-	req2[12] = MPI_COMM_SCALBL.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
+	req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag+12);
-	req1[13] = MPI_COMM_SCALBL.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
+	req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag+13);
-	req2[13] = MPI_COMM_SCALBL.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
+	req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag+13);
-	req1[14] = MPI_COMM_SCALBL.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
+	req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag+14);
-	req2[14] = MPI_COMM_SCALBL.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
+	req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag+14);
-	req1[15] = MPI_COMM_SCALBL.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
+	req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag+15);
-	req2[15] = MPI_COMM_SCALBL.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
+	req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag+15);
-	req1[16] = MPI_COMM_SCALBL.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
+	req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag+16);
-	req2[16] = MPI_COMM_SCALBL.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
+	req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag+16);
-	req1[17] = MPI_COMM_SCALBL.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
+	req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag+17);
-	req2[17] = MPI_COMM_SCALBL.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
+	req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag+17);
 	/* Corners */
-	req1[18] = MPI_COMM_SCALBL.Isend(&sendCount_xyz,1,rank_xyz,sendtag+18);
+	req1[18] = MPI_COMM_SCALBL.Isend(sendbuf_xyz,sendCount_xyz,rank_xyz,sendtag+18);
-	req2[18] = MPI_COMM_SCALBL.Irecv(&recvCount_XYZ,1,rank_XYZ,recvtag+18);
+	req2[18] = MPI_COMM_SCALBL.Irecv(recvbuf_XYZ,recvCount_XYZ,rank_XYZ,recvtag+18);
-	req1[19] = MPI_COMM_SCALBL.Isend(&sendCount_XYz,1,rank_XYz,sendtag+19);
+	req1[19] = MPI_COMM_SCALBL.Isend(sendbuf_XYz,sendCount_XYz,rank_XYz,sendtag+19);
-	req2[19] = MPI_COMM_SCALBL.Irecv(&recvCount_xyZ,1,rank_xyZ,recvtag+19);
+	req2[19] = MPI_COMM_SCALBL.Irecv(recvbuf_xyZ,recvCount_xyZ,rank_xyZ,recvtag+19);
-	req1[20] = MPI_COMM_SCALBL.Isend(&sendCount_Xyz,1,rank_Xyz,sendtag+20);
+	req1[20] = MPI_COMM_SCALBL.Isend(sendbuf_Xyz,sendCount_Xyz,rank_Xyz,sendtag+20);
-	req2[20] = MPI_COMM_SCALBL.Irecv(&recvCount_xYZ,1,rank_xYZ,recvtag+20);
+	req2[20] = MPI_COMM_SCALBL.Irecv(recvbuf_xYZ,recvCount_xYZ,rank_xYZ,recvtag+20);
-	req1[21] = MPI_COMM_SCALBL.Isend(&sendCount_xYz,1,rank_xYz,sendtag+21);
+	req1[21] = MPI_COMM_SCALBL.Isend(sendbuf_xYz,sendCount_xYz,rank_xYz,sendtag+21);
-	req2[21] = MPI_COMM_SCALBL.Irecv(&recvCount_XyZ,1,rank_XyZ,recvtag+21);
+	req2[21] = MPI_COMM_SCALBL.Irecv(recvbuf_XyZ,recvCount_XyZ,rank_XyZ,recvtag+21);
-	req1[22] = MPI_COMM_SCALBL.Isend(&sendCount_xyZ,1,rank_xyZ,sendtag+22);
+	req1[22] = MPI_COMM_SCALBL.Isend(sendbuf_xyZ,sendCount_xyZ,rank_xyZ,sendtag+22);
-	req2[22] = MPI_COMM_SCALBL.Irecv(&recvCount_XYz,1,rank_XYz,recvtag+22);
+	req2[22] = MPI_COMM_SCALBL.Irecv(recvbuf_XYz,recvCount_XYz,rank_XYz,recvtag+22);
-	req1[23] = MPI_COMM_SCALBL.Isend(&sendCount_XYZ,1,rank_XYZ,sendtag+23);
+	req1[23] = MPI_COMM_SCALBL.Isend(sendbuf_XYZ,sendCount_XYZ,rank_XYZ,sendtag+23);
-	req2[23] = MPI_COMM_SCALBL.Irecv(&recvCount_xyz,1,rank_xyz,recvtag+23);
+	req2[23] = MPI_COMM_SCALBL.Irecv(recvbuf_xyz,recvCount_xyz,rank_xyz,recvtag+23);
-	req1[24] = MPI_COMM_SCALBL.Isend(&sendCount_XyZ,1,rank_XyZ,sendtag+24);
+	req1[24] = MPI_COMM_SCALBL.Isend(sendbuf_XyZ,sendCount_XyZ,rank_XyZ,sendtag+24);
-	req2[24] = MPI_COMM_SCALBL.Irecv(&recvCount_xYz,1,rank_xYz,recvtag+24);
+	req2[24] = MPI_COMM_SCALBL.Irecv(recvbuf_xYz,recvCount_xYz,rank_xYz,recvtag+24);
-	req1[25] = MPI_COMM_SCALBL.Isend(&sendCount_xYZ,1,rank_xYZ,sendtag+25);
+	req1[25] = MPI_COMM_SCALBL.Isend(sendbuf_xYZ,sendCount_xYZ,rank_xYZ,sendtag+25);
-	req2[25] = MPI_COMM_SCALBL.Irecv(&recvCount_Xyz,1,rank_Xyz,recvtag+25);
+	req2[25] = MPI_COMM_SCALBL.Irecv(recvbuf_Xyz,recvCount_Xyz,rank_Xyz,recvtag+25);
 	//...................................................................................
 
 }
@@ -302,6 +302,9 @@ void ScaLBLWideHalo_Communicator::Recv(double *data){
 	Utilities::MPI::waitAll(26,req2);
 	ScaLBL_DeviceBarrier();
 	//...................................................................................
+	//printf("Ready to unpack %i to x\n",recvCount_x);
+	//printf("   print first 10 values...\n");
+	//for (int idx=0; idx<10; idx++) printf("     recvBuf[%i]=%f \n",idx,recvbuf_x[idx]);
 	ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh);
 	ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh);
 	ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh);

cpu/Color.cpp

@@ -2489,10 +2489,200 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
 	}	
 }
 
+
+extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
+
+	int nr1,nr2,nr3,nr4,nr5,nr6;
+	double nA,nB; // number density
+	double a1,b1,a2,b2,nAB,delta;
+	double C,nx,ny,nz; //color gradient magnitude and direction
+	double ux,uy,uz;
+	double phi;
+	// Instantiate mass transport distributions
+	// Stationary value - distribution 0
+	for (int n=start; n<finish; n++){
+		/* neighbors */
+		nr1 = neighborList[n+0*Np];
+		nr2 = neighborList[n+1*Np];
+		nr3 = neighborList[n+2*Np];
+		nr4 = neighborList[n+3*Np];
+		nr5 = neighborList[n+4*Np];
+		nr6 = neighborList[n+5*Np];
+		
+		/* load velocity */
+		ux = Vel[n];
+		uy = Vel[Np+n];
+		uz = Vel[2*Np+n];
+	
+		/* load color gradient */
+		nx = ColorGrad[n];
+		ny = ColorGrad[Np+n];
+		nz = ColorGrad[2*Np+n];
+		C = sqrt(nx*nx+ny*ny+nz*nz);
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
+		
+		// read the component number densities
+		nA = Den[n];
+		nB = Den[Np + n];
+
+		// compute phase indicator field
+		phi=(nA-nB)/(nA+nB);
+		nAB = 1.0/(nA+nB);
+		Aq[n] = 0.3333333333333333*nA;
+		Bq[n] = 0.3333333333333333*nB;
+
+		//...............................................
+		// q = 0,2,4
+		// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+		delta = beta*nA*nB*nAB*0.1111111111111111*nx;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+
+		// q = 1
+		//nread = neighborList[n+Np];
+		Aq[nr2] = a1;
+		Bq[nr2] = b1;
+		// q=2
+		//nread = neighborList[n];
+		Aq[nr1] = a2;
+		Bq[nr1] = b2;
+
+		//...............................................
+		// Cq = {0,1,0}
+		delta = beta*nA*nB*nAB*0.1111111111111111*ny;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+
+		// q = 3
+		//nread = neighborList[n+3*Np];
+		Aq[nr4] = a1;
+		Bq[nr4] = b1;
+		// q = 4
+		//nread = neighborList[n+2*Np];
+		Aq[nr3] = a2;
+		Bq[nr3] = b2;
+
+		//...............................................
+		// q = 4
+		// Cq = {0,0,1}
+		delta = beta*nA*nB*nAB*0.1111111111111111*nz;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+
+		// q = 5
+		//nread = neighborList[n+5*Np];
+		Aq[nr6] = a1;
+		Bq[nr6] = b1;
+		// q = 6
+		//nread = neighborList[n+4*Np];
+		Aq[nr5] = a2;
+		Bq[nr5] = b2;
+		//...............................................
+	}	
+}
+
+extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den, 
+		double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta,  int start, int finish, int Np){
+
+	double nA,nB; // number density
+	double a1,b1,a2,b2,nAB,delta;
+	double C,nx,ny,nz; //color gradient magnitude and direction
+	double ux,uy,uz;
+	double phi;
+	// Instantiate mass transport distributions
+	// Stationary value - distribution 0
+	for (int n=start; n<finish; n++){
+		/* load velocity */
+		ux = Vel[n];
+		uy = Vel[Np+n];
+		uz = Vel[2*Np+n];
+	
+		/* load color gradient */
+		nx = ColorGrad[n];
+		ny = ColorGrad[Np+n];
+		nz = ColorGrad[2*Np+n];
+		C = sqrt(nx*nx+ny*ny+nz*nz);
+		double ColorMag = C;
+		if (C==0.0) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;	
+		
+		// read the component number densities
+		nA = Den[n];
+		nB = Den[Np + n];
+		
+		nAB = 1.0/(nA+nB);
+		Aq[n] = 0.3333333333333333*nA;
+		Bq[n] = 0.3333333333333333*nB;
+
+		//...............................................
+		// q = 0,2,4
+		// Cq = {1,0,0}, {0,1,0}, {0,0,1}
+		delta = beta*nA*nB*nAB*0.1111111111111111*nx;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
+
+		Aq[1*Np+n] = a1;
+		Bq[1*Np+n] = b1;
+		Aq[2*Np+n] = a2;
+		Bq[2*Np+n] = b2;
+
+		//...............................................
+		// q = 2
+		// Cq = {0,1,0}
+		delta = beta*nA*nB*nAB*0.1111111111111111*ny;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
+
+		Aq[3*Np+n] = a1;
+		Bq[3*Np+n] = b1;
+		Aq[4*Np+n] = a2;
+		Bq[4*Np+n] = b2;
+		//...............................................
+		// q = 4
+		// Cq = {0,0,1}
+		delta = beta*nA*nB*nAB*0.1111111111111111*nz;
+		if (!(nA*nB*nAB>0)) delta=0;
+		a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
+		b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
+		a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
+		b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
+
+		Aq[5*Np+n] = a1;
+		Bq[5*Np+n] = b1;
+		Aq[6*Np+n] = a2;
+		Bq[6*Np+n] = b2;
+		//...............................................
+
+	}	
+}
+
+		
 extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq, 
 			double *Den, double *Phi, int start, int finish, int Np){
 
-	int idx,n,nread;
+	int idx,nread;
 	double fq,nA,nB;
 
 	for (int n=start; n<finish; n++){
@@ -2579,7 +2769,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
 
 extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi, 
 			int start, int finish, int Np){
-	int idx,n,nread;
+	int idx,nread;
 	double fq,nA,nB;
 	for (int n=start; n<finish; n++){
 		

cpu/FreeLee.cpp

@@ -1,4 +1,5 @@
 #include <math.h>
+#include <stdio.h>
 
 #define STOKES
 
@@ -70,6 +71,8 @@ extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double
 		gqbar[17*Np+n] = 0.0277777777777778*(p-0.5*(Fy-Fz)); ;  //double(100*n)+17.f;
 		gqbar[18*Np+n] = 0.0277777777777778*(p-0.5*(-Fy+Fz));;  //double(100*n)+18.f;
 	}
+	
+	
 }
 
 extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad, 
@@ -101,7 +104,8 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl
 		nz = nz/ColorMag_temp;		
 
         theta = M*cs2_inv*(1-4.0*phi*phi)/W;
-
+        theta = 0; // try more diffusive initial condition
+        
 		hq[0*Np+idx]=0.3333333333333333*(phi);
 		hq[1*Np+idx]=0.1111111111111111*(phi+theta*nx);
 		hq[2*Np+idx]=0.1111111111111111*(phi-theta*nx);
@@ -116,7 +120,7 @@ extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, doubl
 extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, 
                                                           double rhoA, double rhoB, int start, int finish, int Np){
 
-	int idx,n,nread;
+	int idx,nread;
 	double fq,phi;
 
 	for (int n=start; n<finish; n++){
@@ -161,12 +165,15 @@ extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int
 		// save the phase indicator field
 		idx = Map[n];
 		Phi[idx] = phi; 
+		
 	}
 }
 
+
+
 extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi, 
 			                                               double rhoA, double rhoB, int start, int finish, int Np){
-	int idx,n;
+	int idx;
 	double fq,phi;
 	for (int n=start; n<finish; n++){
 		
@@ -207,11 +214,189 @@ extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq,
 	}	
 }
 
-extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *hq, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, 
+extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
-                                                double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
+                                                          double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){
+
+	int idx,nr1,nr2,nr3,nr4,nr5,nr6;
+	double h0,h1,h2,h3,h4,h5,h6;
+	double nx,ny,nz,C;
+	double ux,uy,uz;
+	double phi;
+    double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
+    double factor = 1.0;
+
+	for (int n=start; n<finish; n++){
+
+		/* load phase indicator field */
+		idx = Map[n];
+		phi = Phi[idx];
+
+		/* velocity */
+		ux = Vel[0*Np+n];
+		uy = Vel[1*Np+n];
+		uz = Vel[2*Np+n];
+
+        /*color gradient */
+		nx = ColorGrad[0*Np+n];
+		ny = ColorGrad[1*Np+n];
+		nz = ColorGrad[2*Np+n];
+		
+		//Normalize the Color Gradient
+		C = sqrt(nx*nx+ny*ny+nz*nz);
+		double ColorMag = C;
+		if (C < 1.0e-12) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;		
+
+		// q=1
+		nr1 = neighborList[n]; 
+		nr2 = neighborList[n+Np]; 
+		nr3 = neighborList[n+2*Np]; 
+		nr4 = neighborList[n+3*Np];
+		nr5 = neighborList[n+4*Np];
+		nr6 = neighborList[n+5*Np];
+		
+		//q=0
+		h0 = hq[n];
+		//q=1
+		h1 = hq[nr1]; 
+		//q=2
+		h2 = hq[nr2];  
+		//q=3
+		h3 = hq[nr3];
+		//q=4
+		h4 = hq[nr4];
+		//q=5
+		h5 = hq[nr5];
+		//q=6
+		h6 = hq[nr6];
+
+        //-------------------------------- BGK collison for phase field ---------------------------------//
+		h0 -= (h0 - 0.3333333333333333*phi)/tauM;
+		h1 -= (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - factor*phi*phi))/W)/tauM;
+		h2 -= (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - factor*phi*phi))/W)/tauM;
+		h3 -= (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - factor*phi*phi))/W)/tauM;
+		h4 -= (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - factor*phi*phi))/W)/tauM;
+		h5 -= (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - factor*phi*phi))/W)/tauM;
+		h6 -= (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - factor*phi*phi))/W)/tauM;
+		//........................................................................
+		
+		/*Update the distributions */
+		// q = 0
+		hq[n] = h0;
+		hq[nr2] = h1;
+		hq[nr1] = h2;
+		hq[nr4] = h3;
+		hq[nr3] = h4;
+		hq[nr6] = h5;
+		hq[nr5] = h6;
+		//........................................................................
+
+	}
+}
+
+
+
+extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
+		double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np){
+
+	int idx,n;
+	double h0,h1,h2,h3,h4,h5,h6;
+	double nx,ny,nz,C;
+	double ux,uy,uz;
+	double phi;
+    double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
+    double factor = 1.0;
+
+	for (int n=start; n<finish; n++){
+
+		/* load phase indicator field */
+		idx = Map[n];
+		phi = Phi[idx];
+
+		/* velocity */
+		ux = Vel[0*Np+n];
+		uy = Vel[1*Np+n];
+		uz = Vel[2*Np+n];
+
+		/*color gradient */
+		nx = ColorGrad[0*Np+n];
+		ny = ColorGrad[1*Np+n];
+		nz = ColorGrad[2*Np+n];
+		//Normalize the Color Gradient
+		C = sqrt(nx*nx+ny*ny+nz*nz);
+		double ColorMag = C;
+		if (C < 1.0e-12) ColorMag=1.0;
+		nx = nx/ColorMag;
+		ny = ny/ColorMag;
+		nz = nz/ColorMag;
+
+		h0 = hq[n];
+		h1 = hq[2*Np+n]; 
+		h2 = hq[Np+n];  
+		h3 = hq[4*Np+n];
+		h4 = hq[3*Np+n];
+		h5 = hq[6*Np+n];
+		h6 = hq[5*Np+n];
+
+		//-------------------------------- BGK collison for phase field ---------------------------------//
+		h0 -= (h0 - 0.3333333333333333*phi)/tauM;
+		h1 -= (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - factor*phi*phi))/W)/tauM;
+		h2 -= (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - factor*phi*phi))/W)/tauM;
+		h3 -= (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - factor*phi*phi))/W)/tauM;
+		h4 -= (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - factor*phi*phi))/W)/tauM;
+		h5 -= (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - factor*phi*phi))/W)/tauM;
+		h6 -= (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - factor*phi*phi))/W)/tauM;
+		//........................................................................
+
+		/*Update the distributions */
+		// q = 0
+		hq[n] = h0;
+		hq[Np+n] = h1;
+		hq[2*Np+n] = h2;
+		hq[3*Np+n] = h3;
+		hq[4*Np+n] = h4;
+		hq[5*Np+n] = h5;
+		hq[6*Np+n] = h6;
+		//........................................................................
+		
+	}
+}
+
+extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map,  double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np){
+	int idx,n;
+	double h0,h1,h2,h3,h4,h5,h6;
+	double phi;
+
+	for (int n=start; n<finish; n++){
+
+		h0 = hq[n];
+		h1 = hq[1*Np+n]; 
+		h2 = hq[2*Np+n];  
+		h3 = hq[3*Np+n];
+		h4 = hq[4*Np+n];
+		h5 = hq[5*Np+n];
+		h6 = hq[6*Np+n];
+
+		phi = h0+h1+h2+h3+h4+h5+h6;
+
+		// save the number densities
+		Den[n] = rhoA + 0.5*(1.0-phi)*(rhoB-rhoA);
+
+		// save the phase indicator field
+		idx = Map[n];
+		Phi[idx] = phi; 
+
+	}
+}
+
+
+extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad, 
+                                                double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np){
 	
-	int n,nn,nn2x,ijk;
+	int nn,nn2x,ijk;
 	int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
     double ux,uy,uz;//fluid velocity 
     double p;//pressure
@@ -228,20 +413,22 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
     double mgx,mgy,mgz;//mixed gradient reaching secondary neighbor
 
 	//double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
-    double h0,h1,h2,h3,h4,h5,h6;//distributions for LB phase field
+    //double h0,h1,h2,h3,h4,h5,h6;//distributions for LB phase field
 	double tau;//position dependent LB relaxation time for fluid
-    double C,theta;
+    //double C,theta;
-    double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
+    // double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
 
 	for (int n=start; n<finish; n++){
 
 		rho0 = Den[n];//load density
-		// local relaxation time
-		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
 
 		// Get the 1D index based on regular data layout
 		ijk = Map[n];
         phi = Phi[ijk];// load phase field
+
+		// local relaxation time
+		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
+
 		//					COMPUTE THE COLOR GRADIENT
 		//........................................................................
 		//.................Read Phase Indicator Values............................
@@ -741,62 +928,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
         0.1111111111111111*(-4*chem + (rhoA - rhoB)*(ux*ux + 2*uy + uy*uy + (-2 + uz)*uz))));
         //----------------------------------------------------------------------------------------------------------------------------------------//
 
-
-        // ----------------------------- compute phase field evolution ----------------------------------------
-		//Normalize the Color Gradient
-		C = sqrt(nx*nx+ny*ny+nz*nz);
-		double ColorMag = C;
-		if (C==0.0) ColorMag=1.0;
-		nx = nx/ColorMag;
-		ny = ny/ColorMag;
-		nz = nz/ColorMag;		
-        //compute surface tension-related parameter
-        theta = M*4.5*(1-4.0*phi*phi)/W;
-
-        //load distributions of phase field
-		//q=0
-		h0 = hq[n];
-		//q=1
-		h1 = hq[nr1]; 
-
-		//q=2
-		h2 = hq[nr2];  
-
-		//q=3
-		h3 = hq[nr3];
-
-		//q=4
-		h4 = hq[nr4];
-
-		//q=5
-		h5 = hq[nr5];
-
-		//q=6
-		h6 = hq[nr6];
-
-        //-------------------------------- BGK collison for phase field ---------------------------------//
-		// q = 0
-		hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM;
-
-		// q = 1
-		hq[nr2] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 2
-		hq[nr1] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 3
-		hq[nr4] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 4
-		hq[nr3] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 5
-		hq[nr6] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 6
-		hq[nr5] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM;
-		//........................................................................
-
         //Update velocity on device
 		Vel[0*Np+n] = ux;
 		Vel[1*Np+n] = uy;
@@ -813,11 +944,11 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, dou
 	}
 }
 
-extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *hq, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
+extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den,	double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
-                                                double rhoA, double rhoB, double tauA, double tauB, double tauM, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
+                                                double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz, 
                                                 int strideY, int strideZ, int start, int finish, int Np){
 	
-	int n,nn,nn2x,ijk;
+	int nn,nn2x,ijk;
 	//int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
     double ux,uy,uz;//fluid velocity 
     double p;//pressure
@@ -834,20 +965,22 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
     double mgx,mgy,mgz;//mixed gradient reaching secondary neighbor
 
 	//double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
-    double h0,h1,h2,h3,h4,h5,h6;//distributions for LB phase field
+    //double h0,h1,h2,h3,h4,h5,h6;//distributions for LB phase field
 	double tau;//position dependent LB relaxation time for fluid
-    double C,theta;
+    //double C,theta;
-    double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
+    //double M = 2.0/9.0*(tauM-0.5);//diffusivity (or mobility) for the phase field D3Q7
 
 	for (int n=start; n<finish; n++){
 
 		rho0 = Den[n];//load density
-		// local relaxation time
-		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
 
 		// Get the 1D index based on regular data layout
 		ijk = Map[n];
         phi = Phi[ijk];// load phase field
+
+		// local relaxation time
+		tau=tauA + 0.5*(1.0-phi)*(tauB-tauA);
+
 		//					COMPUTE THE COLOR GRADIENT
 		//........................................................................
 		//.................Read Phase Indicator Values............................
@@ -1053,6 +1186,7 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
         ux = 3.0/rho0*(m1-m2+m7-m8+m9-m10+m11-m12+m13-m14+0.5*(chem*nx+Fx)/3.0);
         uy = 3.0/rho0*(m3-m4+m7-m8-m9+m10+m15-m16+m17-m18+0.5*(chem*ny+Fy)/3.0);
         uz = 3.0/rho0*(m5-m6+m11-m12-m13+m14+m15-m16-m17+m18+0.5*(chem*nz+Fz)/3.0);
+
         //compute pressure
         p = (m0+m2+m1+m4+m3+m6+m5+m8+m7+m10+m9+m12+m11+m14+m13+m16+m15+m18+m17)
                   +0.5*(rhoA-rhoB)/2.0/3.0*(ux*nx+uy*ny+uz*nz);
@@ -1330,62 +1464,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
         0.1111111111111111*(-4*chem + (rhoA - rhoB)*(ux*ux + 2*uy + uy*uy + (-2 + uz)*uz))));
         //----------------------------------------------------------------------------------------------------------------------------------------//
 
-
-        // ----------------------------- compute phase field evolution ----------------------------------------
-		//Normalize the Color Gradient
-		C = sqrt(nx*nx+ny*ny+nz*nz);
-		double ColorMag = C;
-		if (C==0.0) ColorMag=1.0;
-		nx = nx/ColorMag;
-		ny = ny/ColorMag;
-		nz = nz/ColorMag;		
-        //compute surface tension-related parameter
-        theta = M*4.5*(1-4.0*phi*phi)/W;
-
-        //load distributions of phase field
-		//q=0
-		h0 = hq[n];
-		//q=1
-		h1 = hq[2*Np+n]; 
-
-		//q=2
-		h2 = hq[1*Np+n];  
-
-		//q=3
-		h3 = hq[4*Np+n];
-
-		//q=4
-		h4 = hq[3*Np+n];
-
-		//q=5
-		h5 = hq[6*Np+n];
-
-		//q=6
-		h6 = hq[5*Np+n];
-
-        //-------------------------------- BGK collison for phase field ---------------------------------//
-		// q = 0
-		hq[n] = h0 - (h0 - 0.3333333333333333*phi)/tauM;
-
-		// q = 1
-		hq[1*Np+n] = h1 - (h1 - phi*(0.1111111111111111 + 0.5*ux) - (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 2
-		hq[2*Np+n] = h2 - (h2 - phi*(0.1111111111111111 - 0.5*ux) + (0.5*M*nx*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 3
-		hq[3*Np+n] = h3 - (h3 - phi*(0.1111111111111111 + 0.5*uy) - (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 4
-		hq[4*Np+n] = h4 - (h4 - phi*(0.1111111111111111 - 0.5*uy) + (0.5*M*ny*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 5
-		hq[5*Np+n] = h5 - (h5 - phi*(0.1111111111111111 + 0.5*uz) - (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM;
-
-		// q = 6
-		hq[6*Np+n] = h6 - (h6 - phi*(0.1111111111111111 - 0.5*uz) + (0.5*M*nz*(1 - 4*phi*phi))/W)/tauM;
-		//........................................................................
-
         //Update velocity on device
 		Vel[0*Np+n] = ux;
 		Vel[1*Np+n] = uy;
@@ -1405,7 +1483,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double
 extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,  
                                                                 double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
 	
-	int n;
 	int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
     double ux,uy,uz;//fluid velocity 
     double p;//pressure
@@ -1672,7 +1749,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborLis
 extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure, 
                                                                  double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np){
 	
-	int n;
     double ux,uy,uz;//fluid velocity 
     double p;//pressure
 	// distribution functions
@@ -1916,3 +1992,167 @@ extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, d
         Pressure[n] = p;
 	}
 }
+
+extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np){
+
+	int nn,nn2x,ijk;
+	double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
+	double m3,m5,m7;
+	double mm1,mm2,mm4,mm6,mm8,mm9,mm10,mm11,mm12,mm13,mm14,mm15,mm16,mm17,mm18;
+	double mm3,mm5,mm7;
+    //double nx,ny,nz;//normal color gradient
+    double mgx,mgy,mgz;//mixed gradient reaching secondary neighbor
+    double phi;
+
+	for (int n=start; n<finish; n++){
+
+		// Get the 1D index based on regular data layout
+		ijk = Map[n];
+        phi = Phi[ijk];// load phase field
+		//					COMPUTE THE COLOR GRADIENT
+		//........................................................................
+		//.................Read Phase Indicator Values............................
+		//........................................................................
+		nn = ijk-1;							// neighbor index (get convention)
+		m1 = Phi[nn];						// get neighbor for phi - 1
+		//........................................................................
+		nn = ijk+1;							// neighbor index (get convention)
+		m2 = Phi[nn];						// get neighbor for phi - 2
+		//........................................................................
+		nn = ijk-strideY;							// neighbor index (get convention)
+		m3 = Phi[nn];					// get neighbor for phi - 3
+		//........................................................................
+		nn = ijk+strideY;							// neighbor index (get convention)
+		m4 = Phi[nn];					// get neighbor for phi - 4
+		//........................................................................
+		nn = ijk-strideZ;						// neighbor index (get convention)
+		m5 = Phi[nn];					// get neighbor for phi - 5
+		//........................................................................
+		nn = ijk+strideZ;						// neighbor index (get convention)
+		m6 = Phi[nn];					// get neighbor for phi - 6
+		//........................................................................
+		nn = ijk-strideY-1;						// neighbor index (get convention)
+		m7 = Phi[nn];					// get neighbor for phi - 7
+		//........................................................................
+		nn = ijk+strideY+1;						// neighbor index (get convention)
+		m8 = Phi[nn];					// get neighbor for phi - 8
+		//........................................................................
+		nn = ijk+strideY-1;						// neighbor index (get convention)
+		m9 = Phi[nn];					// get neighbor for phi - 9
+		//........................................................................
+		nn = ijk-strideY+1;						// neighbor index (get convention)
+		m10 = Phi[nn];					// get neighbor for phi - 10
+		//........................................................................
+		nn = ijk-strideZ-1;						// neighbor index (get convention)
+		m11 = Phi[nn];					// get neighbor for phi - 11
+		//........................................................................
+		nn = ijk+strideZ+1;						// neighbor index (get convention)
+		m12 = Phi[nn];					// get neighbor for phi - 12
+		//........................................................................
+		nn = ijk+strideZ-1;						// neighbor index (get convention)
+		m13 = Phi[nn];					// get neighbor for phi - 13
+		//........................................................................
+		nn = ijk-strideZ+1;						// neighbor index (get convention)
+		m14 = Phi[nn];					// get neighbor for phi - 14
+		//........................................................................
+		nn = ijk-strideZ-strideY;					// neighbor index (get convention)
+		m15 = Phi[nn];					// get neighbor for phi - 15
+		//........................................................................
+		nn = ijk+strideZ+strideY;					// neighbor index (get convention)
+		m16 = Phi[nn];					// get neighbor for phi - 16
+		//........................................................................
+		nn = ijk+strideZ-strideY;					// neighbor index (get convention)
+		m17 = Phi[nn];					// get neighbor for phi - 17
+		//........................................................................
+		nn = ijk-strideZ+strideY;					// neighbor index (get convention)
+		m18 = Phi[nn];					// get neighbor for phi - 18
+
+        // compute mixed difference (Eq.30, A.Fukhari et al. JCP 315(2016) 434-457)
+		//........................................................................
+		nn2x = ijk-2;							// neighbor index (get convention)
+		mm1 = Phi[nn2x];						// get neighbor for phi - 1
+        mm1 = 0.25*(-mm1+5.0*m1-3.0*phi-m2);
+		//........................................................................
+		nn2x = ijk+2;							// neighbor index (get convention)
+		mm2 = Phi[nn2x];						// get neighbor for phi - 2
+        mm2 = 0.25*(-mm2+5.0*m2-3.0*phi-m1);
+		//........................................................................
+		nn2x = ijk-strideY*2;							// neighbor index (get convention)
+		mm3 = Phi[nn2x];					// get neighbor for phi - 3
+        mm3 = 0.25*(-mm3+5.0*m3-3.0*phi-m4);
+		//........................................................................
+		nn2x = ijk+strideY*2;							// neighbor index (get convention)
+		mm4 = Phi[nn2x];					// get neighbor for phi - 4
+        mm4 = 0.25*(-mm4+5.0*m4-3.0*phi-m3);
+		//........................................................................
+		nn2x = ijk-strideZ*2;						// neighbor index (get convention)
+		mm5 = Phi[nn2x];					// get neighbor for phi - 5
+        mm5 = 0.25*(-mm5+5.0*m5-3.0*phi-m6);
+		//........................................................................
+		nn2x = ijk+strideZ*2;						// neighbor index (get convention)
+		mm6 = Phi[nn2x];					// get neighbor for phi - 6
+        mm6 = 0.25*(-mm6+5.0*m6-3.0*phi-m5);
+		//........................................................................
+		nn2x = ijk-strideY*2-2;						// neighbor index (get convention)
+		mm7 = Phi[nn2x];					// get neighbor for phi - 7
+        mm7 = 0.25*(-mm7+5.0*m7-3.0*phi-m8);
+		//........................................................................
+		nn2x = ijk+strideY*2+2;						// neighbor index (get convention)
+		mm8 = Phi[nn2x];					// get neighbor for phi - 8
+        mm8 = 0.25*(-mm8+5.0*m8-3.0*phi-m7);
+		//........................................................................
+		nn2x = ijk+strideY*2-2;						// neighbor index (get convention)
+		mm9 = Phi[nn2x];					// get neighbor for phi - 9
+        mm9 = 0.25*(-mm9+5.0*m9-3.0*phi-m10);
+		//........................................................................
+		nn2x = ijk-strideY*2+2;						// neighbor index (get convention)
+		mm10 = Phi[nn2x];					// get neighbor for phi - 10
+        mm10 = 0.25*(-mm10+5.0*m10-3.0*phi-m9);
+		//........................................................................
+		nn2x = ijk-strideZ*2-2;						// neighbor index (get convention)
+		mm11 = Phi[nn2x];					// get neighbor for phi - 11
+        mm11 = 0.25*(-mm11+5.0*m11-3.0*phi-m12);
+		//........................................................................
+		nn2x = ijk+strideZ*2+2;						// neighbor index (get convention)
+		mm12 = Phi[nn2x];					// get neighbor for phi - 12
+        mm12 = 0.25*(-mm12+5.0*m12-3.0*phi-m11);
+		//........................................................................
+		nn2x = ijk+strideZ*2-2;						// neighbor index (get convention)
+		mm13 = Phi[nn2x];					// get neighbor for phi - 13
+        mm13 = 0.25*(-mm13+5.0*m13-3.0*phi-m14);
+		//........................................................................
+		nn2x = ijk-strideZ*2+2;						// neighbor index (get convention)
+		mm14 = Phi[nn2x];					// get neighbor for phi - 14
+        mm14 = 0.25*(-mm14+5.0*m14-3.0*phi-m13);
+		//........................................................................
+		nn2x = ijk-strideZ*2-strideY*2;					// neighbor index (get convention)
+		mm15 = Phi[nn2x];					// get neighbor for phi - 15
+        mm15 = 0.25*(-mm15+5.0*m15-3.0*phi-m16);
+		//........................................................................
+		nn2x = ijk+strideZ*2+strideY*2;					// neighbor index (get convention)
+		mm16 = Phi[nn2x];					// get neighbor for phi - 16
+        mm16 = 0.25*(-mm16+5.0*m16-3.0*phi-m15);
+		//........................................................................
+		nn2x = ijk+strideZ*2-strideY*2;					// neighbor index (get convention)
+		mm17 = Phi[nn2x];					// get neighbor for phi - 17
+        mm17 = 0.25*(-mm17+5.0*m17-3.0*phi-m18);
+		//........................................................................
+		nn2x = ijk-strideZ*2+strideY*2;					// neighbor index (get convention)
+		mm18 = Phi[nn2x];					// get neighbor for phi - 18
+        mm18 = 0.25*(-mm18+5.0*m18-3.0*phi-m17);
+
+
+		//............Compute the Color Gradient...................................
+		//nx = -3.0*1.0/18.0*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14));
+		//ny = -3.0*1.0/18.0*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18));
+		//nz = -3.0*1.0/18.0*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18));
+		//............Compute the Mixed Gradient...................................
+		mgx = -3.0*1.0/18.0*(mm1-mm2+0.5*(mm7-mm8+mm9-mm10+mm11-mm12+mm13-mm14));
+		mgy = -3.0*1.0/18.0*(mm3-mm4+0.5*(mm7-mm8-mm9+mm10+mm15-mm16+mm17-mm18));
+		mgz = -3.0*1.0/18.0*(mm5-mm6+0.5*(mm11-mm12-mm13+mm14+mm15-mm16-mm17+mm18));
+
+		ColorGrad[0*Np+n] = mgx;
+		ColorGrad[1*Np+n] = mgy;
+		ColorGrad[2*Np+n] = mgz;
+    }
+}

ctest_script.cmake

@@ -164,7 +164,7 @@ SET( CTEST_OPTIONS )
 SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
 SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" )
 SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" )
-SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
+SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
 IF ( USE_CUDA )
     SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" )
 ELSE()

models/ColorModel.cpp

@@ -531,6 +531,121 @@ void ScaLBL_ColorModel::Initialize(){
 	ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double));
 }
 
+double ScaLBL_ColorModel::Run(int returntime){
+	int nprocs=nprocx*nprocy*nprocz;
+
+	//************ MAIN ITERATION LOOP ***************************************/
+	comm.barrier();
+	PROFILE_START("Loop");
+	//std::shared_ptr<Database> analysis_db;
+	bool Regular = false;
+	auto current_db = db->cloneDatabase();
+	auto t1 = std::chrono::system_clock::now();
+	int START_TIMESTEP = timestep;
+	int EXIT_TIMESTEP = min(timestepMax,returntime);
+	while (timestep < EXIT_TIMESTEP ) {
+		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
+		PROFILE_START("Update");
+		// *************ODD TIMESTEP*************
+		timestep++;
+		// Compute the Phase indicator field
+		// Read for Aq, Bq happens in this routine (requires communication)
+		ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
+		ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
+		ScaLBL_Comm->Barrier();
+		ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
+
+		// Perform the collision operation
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
+		if (BoundaryCondition > 0 && BoundaryCondition < 5){
+			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
+			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
+		}
+		// Halo exchange for phase field
+		ScaLBL_Comm_Regular->SendHalo(Phi);
+
+		ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
+				alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm_Regular->RecvHalo(Phi);
+		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		ScaLBL_Comm->Barrier();
+		// Set BCs
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
+		ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
+				alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
+		ScaLBL_Comm->Barrier(); 
+
+		// *************EVEN TIMESTEP*************
+		timestep++;
+		// Compute the Phase indicator field
+		ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
+		ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
+		ScaLBL_Comm->Barrier();
+		ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
+
+		// Perform the collision operation
+		ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
+		// Halo exchange for phase field
+		if (BoundaryCondition > 0 && BoundaryCondition < 5){
+			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
+			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
+		}
+		ScaLBL_Comm_Regular->SendHalo(Phi);
+		ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
+				alpha, beta, Fx, Fy, Fz,  Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm_Regular->RecvHalo(Phi);
+		ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
+		ScaLBL_Comm->Barrier();
+		// Set boundary conditions
+		if (BoundaryCondition == 3){
+			ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 4){
+			din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
+			ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
+		}
+		else if (BoundaryCondition == 5){
+			ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
+			ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
+		}
+		ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
+				alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
+		ScaLBL_Comm->Barrier(); 
+		//************************************************************************
+	}
+	PROFILE_STOP("Update");
+
+	PROFILE_STOP("Loop");
+	PROFILE_SAVE("lbpm_color_simulator",1);
+	//************************************************************************
+	// Compute the walltime per timestep
+	auto t2 = std::chrono::system_clock::now();
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / (timestep - START_TIMESTEP);
+	// Performance obtained from each node
+	double MLUPS = double(Np)/cputime/1000000;
+
+	if (rank==0) printf("********************************************************\n");
+	if (rank==0) printf("CPU time = %f \n", cputime);
+	if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
+	return(MLUPS);
+	MLUPS *= nprocs;
+
+}
+
 void ScaLBL_ColorModel::Run(){
 	int nprocs=nprocx*nprocy*nprocz;
 	const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
@@ -580,7 +695,6 @@ void ScaLBL_ColorModel::Run(){
 	if (color_db->keyExists( "krA_morph_factor" )){
 		KRA_MORPH_FACTOR  = color_db->getScalar<double>( "krA_morph_factor" );
 	}
-	
 	/* defaults for simulation protocols */
 	auto protocol = color_db->getWithDefault<std::string>( "protocol", "none" );
 	if (protocol == "image sequence"){
@@ -625,7 +739,7 @@ void ScaLBL_ColorModel::Run(){
 	if (analysis_db->keyExists( "seed_water" )){
 		seed_water = analysis_db->getScalar<double>( "seed_water" );
 		if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water);
-		USE_SEED = true;
+		ASSERT(protocol == "seed water");
 	}
 	if (analysis_db->keyExists( "morph_delta" )){
 		morph_delta = analysis_db->getScalar<double>( "morph_delta" );
@@ -656,7 +770,6 @@ void ScaLBL_ColorModel::Run(){
 		MAX_MORPH_TIMESTEPS = analysis_db->getScalar<int>( "max_morph_timesteps" );
 	}
 
-
 	if (rank==0){
 		printf("********************************************************\n");
 		if (protocol == "image sequence"){
@@ -1320,7 +1433,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 	double vF = 0.f;
 	double vS = 0.f;
 	double delta_volume;
-	double WallFactor = 0.0;
+	double WallFactor = 1.0;
 	bool USE_CONNECTED_NWP = false;
 
 	DoubleArray phase(Nx,Ny,Nz);
@@ -1343,6 +1456,11 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
 		}
 	}
 	double volume_initial = Dm->Comm.sumReduce(  count);
+	double PoreVolume = Dm->Volume*Dm->Porosity();
+	/*ensure target isn't an absurdly small fraction of pore volume */
+	if (volume_initial < target_delta_volume*PoreVolume){
+		volume_initial = target_delta_volume*PoreVolume;
+	}
 	/*
 	sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank);
 	FILE *INPUT = fopen(LocalRankFilename,"wb");

models/ColorModel.h

@@ -16,6 +16,10 @@ Implementation of color lattice boltzmann model
 #include "ProfilerApp.h"
 #include "threadpool/thread_pool.h"
 
+
+#ifndef ScaLBL_ColorModel_INC
+#define ScaLBL_ColorModel_INC
+
 class ScaLBL_ColorModel{
 public:
 	ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM);
@@ -29,6 +33,7 @@ public:
 	void Create();
 	void Initialize();
 	void Run();
+	double Run(int returntime);
 	void WriteDebug();
 	void getPhaseField(DoubleArray &f);
 	
@@ -99,4 +104,5 @@ private:
 	int timestep;
 	int timestep_previous;
 };
+#endif
 

models/FreeLeeModel.cpp

@@ -10,9 +10,9 @@ color lattice boltzmann model
 #include <time.h>
 
 ScaLBL_FreeLeeModel::ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM):
-rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),tauM(0),rhoA(0),rhoB(0),W(0),gamma(0),kappa(0),beta(0),
+rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(2),tauA(1.0),tauB(1.0),tauM(1.0),rhoA(1.0),rhoB(1.0),W(5.0),gamma(0.001),kappa(0.0075),beta(0.0024),
 Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
-tau(0),rho0(0),
+tau(1.0),rho0(1.0),
 Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
 {
 	
@@ -20,6 +20,45 @@ Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),
 ScaLBL_FreeLeeModel::~ScaLBL_FreeLeeModel(){
 
 }
+
+
+void ScaLBL_FreeLeeModel::getPhase(DoubleArray &PhaseValues){
+	
+	DoubleArray PhaseWideHalo(Nxh,Nyh,Nzh);
+	ScaLBL_CopyToHost(PhaseWideHalo.data(), Phi, sizeof(double)*Nh);
+	
+	// use halo width = 1 for analysis data
+	for (int k=1; k<Nzh-1; k++){
+		for (int j=1; j<Nyh-1; j++){
+			for (int i=1; i<Nxh-1; i++){
+				PhaseValues(i-1,j-1,k-1) = PhaseWideHalo(i,j,k);
+			}
+		}
+	}
+}
+
+void ScaLBL_FreeLeeModel::getPotential(DoubleArray &PressureValues, DoubleArray &MuValues){
+	
+	ScaLBL_Comm->RegularLayout(Map,Pressure,PressureValues);
+    ScaLBL_Comm->Barrier(); comm.barrier();
+
+	ScaLBL_Comm->RegularLayout(Map,mu_phi,MuValues);
+    ScaLBL_Comm->Barrier(); comm.barrier();
+
+}
+
+void ScaLBL_FreeLeeModel::getVelocity(DoubleArray &Vel_x, DoubleArray &Vel_y, DoubleArray &Vel_z){
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[0],Vel_x);
+    ScaLBL_Comm->Barrier(); comm.barrier();
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[Np],Vel_y);
+    ScaLBL_Comm->Barrier(); comm.barrier();
+
+	ScaLBL_Comm->RegularLayout(Map,&Velocity[2*Np],Vel_z);
+    ScaLBL_Comm->Barrier(); comm.barrier();
+}
+
 void ScaLBL_FreeLeeModel::ReadParams(string filename){
 	// read the input database 
 	db = std::make_shared<Database>( filename );
@@ -737,21 +776,17 @@ void ScaLBL_FreeLeeModel::Initialize_SingleFluid(){
 	}
 }
 
-void ScaLBL_FreeLeeModel::Run_TwoFluid(){
+double ScaLBL_FreeLeeModel::Run_TwoFluid(int returntime){
 	int nprocs=nprocx*nprocy*nprocz;
-	const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
 	
-	if (rank==0){
+	int START_TIME = timestep;
-		printf("********************************************************\n");
+	int EXIT_TIME = min(returntime, timestepMax);
-		printf("No. of timesteps: %i \n", timestepMax);
-		fflush(stdout);
-	}
-
 	//************ MAIN ITERATION LOOP ***************************************/
 	comm.barrier();
     auto t1 = std::chrono::system_clock::now();
 	PROFILE_START("Loop");
-	while (timestep < timestepMax ) {
+	
+	while (timestep < EXIT_TIME ) {
 		//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
 		PROFILE_START("Update");
 		// *************ODD TIMESTEP*************
@@ -760,24 +795,27 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 		// Compute the Phase indicator field
 		// Read for hq happens in this routine (requires communication)
 		ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMAL
-		ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(NeighborList, dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(NeighborList, dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
 		ScaLBL_Comm->RecvD3Q7AA(hq,0); //WRITE INTO OPPOSITE
 		ScaLBL_Comm->Barrier();
-		ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(NeighborList, dvcMap, hq, Den, Phi, rhoA, rhoB,  0, ScaLBL_Comm->LastExterior(), Np);
+		ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(NeighborList, dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, 0, ScaLBL_Comm->LastExterior(), Np);
 
 		// Perform the collision operation
-		ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL
+		// Halo exchange for phase field
+		ScaLBL_D3Q7_ComputePhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, 0, ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm_WideHalo->Send(Phi);
+		ScaLBL_Comm_WideHalo->Recv(Phi);
 		if (BoundaryCondition > 0 && BoundaryCondition < 5){
             //TODO to be revised
+			// Need to add BC for hq!!!
 			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
 			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
 		}
-		// Halo exchange for phase field
+		
-		ScaLBL_Comm_WideHalo->Send(Phi);
+		ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FROM NORMAL		
+		ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB,
+				                        kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
 
-		ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM,
-				                        kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
-		ScaLBL_Comm_WideHalo->Recv(Phi);
 		ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE
 		ScaLBL_Comm->Barrier();
 		// Set BCs
@@ -793,30 +831,34 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 			ScaLBL_Comm->D3Q19_Reflection_BC_z(gqbar);
 			ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar);
 		}
-		ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM,
-				                        kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
-		ScaLBL_Comm->Barrier(); 
 
+		ScaLBL_D3Q19_AAodd_FreeLeeModel(NeighborList, dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, 
+				                        kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np);
+		ScaLBL_Comm->Barrier(); 
+		
+				
 		// *************EVEN TIMESTEP*************
 		timestep++;
 		// Compute the Phase indicator field
-		ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMAL
+		ScaLBL_Comm->SendD3Q7AA(hq,0); //READ FROM NORMA
-		ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB,  ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_D3Q7_AAeven_FreeLee_PhaseField(dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
 		ScaLBL_Comm->RecvD3Q7AA(hq,0); //WRITE INTO OPPOSITE
 		ScaLBL_Comm->Barrier();
-		ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB,  0, ScaLBL_Comm->LastExterior(), Np);
+		ScaLBL_D3Q7_AAeven_FreeLee_PhaseField(dvcMap, hq, Den, Phi, ColorGrad, Velocity, rhoA, rhoB, tauM, W, 0, ScaLBL_Comm->LastExterior(), Np);
 
 		// Perform the collision operation
-		ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FORM NORMAL
 		// Halo exchange for phase field
+		ScaLBL_D3Q7_ComputePhaseField(dvcMap, hq, Den, Phi, rhoA, rhoB, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_Comm_WideHalo->Send(Phi);
+		ScaLBL_Comm_WideHalo->Recv(Phi);
 		if (BoundaryCondition > 0 && BoundaryCondition < 5){
 			ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
 			ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
 		}
-		ScaLBL_Comm_WideHalo->Send(Phi);
+		ScaLBL_Comm->SendD3Q19AA(gqbar); //READ FORM NORMAL
-		ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM,
+
-				                        kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+		ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, 
-		ScaLBL_Comm_WideHalo->Recv(Phi);
+				                        kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
 		ScaLBL_Comm->RecvD3Q19AA(gqbar); //WRITE INTO OPPOSITE
 		ScaLBL_Comm->Barrier();
 		// Set boundary conditions
@@ -832,8 +874,8 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 			ScaLBL_Comm->D3Q19_Reflection_BC_z(gqbar);
 			ScaLBL_Comm->D3Q19_Reflection_BC_Z(gqbar);
 		}
-		ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, hq, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB, tauM,
+		ScaLBL_D3Q19_AAeven_FreeLeeModel(dvcMap, gqbar, Den, Phi, mu_phi, Velocity, Pressure, ColorGrad, rhoA, rhoB, tauA, tauB,
-				                        kappa, beta, W, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
+				                        kappa, beta, W, Fx, Fy, Fz, Nxh, Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np);
 		ScaLBL_Comm->Barrier();
 		//************************************************************************
 		PROFILE_STOP("Update");
@@ -844,18 +886,11 @@ void ScaLBL_FreeLeeModel::Run_TwoFluid(){
 	if (rank==0) printf("-------------------------------------------------------------------\n");
 	// Compute the walltime per timestep
     auto t2 = std::chrono::system_clock::now();
-	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
+	double cputime = std::chrono::duration<double>( t2 - t1 ).count() / (EXIT_TIME-START_TIME);
 	// Performance obtained from each node
 	double MLUPS = double(Np)/cputime/1000000;
 
-	if (rank==0) printf("********************************************************\n");
+	return MLUPS;
-	if (rank==0) printf("CPU time = %f \n", cputime);
-	if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
-	MLUPS *= nprocs;
-	if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
-	if (rank==0) printf("********************************************************\n");
-
-	// ************************************************************************
 }
 
 void ScaLBL_FreeLeeModel::Run_SingleFluid(){
@@ -906,6 +941,7 @@ void ScaLBL_FreeLeeModel::Run_SingleFluid(){
                                                         0, ScaLBL_Comm->LastExterior(), Np);
 		ScaLBL_Comm->Barrier(); 
 
+
 		// *************EVEN TIMESTEP*************
 		timestep++;
         //-------------------------------------------------------------------------------------------------------------------
@@ -960,6 +996,32 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){
 	DoubleArray PhaseData(Nxh,Nyh,Nzh);
 	//ScaLBL_Comm->RegularLayout(Map,Phi,PhaseField);
 	ScaLBL_CopyToHost(PhaseData.data(), Phi, sizeof(double)*Nh);
+	/*
+	IntArray MapData(Np);
+	ScaLBL_CopyToHost(MapData.data(), dvcMap, sizeof(int)*Np);
+	FILE *MAP;
+	sprintf(LocalRankFilename,"Map.%05i.raw",rank);
+	MAP = fopen(LocalRankFilename,"wb");
+	fwrite(MapData.data(),4,Np,MAP);
+	fclose(MAP);
+	
+	FILE *NB;
+	//IntArray Neighbors(18,Np);
+	//ScaLBL_CopyToHost(Neighbors.data(), NeighborList, sizeof(int)*Np*18);
+	sprintf(LocalRankFilename,"neighbors.%05i.raw",rank);
+	NB = fopen(LocalRankFilename,"wb");
+	fwrite(NeighborList,4,18*Np,NB);
+	fclose(NB);
+
+	FILE *DIST;
+	DoubleArray DistData(7, Np);
+	ScaLBL_CopyToHost(DistData.data(), hq, 7*sizeof(double)*Np);
+	sprintf(LocalRankFilename,"h.%05i.raw",rank);
+	DIST = fopen(LocalRankFilename,"wb");
+	fwrite(DistData.data(),8,7*Np,DIST);
+	fclose(DIST);
+	
+	*/
 
 	FILE *OUTFILE;
 	sprintf(LocalRankFilename,"Phase.%05i.raw",rank);
@@ -968,6 +1030,17 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){
 	fclose(OUTFILE);
 
 	DoubleArray PhaseField(Nx,Ny,Nz);
+	FILE *DIST;
+	for (int q=0; q<7; q++){
+		ScaLBL_Comm->RegularLayout(Map,&hq[q*Np],PhaseField);
+
+		sprintf(LocalRankFilename,"h%i.%05i.raw",q,rank);
+		DIST = fopen(LocalRankFilename,"wb");
+		fwrite(PhaseField.data(),8,Nx*Ny*Nz,DIST);
+		fclose(DIST);
+
+	}
+
     ScaLBL_Comm->RegularLayout(Map,Den,PhaseField);
 	FILE *AFILE;
 	sprintf(LocalRankFilename,"Density.%05i.raw",rank);
@@ -1003,7 +1076,7 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){
 	fwrite(PhaseField.data(),8,N,VELZ_FILE);
 	fclose(VELZ_FILE);
 
-/*	ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField);
+	ScaLBL_Comm->RegularLayout(Map,&ColorGrad[0],PhaseField);
 	FILE *CGX_FILE;
 	sprintf(LocalRankFilename,"Gradient_X.%05i.raw",rank);
 	CGX_FILE = fopen(LocalRankFilename,"wb");
@@ -1023,7 +1096,7 @@ void ScaLBL_FreeLeeModel::WriteDebug_TwoFluid(){
 	CGZ_FILE = fopen(LocalRankFilename,"wb");
 	fwrite(PhaseField.data(),8,N,CGZ_FILE);
 	fclose(CGZ_FILE);
-*/
+
 }
 
 void ScaLBL_FreeLeeModel::WriteDebug_SingleFluid(){
@@ -1059,3 +1132,151 @@ void ScaLBL_FreeLeeModel::WriteDebug_SingleFluid(){
 	fwrite(PhaseField.data(),8,N,VELZ_FILE);
 	fclose(VELZ_FILE);
 }
+
+void ScaLBL_FreeLeeModel::Create_DummyPhase_MGTest(){
+	// Initialize communication structures in averaging domain
+	for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = Mask->id[i];
+	Mask->CommInit();
+	Np=Mask->PoreCount();
+	//...........................................................................
+	if (rank==0)    printf ("Create ScaLBL_Communicator \n");
+	// Create a communicator for the device (will use optimized layout)
+	// ScaLBL_Communicator ScaLBL_Comm(Mask); // original
+	ScaLBL_Comm  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+	//ScaLBL_Comm_Regular  = std::shared_ptr<ScaLBL_Communicator>(new ScaLBL_Communicator(Mask));
+	ScaLBL_Comm_WideHalo  = std::shared_ptr<ScaLBLWideHalo_Communicator>(new ScaLBLWideHalo_Communicator(Mask,2));
+
+	// create the layout for the LBM
+	int Npad=(Np/16 + 2)*16;
+	if (rank==0)    printf ("Set up memory efficient layout, %i | %i | %i \n", Np, Npad, N);
+	Map.resize(Nx,Ny,Nz);       Map.fill(-2);
+	auto neighborList= new int[18*Npad];
+	Np = ScaLBL_Comm->MemoryOptimizedLayoutAA(Map,neighborList,Mask->id.data(),Np,1);
+	comm.barrier();
+
+	//...........................................................................
+	//                MAIN  VARIABLES ALLOCATED HERE
+	//...........................................................................
+	// LBM variables
+	if (rank==0)    printf ("Allocating distributions \n");
+	//......................device distributions.................................
+	dist_mem_size = Np*sizeof(double);
+	neighborSize=18*(Np*sizeof(int));
+	//...........................................................................
+	//ScaLBL_AllocateDeviceMemory((void **) &NeighborList, neighborSize);
+	ScaLBL_AllocateDeviceMemory((void **) &dvcMap, sizeof(int)*Np);
+	//ScaLBL_AllocateDeviceMemory((void **) &gqbar, 19*dist_mem_size);
+	//ScaLBL_AllocateDeviceMemory((void **) &hq, 7*dist_mem_size);
+	//ScaLBL_AllocateDeviceMemory((void **) &mu_phi, dist_mem_size);
+	//ScaLBL_AllocateDeviceMemory((void **) &Den, dist_mem_size);
+	ScaLBL_AllocateDeviceMemory((void **) &Phi, sizeof(double)*Nh);		
+	//ScaLBL_AllocateDeviceMemory((void **) &Pressure, sizeof(double)*Np);
+	//ScaLBL_AllocateDeviceMemory((void **) &Velocity, 3*sizeof(double)*Np);
+	ScaLBL_AllocateDeviceMemory((void **) &ColorGrad, 3*sizeof(double)*Np);
+	//...........................................................................
+	// Update GPU data structures
+	if (rank==0)	printf ("Setting up device map and neighbor list \n");
+	fflush(stdout);
+	int *TmpMap;
+	TmpMap=new int[Np];
+	for (int k=1; k<Nz-1; k++){
+		for (int j=1; j<Ny-1; j++){
+			for (int i=1; i<Nx-1; i++){
+				int idx=Map(i,j,k);
+				if (!(idx < 0))
+					TmpMap[idx] = ScaLBL_Comm_WideHalo->Map(i,j,k);
+			}
+		}
+	}
+	// check that TmpMap is valid
+	for (int idx=0; idx<ScaLBL_Comm->LastExterior(); idx++){
+		auto n = TmpMap[idx];
+		if (n > Nxh*Nyh*Nzh){
+			printf("Bad value! idx=%i \n", n);
+			TmpMap[idx] = Nxh*Nyh*Nzh-1;
+		}
+	}
+	for (int idx=ScaLBL_Comm->FirstInterior(); idx<ScaLBL_Comm->LastInterior(); idx++){
+		auto n = TmpMap[idx];
+		if ( n > Nxh*Nyh*Nzh ){
+			printf("Bad value! idx=%i \n",n);
+			TmpMap[idx] = Nxh*Nyh*Nzh-1;
+		}
+	}
+    // copy the device map
+	ScaLBL_CopyToDevice(dvcMap, TmpMap, sizeof(int)*Np);
+	// copy the neighbor list 
+	//ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
+	comm.barrier();
+
+	double *phase;
+	phase = new double[Nh];
+
+	for (int k=0;k<Nzh;k++){
+		for (int j=0;j<Nyh;j++){
+			for (int i=0;i<Nxh;i++){
+
+                //idx for double-halo array 'phase'
+				int nh = k*Nxh*Nyh+j*Nxh+i;
+
+                //idx for single-halo array Mask->id[n]
+                int x=i-1;
+                int y=j-1;
+                int z=k-1;
+				if (x<0)   x=0;
+				if (y<0)   y=0;
+				if (z<0)   z=0;
+				if (x>=Nx) x=Nx-1;
+				if (y>=Ny) y=Ny-1;
+				if (z>=Nz) z=Nz-1;
+				int n = z*Nx*Ny+y*Nx+x;
+				phase[nh]=id[n];
+			}
+		}
+	}
+	ScaLBL_CopyToDevice(Phi, phase, Nh*sizeof(double));
+	ScaLBL_Comm->Barrier();
+	comm.barrier();
+	delete [] TmpMap;
+	delete [] neighborList;
+    delete [] phase;
+}
+
+void ScaLBL_FreeLeeModel::MGTest(){
+
+	comm.barrier();
+
+	ScaLBL_Comm_WideHalo->Send(Phi);
+    ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nxh,Nxh*Nyh, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
+	ScaLBL_Comm_WideHalo->Recv(Phi);
+    ScaLBL_D3Q9_MGTest(dvcMap,Phi,ColorGrad,Nxh,Nxh*Nyh, 0, ScaLBL_Comm->LastExterior(), Np);
+
+    //check the sum of ColorGrad
+    double cgx_loc = 0.0;
+    double cgy_loc = 0.0;
+    double cgz_loc = 0.0;
+    double cgx,cgy,cgz;
+    double *ColorGrad_host;
+	ColorGrad_host = new double [3*Np];
+	ScaLBL_CopyToHost(&ColorGrad_host[0],&ColorGrad[0], 3*Np*sizeof(double));
+    for (int i = ScaLBL_Comm->FirstInterior(); i<ScaLBL_Comm->LastInterior();i++){
+        cgx_loc+=ColorGrad_host[0*Np+i];
+        cgy_loc+=ColorGrad_host[1*Np+i];
+        cgz_loc+=ColorGrad_host[2*Np+i];
+    }
+    for (int i = 0; i<ScaLBL_Comm->LastExterior();i++){
+        cgx_loc+=ColorGrad_host[0*Np+i];
+        cgy_loc+=ColorGrad_host[1*Np+i];
+        cgz_loc+=ColorGrad_host[2*Np+i];
+    }
+    cgx=Dm->Comm.sumReduce( cgx_loc);
+    cgy=Dm->Comm.sumReduce( cgy_loc);
+    cgz=Dm->Comm.sumReduce( cgz_loc);
+    if (rank==0){
+        printf("Sum of all x-component of the mixed gradient = %.2g \n",cgx);
+        printf("Sum of all y-component of the mixed gradient = %.2g \n",cgy);
+        printf("Sum of all z-component of the mixed gradient = %.2g \n",cgz);
+    }
+
+    delete [] ColorGrad_host;
+}

models/FreeLeeModel.h

@@ -16,6 +16,9 @@ Implementation of Lee et al JCP 2016 lattice boltzmann model
 #include "common/ScaLBL.h"
 #include "common/WideHalo.h"
 
+#ifndef ScaLBL_FreeLeeModel_INC
+#define ScaLBL_FreeLeeModel_INC
+
 class ScaLBL_FreeLeeModel{
 public:
   ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM);
@@ -28,12 +31,17 @@ public:
 	void ReadInput();
 	void Create_TwoFluid();
 	void Initialize_TwoFluid();
-	void Run_TwoFluid();
+	double Run_TwoFluid(int returntime);
+
 	void WriteDebug_TwoFluid();
 	void Create_SingleFluid();
 	void Initialize_SingleFluid();
 	void Run_SingleFluid();
+	
 	void WriteDebug_SingleFluid();
+    // test utilities
+    void Create_DummyPhase_MGTest();
+    void MGTest();
 	
 	bool Restart,pBC;
 	int timestep,timestepMax;
@@ -73,8 +81,12 @@ public:
 	double *Velocity;
 	double *Pressure;
 	
+	void getPhase(DoubleArray &PhaseValues);
+	void getPotential(DoubleArray &PressureValues, DoubleArray &MuValues);
+	void getVelocity(DoubleArray &Vx, DoubleArray &Vy, DoubleArray &Vz);
+	
 	DoubleArray SignDist;
-		
+	
 private:
 	Utilities::MPI comm;
     
@@ -90,4 +102,4 @@ private:
 	void AssignComponentLabels_ChemPotential_ColorGrad();
 
 };
-
+#endif

sample_scripts/config_build_eos

@@ -33,7 +33,6 @@ cmake                                        \
     -D CMAKE_CXX_STANDARD=14					                \
     -D USE_TIMER=false			     \
     -D TIMER_DIRECTORY=${HOME}/timerutility/build/opt  \
-    -D MPI_COMPILER:BOOL=TRUE                \
        -D MPIEXEC=aprun                      \
        -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D USE_CUDA=0                            \

sample_scripts/config_build_rhea

@@ -27,7 +27,6 @@ cmake                                        \
     -D CMAKE_CXX_STD=11 			     \
     -D USE_TIMER=false			     \
     -D TIMER_DIRECTORY=${HOME}/timerutility/build/opt  \
-    -D MPI_COMPILER:BOOL=TRUE                \
        -D MPIEXEC=aprun                      \
        -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D USE_CUDA=0                            \

sample_scripts/config_build_titan

@@ -25,7 +25,6 @@ cmake                                       \
         -D CMAKE_CUDA_FLAGS="-arch sm_35"   \
         -D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
     -D USE_MPI=1                            \
-        -D MPI_COMPILER:BOOL=TRUE           \
         -D MPIEXEC=aprun                    \
         -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D USE_SILO=1			                \

sample_scripts/config_build_titan_silo

@@ -20,7 +20,6 @@ cmake                                       \
         -D CMAKE_CUDA_FLAGS="-arch sm_35"   \
         -D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
     -D USE_MPI=1                            \
-        -D MPI_COMPILER:BOOL=TRUE           \
         -D MPIEXEC=aprun                    \
         -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D USE_NETCDF=0                         \

sample_scripts/config_poplar_hip

@@ -36,6 +36,5 @@ cmake                                       \
 
 # MPI_THREAD_MULTIPLE=1 MV2_USE_RDMA_CM=0 MV2_USE_RDMA_CM= MV2_NUM_HCAS=1 MV2_USE_CUDA=1 MV2_ENABLE_AFFINITY=0 srun -n 2 -N 1 --cpu-bind=v -c 1 ./test_MPI
 
-#        -D MPI_COMPILER:BOOL=TRUE           \
 #        -D MPIEXEC=mpirun                   \
 #        -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \

sample_scripts/config_summit_hip

@@ -21,7 +21,6 @@ cmake                                       \
         -D HIP_NVCC_OPTIONS="-arch sm_70"   \
         -D LINK_LIBRARIES="/sw/summit/cuda/9.2.148/lib64/libcudart.so"   \
     -D USE_MPI=1                            \
-        -D MPI_COMPILER:BOOL=TRUE           \
     -D USE_NETCDF=0                         \
     -D USE_SILO=1                           \
         -D SILO_DIRECTORY=${TPL_DIR}/silo   \

sample_scripts/config_titan

@@ -9,7 +9,6 @@ cmake                                    \
         -D CMAKE_CUDA_FLAGS="-arch sm_35" \
         -D CMAKE_CUDA_HOST_COMPILER="/usr/bin/gcc" \
     -D USE_MPI=1                            \
-        -D MPI_COMPILER:BOOL=TRUE           \
         -D MPIEXEC=aprun                   \
         -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     ${HOME}/LBPM-WIA

sample_scripts/configure_arc_cluster

@@ -10,7 +10,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=mpicxx        \
     -D CMAKE_C_FLAGS=""            \
     -D CMAKE_CXX_FLAGS=""          \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_arden

@@ -5,7 +5,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc          \
     -D CMAKE_C_FLAGS="-O3 -fPIC"         \
     -D CMAKE_CXX_FLAGS="-O3 -fPIC "      \
     -D CMAKE_CXX_STANDARD=14    \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_basic_cluster

@@ -8,7 +8,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=mpicxx        \
     -D CMAKE_C_FLAGS=""            \
     -D CMAKE_CXX_FLAGS=""          \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_blueridge

@@ -8,7 +8,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=mpicxx        \
     -D CMAKE_C_FLAGS=""            \
     -D CMAKE_CXX_FLAGS=""          \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_cascades_cpu

@@ -11,7 +11,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=mpicxx        \
     -D CMAKE_C_FLAGS="-fPIC"            \
     -D CMAKE_CXX_FLAGS="-fPIC"          \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_desktop

@@ -7,7 +7,6 @@ cmake                                    \
     -D CMAKE_C_FLAGS="-g "         \
     -D CMAKE_CXX_FLAGS="-g "      \
     -D CMAKE_CXX_STANDARD=14 	   \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_huckleberry

@@ -12,7 +12,6 @@ cmake                                    \
     -D CMAKE_CXX_COMPILER:PATH=mpicxx        \
     -D CMAKE_C_FLAGS="-fPIC"            \
     -D CMAKE_CXX_FLAGS="-fPIC"          \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/configure_summit

@@ -20,7 +20,6 @@ cmake                                       \
         -D CMAKE_CUDA_FLAGS="-arch sm_70 -Xptxas=-v -Xptxas -dlcm=cg -lineinfo" \
         -D CMAKE_CUDA_HOST_COMPILER="/sw/summit/gcc/6.4.0/bin/gcc" \
     -D USE_MPI=1                            \
-        -D MPI_COMPILER:BOOL=TRUE           \
         -D MPIEXEC=mpirun                   \
         -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D USE_HDF5=1                        \

sample_scripts/configure_titan_jem

@@ -15,7 +15,6 @@ cmake                                    \
     -D CMAKE_C_COMPILER:PATH=cc          \
     -D CMAKE_CXX_COMPILER:PATH=CC        \
     -D CMAKE_CXX_COMPILER:PATH=CC        \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=aprun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Debug     \

sample_scripts/configure_ubuntu

@@ -5,9 +5,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc          \
     -D CMAKE_C_FLAGS="-O3 -fPIC"         \
     -D CMAKE_CXX_FLAGS="-O3 -fPIC "      \
     -D CMAKE_CXX_STANDARD=14    \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
-    -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \
     -D CUDA_FLAGS="-arch sm_35"          \
     -D CUDA_HOST_COMPILER="/usr/bin/gcc" \
@@ -16,7 +14,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc          \
     -D USE_SILO=1 \
     -D SILO_LIB="/opt/arden/silo/4.10.2/lib/libsiloh5.a" \
     -D SILO_DIRECTORY="/opt/arden/silo/4.10.2" \
-    -D USE_NETCDF=1  \
+    -D USE_NETCDF=0  \
     -D NETCDF_DIRECTORY="/opt/arden/netcdf/4.6.1" \
     -D USE_CUDA=0                        \
     -D USE_TIMER=0 \

sample_scripts/daedalus_config

@@ -12,7 +12,6 @@ i    -D CMAKE_CXX_COMPILER:PATH=/home/christopher/openmpi/install_dir/bin/mpicxx
     -D USE_DOXYGEN=false \
  #   -D CMAKE_C_FLAGS="-std=gnu++11 -w"    \
  #   -D CMAKE_CXX_FLAGS="-std=gnu++11 -w"    \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=/home/christopher/openmpi/install_dir/bin/mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Release     \

sample_scripts/promethius_config

@@ -7,7 +7,6 @@ cmake                                    \
     -D CMAKE_C_FLAGS="-g "         \
     -D CMAKE_CXX_FLAGS="-g -Wno-deprecated-declarations"      \
     -D CXX_STD=11 	   \
-    -D MPI_COMPILER:BOOL=TRUE            \
     -D MPIEXEC=mpirun                     \
     -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
     -D CMAKE_BUILD_TYPE:STRING=Debug     \

tests/CMakeLists.txt

@@ -35,14 +35,15 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest )
 #ADD_LBPM_EXECUTABLE( BlobAnalysis )
 #ADD_LBPM_EXECUTABLE( BlobIdentify )
 #ADD_LBPM_EXECUTABLE( BlobIdentifyParallel )
-#ADD_LBPM_EXECUTABLE( convertIO )
+ADD_LBPM_EXECUTABLE( convertIO )
-#ADD_LBPM_EXECUTABLE( DataAggregator )
+ADD_LBPM_EXECUTABLE( DataAggregator )
 #ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )(
 ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar )
 ADD_LBPM_EXECUTABLE( TestPoissonSolver )
 ADD_LBPM_EXECUTABLE( TestIonModel )
 ADD_LBPM_EXECUTABLE( TestNernstPlanck )
 ADD_LBPM_EXECUTABLE( TestPNP_Stokes )
+ADD_LBPM_EXECUTABLE( TestMixedGrad )
 
 
 
@@ -61,6 +62,7 @@ ADD_LBPM_TEST( TestMap )
 ADD_LBPM_TEST( TestWideHalo )
 ADD_LBPM_TEST( TestColorGradDFH )
 ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db)
+ADD_LBPM_TEST( testGlobalMassFreeLee ../example/Bubble/input.db)
 #ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db)
 ADD_LBPM_TEST( TestPressVel  ../example/Bubble/input.db)
 ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db)

tests/TestMixedGrad.cpp

@@ -0,0 +1,199 @@
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#include "common/Utilities.h"
+#include "models/FreeLeeModel.h"
+
+inline void Initialize_Mask(ScaLBL_FreeLeeModel &LeeModel){
+	// initialize a bubble
+	int i,j,k,n;
+	int rank = LeeModel.Mask->rank();
+	int Nx = LeeModel.Mask->Nx;
+	int Ny = LeeModel.Mask->Ny;
+	int Nz = LeeModel.Mask->Nz;
+	if (rank == 0) printf(" initialize mask...\n");
+
+	for (k=0;k<Nz;k++){
+		for (j=0;j<Ny;j++){
+			for (i=0;i<Nx;i++){
+				n = k*Nx*Ny + j*Nz + i;
+                LeeModel.Mask->id[n]=1;
+                LeeModel.id[n] = LeeModel.Mask->id[n];
+			}
+		}
+	}
+}
+
+
+inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel, double ax, double ay, double az){
+	// initialize a bubble
+	int i,j,k,n;
+	int rank = LeeModel.Mask->rank();
+	int Nx = LeeModel.Mask->Nx;
+	int Ny = LeeModel.Mask->Ny;
+	int Nz = LeeModel.Mask->Nz;
+	if (rank == 0) printf("Setting up dummy phase field with gradient {x,y,z} = {%f , %f , %f}...\n",ax,ay,az);
+	
+	double * Dummy;
+	int Nh = (Nx+2)*(Ny+2)*(Nz+2);
+	Dummy = new double [(Nx+2)*(Ny+2)*(Nz+2)];
+	for (k=0;k<Nz;k++){
+		for (j=0;j<Ny;j++){
+			for (i=0;i<Nx;i++){
+				n = k*Nx*Ny + j*Nz + i;
+                LeeModel.Mask->id[n]=1;
+                LeeModel.id[n] = LeeModel.Mask->id[n];
+                int nh = (k+1)*(Nx+2)*(Ny+2) + (j+1)*(Nx+2) + i+1;
+                Dummy[nh] = ax*double(i) + ay*double(j) + az*double(k);
+			}
+		}
+	}
+	ScaLBL_CopyToDevice(LeeModel.Phi, Dummy, sizeof(double)*Nh);
+	
+	LeeModel.MGTest();
+}
+
+inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){
+	int i,j,k,iq,stride,nread;
+	int Nxh = LeeModel.Nxh;
+	int Nyh = LeeModel.Nyh;
+	int Np = LeeModel.Np;
+	
+	int *TmpMap;
+	TmpMap = new int[Np];
+	ScaLBL_CopyToHost(TmpMap, LeeModel.dvcMap, Np*sizeof(int));
+	
+	int *neighborList;
+	neighborList = new int[18*Np];
+	ScaLBL_CopyToHost(neighborList, LeeModel.NeighborList, 18*Np*sizeof(int));
+	printf("Check stride for interior neighbors \n");
+	int count = 0;
+	for (int n=LeeModel.ScaLBL_Comm->FirstInterior(); n<LeeModel.ScaLBL_Comm->LastInterior(); n++){
+		// q=0
+			int idx = TmpMap[n];
+			k = idx/Nxh/Nyh;
+			j = (idx-k*Nxh*Nyh)/Nxh;
+			i = (idx-k*Nxh*Nyh -j*Nxh);
+
+			// q=1
+			nread = neighborList[n]; 
+			iq = TmpMap[nread%Np];
+			stride = idx - iq;
+			if (stride != 1){
+				printf(" %i, %i, %i   q = 1 stride=%i \n ",i,j,k,stride);
+				count++;
+			}
+			
+			// q=2
+			nread = neighborList[n+Np]; 
+			iq = TmpMap[nread%Np];
+			stride = iq - idx;
+			if (stride != 1){
+				printf(" %i, %i, %i   q = 2 stride=%i \n ",i,j,k,stride);
+				count++;
+			}
+			
+
+			// q=3
+			nread = neighborList[n+2*Np]; 
+			iq = TmpMap[nread%Np];
+			stride = idx - iq;
+			if (stride != Nxh){
+				printf(" %i, %i, %i   q = 3 stride=%i \n ",i,j,k,stride);
+				count++;
+			}
+
+			// q = 4
+			nread = neighborList[n+3*Np]; 
+			iq = TmpMap[nread%Np];
+			stride = iq-idx;
+			if (stride != Nxh){
+				printf(" %i, %i, %i   q = 4 stride=%i \n ",i,j,k,stride);
+				count++;
+			}
+			
+
+			// q=5
+			nread = neighborList[n+4*Np];
+			iq = TmpMap[nread%Np];
+			stride = idx - iq;
+			if (stride != Nxh*Nyh){
+				count++;
+				printf(" %i, %i, %i   q = 5 stride=%i \n ",i,j,k,stride);
+			}
+			
+			// q = 6
+			nread = neighborList[n+5*Np];
+			iq = TmpMap[nread%Np];
+			stride = iq - idx;
+			if (stride != Nxh*Nyh){
+				count++;
+				printf(" %i, %i, %i   q = 6 stride=%i \n ",i,j,k,stride);
+			}
+
+	}
+	return count;
+}
+
+int main( int argc, char **argv )
+{
+
+    // Initialize
+    Utilities::startup( argc, argv );
+    int errors = 0;
+    // Load the input database
+    auto db = std::make_shared<Database>( argv[1] );
+
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        Utilities::MPI comm( MPI_COMM_WORLD );
+        int rank   = comm.getRank();
+        int nprocs = comm.getSize();
+
+        if ( rank == 0 ) {
+            printf( "********************************************************\n" );
+            printf( "Running Mixed Gradient Test	\n" );
+            printf( "********************************************************\n" );
+        }
+        // Initialize compute device
+        int device = ScaLBL_SetDevice( rank );
+        NULL_USE( device );
+        ScaLBL_DeviceBarrier();
+        comm.barrier();
+
+        PROFILE_ENABLE( 1 );
+        // PROFILE_ENABLE_TRACE();
+        // PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START( "Main" );
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_FreeLeeModel LeeModel( rank, nprocs, comm );
+        LeeModel.ReadParams( filename );
+        LeeModel.SetDomain();
+        Initialize_Mask(LeeModel);
+        //LeeModel.Create_DummyPhase_MGTest(); 
+        LeeModel.Create_TwoFluid(); 
+        
+        errors=MultiHaloNeighborCheck(LeeModel);
+        
+        Initialize_DummyPhaseField(LeeModel,1.0, 2.0, 3.0);
+        LeeModel.WriteDebug_TwoFluid();
+
+        PROFILE_STOP( "Main" );
+        PROFILE_SAVE( file, level );
+        // ****************************************************
+
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+    return errors;
+
+}

tests/TestWriter.cpp

@@ -1,219 +1,232 @@
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <stdexcept>
 #include <stdio.h>
 #include <stdlib.h>
-#include <iostream>
-#include <exception>
-#include <stdexcept>
-#include <fstream>
-#include <memory>
 
-#include "common/UnitTest.h"
-#include "common/Utilities.h"
-#include "common/MPI.h"
 #include "IO/MeshDatabase.h"
 #include "IO/Reader.h"
 #include "IO/Writer.h"
 #include "ProfilerApp.h"
+#include "common/MPI.h"
+#include "common/UnitTest.h"
+#include "common/Utilities.h"
 
-inline bool approx_equal( const Point& A, const Point& B )
+inline bool approx_equal( const Point &A, const Point &B )
 {
-    double tol = 1e-7*sqrt(A.x*A.x+A.y*A.y+A.z*A.z); 
+    double tol = 1e-7 * sqrt( A.x * A.x + A.y * A.y + A.z * A.z );
-    return fabs(A.x-B.x)<=tol && fabs(A.y-B.y)<=tol && fabs(A.z-B.z)<=tol;
+    return fabs( A.x - B.x ) <= tol && fabs( A.y - B.y ) <= tol && fabs( A.z - B.z ) <= tol;
 }
-inline bool approx_equal( const double& A, const double& B )
+inline bool approx_equal( const double &A, const double &B )
 {
-    return fabs(A-B) <= std::max<double>(1e-7*fabs(A+B),1e-20);
+    return fabs( A - B ) <= std::max<double>( 1e-7 * fabs( A + B ), 1e-20 );
 }
 
 
-inline double distance( const Point& p )
+inline double distance( const Point &p ) { return sqrt( p.x * p.x + p.y * p.y + p.z * p.z ); }
+
+
+bool checkMesh( const std::vector<IO::MeshDataStruct> &meshData, const std::string &format,
+    std::shared_ptr<IO::Mesh> mesh )
 {
-    return sqrt(p.x*p.x+p.y*p.y+p.z*p.z);
+
+    // Get direct access to the meshes used to test the reader
+    const auto pointmesh = dynamic_cast<IO::PointList *>( meshData[0].mesh.get() );
+    const auto trimesh   = dynamic_cast<IO::TriMesh *>( meshData[1].mesh.get() );
+    const auto trilist   = dynamic_cast<IO::TriList *>( meshData[2].mesh.get() );
+    const auto domain    = dynamic_cast<IO::DomainMesh *>( meshData[3].mesh.get() );
+    const size_t N_tri   = trimesh->A.size();
+    if ( mesh->className() == "pointmesh" ) {
+        // Check the pointmesh
+        auto pmesh = IO::getPointList( mesh );
+        if ( pmesh.get() == NULL )
+            return false;
+        if ( pmesh->points.size() != pointmesh->points.size() )
+            return false;
+    }
+    if ( mesh->className() == "trimesh" || mesh->className() == "trilist" ) {
+        // Check the trimesh/trilist
+        auto mesh1 = IO::getTriMesh( mesh );
+        auto mesh2 = IO::getTriList( mesh );
+        if ( mesh1.get() == NULL || mesh2.get() == NULL )
+            return false;
+        if ( mesh1->A.size() != N_tri || mesh1->B.size() != N_tri || mesh1->C.size() != N_tri ||
+             mesh2->A.size() != N_tri || mesh2->B.size() != N_tri || mesh2->C.size() != N_tri )
+            return false;
+        const std::vector<Point> &P1 = mesh1->vertices->points;
+        const std::vector<int> &A1   = mesh1->A;
+        const std::vector<int> &B1   = mesh1->B;
+        const std::vector<int> &C1   = mesh1->C;
+        const std::vector<Point> &A2 = mesh2->A;
+        const std::vector<Point> &B2 = mesh2->B;
+        const std::vector<Point> &C2 = mesh2->C;
+        const std::vector<Point> &A  = trilist->A;
+        const std::vector<Point> &B  = trilist->B;
+        const std::vector<Point> &C  = trilist->C;
+        for ( size_t i = 0; i < N_tri; i++ ) {
+            if ( !approx_equal( P1[A1[i]], A[i] ) || !approx_equal( P1[B1[i]], B[i] ) ||
+                 !approx_equal( P1[C1[i]], C[i] ) )
+                return false;
+            if ( !approx_equal( A2[i], A[i] ) || !approx_equal( B2[i], B[i] ) ||
+                 !approx_equal( C2[i], C[i] ) )
+                return false;
+        }
+    }
+    if ( mesh->className() == "domain" && format != "old" ) {
+        // Check the domain mesh
+        const IO::DomainMesh &mesh1 = *std::dynamic_pointer_cast<IO::DomainMesh>( mesh );
+        if ( mesh1.nprocx != domain->nprocx || mesh1.nprocy != domain->nprocy ||
+             mesh1.nprocz != domain->nprocz )
+            return false;
+        if ( mesh1.nx != domain->nx || mesh1.ny != domain->ny || mesh1.nz != domain->nz )
+            return false;
+        if ( mesh1.Lx != domain->Lx || mesh1.Ly != domain->Ly || mesh1.Lz != domain->Lz )
+            return false;
+    }
+    return true;
+}
+
+
+bool checkVar( const std::string &format, std::shared_ptr<IO::Mesh> mesh,
+    std::shared_ptr<IO::Variable> variable1, std::shared_ptr<IO::Variable> variable2 )
+{
+    if ( format == "new" )
+        IO::reformatVariable( *mesh, *variable2 );
+    bool pass                = true;
+    const IO::Variable &var1 = *variable1;
+    const IO::Variable &var2 = *variable2;
+    pass                     = var1.name == var2.name;
+    pass                     = pass && var1.dim == var2.dim;
+    pass                     = pass && var1.type == var2.type;
+    pass                     = pass && var1.data.length() == var2.data.length();
+    if ( pass ) {
+        for ( size_t m = 0; m < var1.data.length(); m++ )
+            pass = pass && approx_equal( var1.data( m ), var2.data( m ) );
+    }
+    return pass;
 }
 
 
 // Test writing and reading the given format
-void testWriter( const std::string& format, std::vector<IO::MeshDataStruct>& meshData, UnitTest& ut )
+void testWriter(
+    const std::string &format, std::vector<IO::MeshDataStruct> &meshData, UnitTest &ut )
 {
+    PROFILE_SCOPED( path, 0, timer );
+
     Utilities::MPI comm( MPI_COMM_WORLD );
     int nprocs = comm.getSize();
     comm.barrier();
 
+
+    // Set the path for the writer
+    std::string path = "test_" + format;
+
+
     // Get the format
     std::string format2 = format;
-    auto precision = IO::DataType::Double;
+    auto precision      = IO::DataType::Double;
     if ( format == "silo-double" ) {
-        format2 = "silo";
+        format2   = "silo";
         precision = IO::DataType::Double;
     } else if ( format == "silo-float" ) {
-        format2 = "silo";
+        format2   = "silo";
         precision = IO::DataType::Float;
     }
 
+
     // Set the precision for the variables
-    for ( auto& data : meshData ) {
+    for ( auto &data : meshData ) {
         data.precision = precision;
-        for ( auto& var : data.vars )
+        for ( auto &var : data.vars )
             var->precision = precision;
     }
 
     // Write the data
-    PROFILE_START(format+"-write");
+    IO::initialize( path, format2, false );
-    IO::initialize( "test_"+format, format2, false );
     IO::writeData( 0, meshData, comm );
     IO::writeData( 3, meshData, comm );
     comm.barrier();
-    PROFILE_STOP(format+"-write");
 
-    // Get the summary name for reading
+
-    std::string path = "test_" + format;
+    // Get a list of the timesteps
-    std::string summary_name;
+    auto timesteps = IO::readTimesteps( path, format2 );
-    if ( format=="old" || format=="new" )
+    if ( timesteps.size() == 2 )
-        summary_name = "summary.LBM";
+        ut.passes( format + ": Corrent number of timesteps" );
-    else if ( format=="silo-float" || format=="silo-double" )
-        summary_name = "LBM.visit";
     else
-        ERROR("Unknown format");
+        ut.failure( format + ": Incorrent number of timesteps" );
 
-    // Get direct access to the meshes used to test the reader
-    const auto pointmesh = dynamic_cast<IO::PointList*>(  meshData[0].mesh.get() );
-    const auto trimesh   = dynamic_cast<IO::TriMesh*>(    meshData[1].mesh.get() );
-    const auto trilist   = dynamic_cast<IO::TriList*>(    meshData[2].mesh.get() );
-    const auto domain    = dynamic_cast<IO::DomainMesh*>( meshData[3].mesh.get() );
-    const size_t N_tri = trimesh->A.size();
 
-    // Get a list of the timesteps 
+    // Test the simple read interface
-    PROFILE_START(format+"-read-timesteps");
+    bool pass = true;
-    auto timesteps = IO::readTimesteps( path + "/" + summary_name );
+    for ( const auto &timestep : timesteps ) {
-    PROFILE_STOP(format+"-read-timesteps");
+        auto data = IO::readData( path, timestep, comm.getRank() );
-    if ( timesteps.size()==2 )
+        pass      = pass && data.size() == meshData.size();
-        ut.passes(format+": Corrent number of timesteps");
+        for ( size_t i = 0; i < data.size(); i++ ) {
-    else
+            pass = pass && checkMesh( meshData, format, data[i].mesh );
-        ut.failure(format+": Incorrent number of timesteps");
-
-    // Check the mesh lists
-    for ( const auto& timestep : timesteps ) {
-        // Load the list of meshes and check its size
-        PROFILE_START(format+"-read-getMeshList");
-        auto databaseList = IO::getMeshList(path,timestep);
-        PROFILE_STOP(format+"-read-getMeshList");
-        if ( databaseList.size()==meshData.size() )
-            ut.passes(format+": Corrent number of meshes found");
-        else
-            ut.failure(format+": Incorrent number of meshes found");
-        // Check the number of domains for each mesh
-        bool pass = true;
-        for ( const auto& database : databaseList )
-            pass = pass && (int)database.domains.size()==nprocs;
-        if ( pass ) {
-            ut.passes(format+": Corrent number of domains for mesh");
-        } else {
-            ut.failure(format+": Incorrent number of domains for mesh");
-            continue;
         }
-        // For each domain, load the mesh and check its data
+    }
-        for ( const auto& database : databaseList ) {
+    if ( pass )
-            pass = true;
+        ut.passes( format + ": Simple read interface" );
-            for (size_t k=0; k<database.domains.size(); k++) {
+    else
-                PROFILE_START(format+"-read-getMesh");
+        ut.failure( format + ": Simple read interface" );
-                auto mesh = IO::getMesh(path,timestep,database,k);
+
-                PROFILE_STOP(format+"-read-getMesh");
+
-                if ( mesh.get()==NULL ) {
+    // Test reading each mesh domain
-                    printf("Failed to load %s\n",database.name.c_str());
+    for ( const auto &timestep : timesteps ) {
+        // Load the list of meshes and check its size
+        auto databaseList = IO::getMeshList( path, timestep );
+        if ( databaseList.size() == meshData.size() )
+            ut.passes( format + ": Corrent number of meshes found" );
+        else
+            ut.failure( format + ": Incorrent number of meshes found" );
+        // Check the number of domains for each mesh
+        for ( const auto &database : databaseList ) {
+            int N_domains = database.domains.size();
+            if ( N_domains != nprocs ) {
+                ut.failure( format + ": Incorrent number of domains for mesh" );
+                continue;
+            }
+            // For each domain, load the mesh and check its data
+            bool pass = true;
+            for ( int k = 0; k < N_domains; k++ ) {
+                auto mesh = IO::getMesh( path, timestep, database, k );
+                if ( !mesh ) {
+                    ut.failure( "Failed to load " + database.name );
                     pass = false;
-                    break;
+                } else {
-                }
+                    pass = pass && checkMesh( meshData, format, mesh );
-                if ( database.name=="pointmesh" ) {
-                    // Check the pointmesh
-                    auto pmesh = IO::getPointList(mesh);
-                    if ( pmesh.get()==NULL ) {
-                        pass = false;
-                        break;
-                    }
-                    if ( pmesh->points.size() != pointmesh->points.size() ) {
-                        pass = false;
-                        break;
-                    }                    
-                }
-                if ( database.name=="trimesh" || database.name=="trilist" ) {
-                    // Check the trimesh/trilist
-                    auto mesh1 = IO::getTriMesh(mesh);
-                    auto mesh2 = IO::getTriList(mesh);
-                    if ( mesh1.get()==NULL || mesh2.get()==NULL ) {
-                        pass = false;
-                        break;
-                    }
-                    if ( mesh1->A.size()!=N_tri || mesh1->B.size()!=N_tri || mesh1->C.size()!=N_tri || 
-                         mesh2->A.size()!=N_tri || mesh2->B.size()!=N_tri || mesh2->C.size()!=N_tri ) 
-                    {
-                        pass = false;
-                        break;
-                    }
-                    const std::vector<Point>& P1 = mesh1->vertices->points;
-                    const std::vector<int>& A1 = mesh1->A;
-                    const std::vector<int>& B1 = mesh1->B;
-                    const std::vector<int>& C1 = mesh1->C;
-                    const std::vector<Point>& A2 = mesh2->A;
-                    const std::vector<Point>& B2 = mesh2->B;
-                    const std::vector<Point>& C2 = mesh2->C;
-                    const std::vector<Point>& A = trilist->A;
-                    const std::vector<Point>& B = trilist->B;
-                    const std::vector<Point>& C = trilist->C;
-                    for (size_t i=0; i<N_tri; i++) {
-                        if ( !approx_equal(P1[A1[i]],A[i]) || !approx_equal(P1[B1[i]],B[i]) || !approx_equal(P1[C1[i]],C[i]) )
-                            pass = false;
-                        if ( !approx_equal(A2[i],A[i]) || !approx_equal(B2[i],B[i]) || !approx_equal(C2[i],C[i]) )
-                            pass = false;
-                    }
-                }
-                if ( database.name=="domain" && format!="old" ) {
-                    // Check the domain mesh
-                    const IO::DomainMesh& mesh1 = *std::dynamic_pointer_cast<IO::DomainMesh>(mesh);
-                    if ( mesh1.nprocx!=domain->nprocx || mesh1.nprocy!=domain->nprocy || mesh1.nprocz!=domain->nprocz )
-                        pass = false;
-                    if ( mesh1.nx!=domain->nx || mesh1.ny!=domain->ny || mesh1.nz!=domain->nz )
-                        pass = false;
-                    if ( mesh1.Lx!=domain->Lx || mesh1.Ly!=domain->Ly || mesh1.Lz!=domain->Lz )
-                        pass = false;
                 }
             }
             if ( pass ) {
-                ut.passes(format+": Mesh \"" + database.name + "\" loaded correctly");
+                ut.passes( format + ": Mesh \"" + database.name + "\" loaded correctly" );
             } else {
-                ut.failure(format+": Mesh \"" + database.name + "\" did not load correctly");
+                ut.failure( format + ": Mesh \"" + database.name + "\" did not load correctly" );
                 continue;
             }
             // Load the variables and check their data
-            if ( format=="old" )
+            if ( format == "old" )
-                continue;   // Old format does not support variables
+                continue; // Old format does not support variables
-            const IO::MeshDataStruct* mesh0 = NULL;
+            const IO::MeshDataStruct *mesh0 = nullptr;
-            for (size_t k=0; k<meshData.size(); k++) {
+            for ( size_t k = 0; k < meshData.size(); k++ ) {
                 if ( meshData[k].meshName == database.name ) {
                     mesh0 = &meshData[k];
                     break;
                 }
             }
-            for (size_t k=0; k<database.domains.size(); k++) {
+            for ( int k = 0; k < N_domains; k++ ) {
-                auto mesh = IO::getMesh(path,timestep,database,k);
+                auto mesh = IO::getMesh( path, timestep, database, k );
-                for (size_t v=0; v<mesh0->vars.size(); v++) {
+                for ( size_t v = 0; v < mesh0->vars.size(); v++ ) {
-                    PROFILE_START(format+"-read-getVariable");
+                    PROFILE_START( format + "-read-getVariable" );
-                    auto variable = IO::getVariable(path,timestep,database,k,mesh0->vars[v]->name);
+                    auto variable =
-                    if ( format=="new" )
+                        IO::getVariable( path, timestep, database, k, mesh0->vars[v]->name );
-                        IO::reformatVariable( *mesh, *variable );
+                    pass = checkVar( format, mesh, mesh0->vars[v], variable );
-                    PROFILE_STOP(format+"-read-getVariable");
-                    const IO::Variable& var1 = *mesh0->vars[v];
-                    const IO::Variable& var2 = *variable;
-                    pass = var1.name == var2.name;
-                    pass = pass && var1.dim == var2.dim;
-                    pass = pass && var1.type == var2.type;
-                    pass = pass && var1.data.length() == var2.data.length();
                     if ( pass ) {
-                        for (size_t m=0; m<var1.data.length(); m++)
+                        ut.passes( format + ": Variable \"" + variable->name + "\" matched" );
-                            pass = pass && approx_equal(var1.data(m),var2.data(m));
-                    }
-                    if ( pass ) {
-                        ut.passes(format+": Variable \"" + variable->name + "\" matched");
                     } else {
-                        ut.failure(format+": Variable \"" + variable->name + "\" did not match");
+                        ut.failure(
+                            format + ": Variable \"" + variable->name + "\" did not match" );
                         break;
                     }
                 }
@@ -224,157 +237,161 @@ void testWriter( const std::string& format, std::vector<IO::MeshDataStruct>& mes
 
 
 // Main
-int main(int argc, char **argv)
+int main( int argc, char **argv )
 {
     Utilities::startup( argc, argv );
     Utilities::MPI comm( MPI_COMM_WORLD );
-    int rank = comm.getRank();
+    int rank   = comm.getRank();
     int nprocs = comm.getSize();
-    Utilities::setAbortBehavior(true,2);
+    Utilities::setAbortBehavior( true, 2 );
     Utilities::setErrorHandlers();
     UnitTest ut;
 
     // Create some points
     const int N_points = 8;
-    const int N_tri = 12;
+    const int N_tri    = 12;
-    double x[8] = { 0, 1, 0, 1, 0, 1, 0, 1 };
+    double x[8]        = { 0, 1, 0, 1, 0, 1, 0, 1 };
-    double y[8] = { 0, 0, 1, 1, 0, 0, 1, 1 };
+    double y[8]        = { 0, 0, 1, 1, 0, 0, 1, 1 };
-    double z[8] = { 0, 0, 0, 0, 1, 1, 1, 1 };
+    double z[8]        = { 0, 0, 0, 0, 1, 1, 1, 1 };
-    int tri[N_tri][3] = { 
+    int tri[N_tri][3]  = {
-        {0,1,3}, {0,3,2}, {4,5,7}, {4,7,6}, // z faces
+        { 0, 1, 3 }, { 0, 3, 2 }, { 4, 5, 7 }, { 4, 7, 6 }, // z faces
-        {0,1,4}, {1,4,5}, {2,3,6}, {3,6,7}, // y faces
+        { 0, 1, 4 }, { 1, 4, 5 }, { 2, 3, 6 }, { 3, 6, 7 }, // y faces
-        {0,2,4}, {2,4,6}, {1,3,5}, {3,5,7}  // x faces
+        { 0, 2, 4 }, { 2, 4, 6 }, { 1, 3, 5 }, { 3, 5, 7 }  // x faces
     };
 
     // Create the meshes
-    auto set1 = std::make_shared<IO::PointList>(N_points);
+    auto set1 = std::make_shared<IO::PointList>( N_points );
-    for (int i=0; i<N_points; i++) {
+    for ( int i = 0; i < N_points; i++ ) {
         set1->points[i].x = x[i];
         set1->points[i].y = y[i];
         set1->points[i].z = z[i];
     }
-    auto trimesh = std::make_shared<IO::TriMesh>(N_tri,set1);
+    auto trimesh = std::make_shared<IO::TriMesh>( N_tri, set1 );
-    for (int i=0; i<N_tri; i++) {
+    for ( int i = 0; i < N_tri; i++ ) {
         trimesh->A[i] = tri[i][0];
         trimesh->B[i] = tri[i][1];
         trimesh->C[i] = tri[i][2];
     }
-    auto trilist = std::make_shared<IO::TriList>(*trimesh);
+    auto trilist = std::make_shared<IO::TriList>( *trimesh );
-    for (int i=0; i<N_tri; i++) {
+    for ( int i = 0; i < N_tri; i++ ) {
-        Point A(x[tri[i][0]],y[tri[i][0]],z[tri[i][0]]);
+        Point A( x[tri[i][0]], y[tri[i][0]], z[tri[i][0]] );
-        Point B(x[tri[i][1]],y[tri[i][1]],z[tri[i][1]]);
+        Point B( x[tri[i][1]], y[tri[i][1]], z[tri[i][1]] );
-        Point C(x[tri[i][2]],y[tri[i][2]],z[tri[i][2]]);
+        Point C( x[tri[i][2]], y[tri[i][2]], z[tri[i][2]] );
-        if ( !approx_equal(trilist->A[i],A) || !approx_equal(trilist->B[i],B) || !approx_equal(trilist->C[i],C) )
+        if ( !approx_equal( trilist->A[i], A ) || !approx_equal( trilist->B[i], B ) ||
-        {
+             !approx_equal( trilist->C[i], C ) ) {
-            printf("Failed to create trilist\n");
+            printf( "Failed to create trilist\n" );
             return -1;
         }
     }
     RankInfoStruct rank_data( rank, nprocs, 1, 1 );
-    auto domain = std::make_shared<IO::DomainMesh>(rank_data,6,7,8,1.0,1.0,1.0);
+    auto domain = std::make_shared<IO::DomainMesh>( rank_data, 6, 7, 8, 1.0, 1.0, 1.0 );
 
     // Create the variables
     const auto NodeVar   = IO::VariableType::NodeVariable;
     const auto VolVar    = IO::VariableType::VolumeVariable;
-    auto set_node_mag    = std::make_shared<IO::Variable>(1,NodeVar,"Node_set_mag");
+    auto set_node_mag    = std::make_shared<IO::Variable>( 1, NodeVar, "Node_set_mag" );
-    auto set_node_vec    = std::make_shared<IO::Variable>(3,NodeVar,"Node_set_vec");
+    auto set_node_vec    = std::make_shared<IO::Variable>( 3, NodeVar, "Node_set_vec" );
-    auto list_node_mag   = std::make_shared<IO::Variable>(1,NodeVar,"Node_list_mag");
+    auto list_node_mag   = std::make_shared<IO::Variable>( 1, NodeVar, "Node_list_mag" );
-    auto list_node_vec   = std::make_shared<IO::Variable>(3,NodeVar,"Node_list_vec");
+    auto list_node_vec   = std::make_shared<IO::Variable>( 3, NodeVar, "Node_list_vec" );
-    auto point_node_mag  = std::make_shared<IO::Variable>(1,NodeVar,"Node_point_mag");
+    auto point_node_mag  = std::make_shared<IO::Variable>( 1, NodeVar, "Node_point_mag" );
-    auto point_node_vec  = std::make_shared<IO::Variable>(3,NodeVar,"Node_point_vec");
+    auto point_node_vec  = std::make_shared<IO::Variable>( 3, NodeVar, "Node_point_vec" );
-    auto domain_node_mag = std::make_shared<IO::Variable>(1,NodeVar,"Node_domain_mag");
+    auto domain_node_mag = std::make_shared<IO::Variable>( 1, NodeVar, "Node_domain_mag" );
-    auto domain_node_vec = std::make_shared<IO::Variable>(3,NodeVar,"Node_domain_vec");
+    auto domain_node_vec = std::make_shared<IO::Variable>( 3, NodeVar, "Node_domain_vec" );
-    auto set_cell_mag    = std::make_shared<IO::Variable>(1,VolVar,"Cell_set_mag");
+    auto set_cell_mag    = std::make_shared<IO::Variable>( 1, VolVar, "Cell_set_mag" );
-    auto set_cell_vec    = std::make_shared<IO::Variable>(3,VolVar,"Cell_set_vec");
+    auto set_cell_vec    = std::make_shared<IO::Variable>( 3, VolVar, "Cell_set_vec" );
-    auto list_cell_mag   = std::make_shared<IO::Variable>(1,VolVar,"Cell_list_mag");
+    auto list_cell_mag   = std::make_shared<IO::Variable>( 1, VolVar, "Cell_list_mag" );
-    auto list_cell_vec   = std::make_shared<IO::Variable>(3,VolVar,"Cell_list_vec");
+    auto list_cell_vec   = std::make_shared<IO::Variable>( 3, VolVar, "Cell_list_vec" );
-    auto domain_cell_mag = std::make_shared<IO::Variable>(1,VolVar,"Cell_domain_mag");
+    auto domain_cell_mag = std::make_shared<IO::Variable>( 1, VolVar, "Cell_domain_mag" );
-    auto domain_cell_vec = std::make_shared<IO::Variable>(3,VolVar,"Cell_domain_vec");
+    auto domain_cell_vec = std::make_shared<IO::Variable>( 3, VolVar, "Cell_domain_vec" );
     point_node_mag->data.resize( N_points );
     point_node_vec->data.resize( N_points, 3 );
-    for (int i=0; i<N_points; i++) {
+    for ( int i = 0; i < N_points; i++ ) {
-        point_node_mag->data(i) = distance(set1->points[i]);
+        point_node_mag->data( i )    = distance( set1->points[i] );
-        point_node_vec->data(i,0) = set1->points[i].x;
+        point_node_vec->data( i, 0 ) = set1->points[i].x;
-        point_node_vec->data(i,1) = set1->points[i].y;
+        point_node_vec->data( i, 1 ) = set1->points[i].y;
-        point_node_vec->data(i,2) = set1->points[i].z;
+        point_node_vec->data( i, 2 ) = set1->points[i].z;
     }
     set_node_mag->data = point_node_mag->data;
     set_node_vec->data = point_node_vec->data;
-    list_node_mag->data.resize( 3*N_tri );
+    list_node_mag->data.resize( 3 * N_tri );
-    list_node_vec->data.resize( 3*N_tri, 3 );
+    list_node_vec->data.resize( 3 * N_tri, 3 );
-    for (int i=0; i<N_points; i++) {
+    for ( int i = 0; i < N_points; i++ ) {
-        list_node_mag->data(3*i+0) = distance(trilist->A[i]);
+        list_node_mag->data( 3 * i + 0 )    = distance( trilist->A[i] );
-        list_node_mag->data(3*i+1) = distance(trilist->B[i]);
+        list_node_mag->data( 3 * i + 1 )    = distance( trilist->B[i] );
-        list_node_mag->data(3*i+2) = distance(trilist->C[i]);
+        list_node_mag->data( 3 * i + 2 )    = distance( trilist->C[i] );
-        list_node_vec->data(3*i+0,0) = trilist->A[i].x;
+        list_node_vec->data( 3 * i + 0, 0 ) = trilist->A[i].x;
-        list_node_vec->data(3*i+0,1) = trilist->A[i].y;
+        list_node_vec->data( 3 * i + 0, 1 ) = trilist->A[i].y;
-        list_node_vec->data(3*i+0,2) = trilist->A[i].z;
+        list_node_vec->data( 3 * i + 0, 2 ) = trilist->A[i].z;
-        list_node_vec->data(3*i+1,0) = trilist->B[i].x;
+        list_node_vec->data( 3 * i + 1, 0 ) = trilist->B[i].x;
-        list_node_vec->data(3*i+1,1) = trilist->B[i].y;
+        list_node_vec->data( 3 * i + 1, 1 ) = trilist->B[i].y;
-        list_node_vec->data(3*i+1,2) = trilist->B[i].z;
+        list_node_vec->data( 3 * i + 1, 2 ) = trilist->B[i].z;
-        list_node_vec->data(3*i+2,0) = trilist->C[i].x;
+        list_node_vec->data( 3 * i + 2, 0 ) = trilist->C[i].x;
-        list_node_vec->data(3*i+2,1) = trilist->C[i].y;
+        list_node_vec->data( 3 * i + 2, 1 ) = trilist->C[i].y;
-        list_node_vec->data(3*i+2,2) = trilist->C[i].z;
+        list_node_vec->data( 3 * i + 2, 2 ) = trilist->C[i].z;
     }
-    domain_node_mag->data.resize(domain->nx+1,domain->ny+1,domain->nz+1);
+    domain_node_mag->data.resize( domain->nx + 1, domain->ny + 1, domain->nz + 1 );
-    domain_node_vec->data.resize({(size_t)domain->nx+1,(size_t)domain->ny+1,(size_t)domain->nz+1,3});
+    domain_node_vec->data.resize(
-    for (int i=0; i<domain->nx+1; i++) {
+        { (size_t) domain->nx + 1, (size_t) domain->ny + 1, (size_t) domain->nz + 1, 3 } );
-        for (int j=0; j<domain->ny+1; j++) {
+    for ( int i = 0; i < domain->nx + 1; i++ ) {
-            for (int k=0; k<domain->nz+1; k++) {
+        for ( int j = 0; j < domain->ny + 1; j++ ) {
-                domain_node_mag->data(i,j,k) = distance(Point(i,j,k));
+            for ( int k = 0; k < domain->nz + 1; k++ ) {
-                domain_node_vec->data(i,j,k,0) = Point(i,j,k).x;
+                domain_node_mag->data( i, j, k )    = distance( Point( i, j, k ) );
-                domain_node_vec->data(i,j,k,1) = Point(i,j,k).y;
+                domain_node_vec->data( i, j, k, 0 ) = Point( i, j, k ).x;
-                domain_node_vec->data(i,j,k,2) = Point(i,j,k).z;
+                domain_node_vec->data( i, j, k, 1 ) = Point( i, j, k ).y;
+                domain_node_vec->data( i, j, k, 2 ) = Point( i, j, k ).z;
             }
         }
     }
     set_cell_mag->data.resize( N_tri );
     set_cell_vec->data.resize( N_tri, 3 );
-    for (int i=0; i<N_tri; i++) {
+    for ( int i = 0; i < N_tri; i++ ) {
-        set_cell_mag->data(i) = i;
+        set_cell_mag->data( i )    = i;
-        set_cell_vec->data(i,0) = 3*i+0;
+        set_cell_vec->data( i, 0 ) = 3 * i + 0;
-        set_cell_vec->data(i,1) = 3*i+1;
+        set_cell_vec->data( i, 1 ) = 3 * i + 1;
-        set_cell_vec->data(i,2) = 3*i+2;
+        set_cell_vec->data( i, 2 ) = 3 * i + 2;
     }
     list_cell_mag->data = set_cell_mag->data;
     list_cell_vec->data = set_cell_vec->data;
-    domain_cell_mag->data.resize(domain->nx,domain->ny,domain->nz);
+    domain_cell_mag->data.resize( domain->nx, domain->ny, domain->nz );
-    domain_cell_vec->data.resize({(size_t)domain->nx,(size_t)domain->ny,(size_t)domain->nz,3});
+    domain_cell_vec->data.resize(
-    for (int i=0; i<domain->nx; i++) {
+        { (size_t) domain->nx, (size_t) domain->ny, (size_t) domain->nz, 3 } );
-        for (int j=0; j<domain->ny; j++) {
+    for ( int i = 0; i < domain->nx; i++ ) {
-            for (int k=0; k<domain->nz; k++) {
+        for ( int j = 0; j < domain->ny; j++ ) {
-                domain_cell_mag->data(i,j,k) = distance(Point(i,j,k));
+            for ( int k = 0; k < domain->nz; k++ ) {
-                domain_cell_vec->data(i,j,k,0) = Point(i,j,k).x;
+                domain_cell_mag->data( i, j, k )    = distance( Point( i, j, k ) );
-                domain_cell_vec->data(i,j,k,1) = Point(i,j,k).y;
+                domain_cell_vec->data( i, j, k, 0 ) = Point( i, j, k ).x;
-                domain_cell_vec->data(i,j,k,2) = Point(i,j,k).z;
+                domain_cell_vec->data( i, j, k, 1 ) = Point( i, j, k ).y;
+                domain_cell_vec->data( i, j, k, 2 ) = Point( i, j, k ).z;
             }
         }
     }
 
     // Create the MeshDataStruct
-    std::vector<IO::MeshDataStruct> meshData(4);
+    std::vector<IO::MeshDataStruct> meshData( 4 );
     meshData[0].meshName = "pointmesh";
-    meshData[0].mesh = set1;
+    meshData[0].mesh     = set1;
-    meshData[0].vars.push_back(point_node_mag);
+    meshData[0].vars.push_back( point_node_mag );
-    meshData[0].vars.push_back(point_node_vec);
+    meshData[0].vars.push_back( point_node_vec );
     meshData[1].meshName = "trimesh";
-    meshData[1].mesh = trimesh;
+    meshData[1].mesh     = trimesh;
-    meshData[1].vars.push_back(set_node_mag);
+    meshData[1].vars.push_back( set_node_mag );
-    meshData[1].vars.push_back(set_node_vec);
+    meshData[1].vars.push_back( set_node_vec );
-    meshData[1].vars.push_back(set_cell_mag);
+    meshData[1].vars.push_back( set_cell_mag );
-    meshData[1].vars.push_back(set_cell_vec);
+    meshData[1].vars.push_back( set_cell_vec );
     meshData[2].meshName = "trilist";
-    meshData[2].mesh = trilist;
+    meshData[2].mesh     = trilist;
-    meshData[2].vars.push_back(list_node_mag);
+    meshData[2].vars.push_back( list_node_mag );
-    meshData[2].vars.push_back(list_node_vec);
+    meshData[2].vars.push_back( list_node_vec );
-    meshData[2].vars.push_back(list_cell_mag);
+    meshData[2].vars.push_back( list_cell_mag );
-    meshData[2].vars.push_back(list_cell_vec);
+    meshData[2].vars.push_back( list_cell_vec );
     meshData[3].meshName = "domain";
-    meshData[3].mesh = domain;
+    meshData[3].mesh     = domain;
-    meshData[3].vars.push_back(domain_node_mag);
+    meshData[3].vars.push_back( domain_node_mag );
-    meshData[3].vars.push_back(domain_node_vec);
+    meshData[3].vars.push_back( domain_node_vec );
-    meshData[3].vars.push_back(domain_cell_mag);
+    meshData[3].vars.push_back( domain_cell_mag );
-    meshData[3].vars.push_back(domain_cell_vec);
+    meshData[3].vars.push_back( domain_cell_vec );
+    for ( const auto &data : meshData )
+        ASSERT( data.check( true ) );
 
     // Run the tests
     testWriter( "old", meshData, ut );
@@ -384,11 +401,9 @@ int main(int argc, char **argv)
 
     // Finished
     ut.report();
-    PROFILE_SAVE("TestWriter",true);
+    PROFILE_SAVE( "TestWriter", true );
     int N_errors = ut.NumFailGlobal();
     comm.barrier();
     Utilities::shutdown();
     return N_errors;
 }
-
-

tests/convertIO.cpp

@@ -5,85 +5,66 @@
 #include <stdexcept>
 #include <fstream>
 
-#include "common/MPI_Helpers.h"
+#include "common/MPI.h"
-#include "common/Communication.h"
 #include "common/Utilities.h"
 #include "IO/Mesh.h"
 #include "IO/Reader.h"
 #include "IO/Writer.h"
 #include "ProfilerApp.h"
 
-
 int main(int argc, char **argv)
 {
-  // Initialize MPI
+    // Initialize MPI
-  Utilities::startup( argc, argv );
+    Utilities::startup( argc, argv );
-  Utilities::MPI comm( MPI_COMM_WORLD );
+    Utilities::setErrorHandlers();
-  int rank = comm.getRank();
+    PROFILE_ENABLE(2);
-  int nprocs = comm.getSize();
+    PROFILE_ENABLE_TRACE();
-  Utilities::setErrorHandlers();
+    PROFILE_START("Main");
-  PROFILE_ENABLE(2);
-  PROFILE_ENABLE_TRACE();
-  PROFILE_START("Main");
-  { // Limit scope
 
-    // Get inputs
+    { // Limit scope
-    if ( argc != 3 ) {
-        std::cerr << "Error calling convertIO:\n";
-        std::cerr << "   convertIO input_file format\n";
-        return -1;
-    }
-    std::string filename = argv[1];
-    std::string format = argv[2];
-    std::string path = IO::getPath( filename );
 
-    // Read the timesteps
-    auto timesteps = IO::readTimesteps( filename );
 
-    // Loop through the timesteps, reading/writing the data
+        Utilities::MPI comm( MPI_COMM_WORLD );
-    IO::initialize( "", format, false );
+        // Get inputs
-    for ( auto timestep : timesteps ) {
+        if ( argc != 5 ) {
-        
+            std::cerr << "Error calling convertIO:\n";
-        // Read the list of MeshDatabase
+            std::cerr << "   convertIO <input_path> <input_format> <output_path> <output_format>\n";
-        auto databases = IO::getMeshList( path, timestep );
+            return -1;
-
-        // Build the MeshDataStruct
-        std::vector<IO::MeshDataStruct> meshData(databases.size());
-
-        // Loop through the database
-        int i = 0;
-        PROFILE_START("Read");
-        for ( const auto& database : databases ) {
-            
-            // Read the appropriate mesh domain
-            ASSERT( (int) database.domains.size() == nprocs );
-            meshData[i].meshName = database.name;
-            meshData[i].mesh = IO::getMesh( path, timestep, database, rank );
-
-            // Read the variables
-            for ( auto var : database.variables ) {
-                auto varData = IO::getVariable( path, timestep, database, rank, var.name );
-                IO::reformatVariable( *meshData[i].mesh, *varData );
-                meshData[i].vars.push_back( varData );
-            }
-
-            i++;
         }
-        MPI_Barrier(comm);
+        std::string path_in = argv[1];
-        PROFILE_STOP("Read");
+        std::string format_in = argv[2];
+        std::string path_out = argv[3];
+        std::string format_out = argv[4];
 
-        // Save the mesh data to a new file
+        // Check that we have enough ranks to load and write the data
-        PROFILE_START("Write");
+        // This is really only a bottleneck for the writer
-        IO::writeData( timestep, meshData, MPI_COMM_WORLD );
+        int N_domains = IO::maxDomains( path_in, format_in, comm );
-        MPI_Barrier(comm);
+        ASSERT( comm.getSize() == N_domains );
-        PROFILE_STOP("Write");
-    }
 
-  } // Limit scope
+        // Read the timesteps
-  PROFILE_STOP("Main");
+        auto timesteps = IO::readTimesteps( path_in, format_in );
-  PROFILE_SAVE("convertData",true);
+
-  comm.barrier();
+        // Loop through the timesteps, reading/writing the data
-  Utilities::shutdown();
+        IO::initialize( path_out, format_out, false );
-  return 0;
+        for ( auto timestep : timesteps ) {
+            
+            // Set the domain to read (needs to be the current rank for the writer to be valid)
+            int domain = comm.getRank();
+
+            // Get the maximum number of domains for the 
+            auto data = IO::readData( path_in, timestep, domain );
+
+            // Save the mesh data to a new file
+            IO::writeData( timestep, data, comm );
+
+        }
+
+    } // Limit scope
+
+    // shutdown
+    PROFILE_STOP("Main");
+    PROFILE_SAVE("convertData",true);
+    Utilities::shutdown();
+    return 0;
 }
 

tests/lbpm_color_simulator.cpp

@@ -27,19 +27,24 @@ int main( int argc, char **argv )
     // Initialize
     Utilities::startup( argc, argv );
 
-    // Load the input database
-    auto db = std::make_shared<Database>( argv[1] );
-
     { // Limit scope so variables that contain communicators will free before MPI_Finialize
 
         Utilities::MPI comm( MPI_COMM_WORLD );
         int rank   = comm.getRank();
         int nprocs = comm.getSize();
+        std::string SimulationMode = "production";
+        // Load the input database
+        auto db = std::make_shared<Database>( argv[1] );
+        if (argc > 2) {
+        	SimulationMode = "development";
+        }
 
         if ( rank == 0 ) {
             printf( "********************************************************\n" );
             printf( "Running Color LBM	\n" );
             printf( "********************************************************\n" );
+            if (SimulationMode == "development")
+            	printf("**** DEVELOPMENT MODE ENABLED *************\n");
         }
         // Initialize compute device
         int device = ScaLBL_SetDevice( rank );
@@ -62,8 +67,29 @@ int main( int argc, char **argv )
         ColorModel.Create();     // creating the model will create data structure to match the pore
                                  // structure and allocate variables
         ColorModel.Initialize(); // initializing the model will set initial conditions for variables
-        ColorModel.Run();
+       
-        // ColorModel.WriteDebug();
+        if (SimulationMode == "development"){
+            double MLUPS=0.0;
+            int timestep = 0;
+            int analysis_interval = ColorModel.timestepMax;
+        	if (ColorModel.analysis_db->keyExists( "" )){
+        		analysis_interval = ColorModel.analysis_db->getScalar<int>( "analysis_interval" );
+        	}
+        	FlowAdaptor Adapt(ColorModel);
+        	runAnalysis analysis(ColorModel);
+            while (ColorModel.timestep < ColorModel.timestepMax){
+            	timestep += analysis_interval;
+            	MLUPS = ColorModel.Run(timestep);
+            	if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS);
+            	
+            	Adapt.MoveInterface(ColorModel);
+            }
+        }            	//Analysis.WriteVis(LeeModel,LeeModel.db, timestep);
+
+        else
+        	ColorModel.Run();
+        
+        ColorModel.WriteDebug();
 
         PROFILE_STOP( "Main" );
         auto file  = db->getWithDefault<std::string>( "TimerFile", "lbpm_color_simulator" );

tests/lbpm_freelee_simulator.cpp

@@ -8,6 +8,7 @@
 
 #include "common/Utilities.h"
 #include "models/FreeLeeModel.h"
+#include "analysis/FreeEnergy.h"
 
 //*******************************************************************
 // Implementation of Free-Energy Two-Phase LBM (Lee model)
@@ -52,10 +53,33 @@ int main( int argc, char **argv )
         LeeModel.SetDomain();    
         LeeModel.ReadInput();    
         LeeModel.Create_TwoFluid();       
+        
+        FreeEnergyAnalyzer Analysis(LeeModel.Dm);
+        
         LeeModel.Initialize_TwoFluid();   
-        LeeModel.Run_TwoFluid();	       
+        
-        LeeModel.WriteDebug_TwoFluid();
+        /*** RUN MAIN TIMESTEPS HERE ************/
-
+        double MLUPS=0.0;
+        int timestep = 0;
+        int visualization_time = LeeModel.timestepMax;
+    	if (LeeModel.vis_db->keyExists( "visualization_interval" )){
+    		visualization_time = LeeModel.vis_db->getScalar<int>( "visualization_interval" );
+    		timestep += visualization_time;
+    	}
+        while (LeeModel.timestep < LeeModel.timestepMax){
+        	MLUPS = LeeModel.Run_TwoFluid(timestep);
+        	if (rank==0) printf("Lattice update rate (per MPI process)= %f MLUPS \n", MLUPS);
+        	Analysis.WriteVis(LeeModel,LeeModel.db, timestep);
+        	timestep += visualization_time;
+        }
+        //LeeModel.WriteDebug_TwoFluid();
+    	if (rank==0) printf("********************************************************\n");
+    	if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
+    	MLUPS *= nprocs;
+    	if (rank==0) printf("Lattice update rate (total)= %f MLUPS \n", MLUPS);
+    	if (rank==0) printf("********************************************************\n");
+    	// ************************************************************************
+    	
         PROFILE_STOP("Main");
         auto file = db->getWithDefault<std::string>( "TimerFile", "lbpm_freelee_simulator" );
         auto level = db->getWithDefault<int>( "TimerLevel", 1 );

tests/testGlobalMassFreeLee.cpp

@@ -0,0 +1,101 @@
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#include "common/Utilities.h"
+#include "models/FreeLeeModel.h"
+
+//*******************************************************************
+// Implementation of Free-Energy Two-Phase LBM (Lee model)
+//*******************************************************************
+
+int main( int argc, char **argv )
+{
+	  
+    // Initialize
+    Utilities::startup( argc, argv );
+
+    // Load the input database
+    auto db = std::make_shared<Database>( argv[1] );
+  
+    { // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+        Utilities::MPI comm( MPI_COMM_WORLD );
+        int rank = comm.getRank();
+        int nprocs = comm.getSize();
+
+        if (rank == 0){
+            printf("********************************************************\n");
+            printf("Running Free Energy Lee LBM	\n");
+            printf("********************************************************\n");
+        }
+        // Initialize compute device
+        int device=ScaLBL_SetDevice(rank);
+        NULL_USE( device );
+        ScaLBL_DeviceBarrier();
+        comm.barrier();
+
+        PROFILE_ENABLE(1);
+        //PROFILE_ENABLE_TRACE();
+        //PROFILE_ENABLE_MEMORY();
+        PROFILE_SYNCHRONIZE();
+        PROFILE_START("Main");
+        Utilities::setErrorHandlers();
+
+        auto filename = argv[1];
+        ScaLBL_FreeLeeModel LeeModel( rank,nprocs,comm );
+        LeeModel.ReadParams( filename );
+        LeeModel.SetDomain();    
+        LeeModel.ReadInput();    
+        LeeModel.Create_TwoFluid();       
+        LeeModel.Initialize_TwoFluid(); 
+        /* check neighbors */
+        
+        
+        /* Copy the initial density to test that global mass is conserved */
+        int Nx = LeeModel.Dm->Nx;
+        int Ny = LeeModel.Dm->Ny;
+        int Nz = LeeModel.Dm->Nz;
+        DoubleArray DensityInit(Nx,Ny,Nz);
+        LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityInit);
+        
+        double MLUPS = LeeModel.Run_TwoFluid(LeeModel.timestepMax);	
+        
+        DoubleArray DensityFinal(Nx,Ny,Nz);
+        LeeModel.ScaLBL_Comm->RegularLayout(LeeModel.Map,LeeModel.Den,DensityFinal);
+        
+        DoubleArray DensityChange(Nx,Ny,Nz);
+        double totalChange=0.0;
+        for (int k=1; k<Nz-1; k++){
+            for (int j=1; j<Ny-1; j++){
+                for (int i=1; i<Nx-1; i++){
+                	double change = DensityFinal(i,j,k)-DensityInit(i,j,k);
+                	DensityChange(i,j,k) = change;
+                	totalChange += change;
+                }
+            }
+        }
+        printf("Density change, %f\n", totalChange);
+        
+    	FILE *OUTFILE;
+        char LocalRankFilename[40];
+    	sprintf(LocalRankFilename,"DensityChange.%05i.raw",rank);
+    	OUTFILE = fopen(LocalRankFilename,"wb");
+    	fwrite(DensityChange.data(),8,Nx*Ny*Nz,OUTFILE);
+    	fclose(OUTFILE);
+   
+        //LeeModel.WriteDebug_TwoFluid();
+
+        PROFILE_STOP("Main");
+        // ****************************************************
+
+
+    } // Limit scope so variables that contain communicators will free before MPI_Finialize
+
+    Utilities::shutdown();
+    return 0;
+}