OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'FOM' of github.com:JamesEMcClure/LBPM-WIA into FOM

Browse Source
This commit is contained in:
James E McClure 2021-03-25 17:31:46 -04:00
commit 1068e19b37
111 changed files with 21174 additions and 6765 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
.clang-format Normal file
View File

@ -0,0 +1,108 @@
# To run clang tools:
# cd to root directory
# To update format only:
# find . -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
# git status -s . | sed s/^...// | grep -E "(\.cpp|\.h|\.cc|\.hpp|\.I)" | xargs -I{} clang-format -i {}
# To run modernize
# export CLANG_PATH=/packages/llvm/build/llvm-60
# export PATH=${CLANG_PATH}/bin:${CLANG_PATH}/share/clang:$PATH
# find src -name "*.cpp" -or -name "*.cc" | xargs -I{} clang-tidy -checks=modernize* -p=/projects/AtomicModel/build/debug -fix {}
# find src -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
---
Language: Cpp
# BasedOnStyle: LLVM
AccessModifierOffset: -4
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: true
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: true
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: true
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
BreakBeforeBinaryOperators: None
#BreakBeforeBraces: Stroustrup
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
ColumnLimit: 100
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: false
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IndentCaseLabels: false
IndentWidth: 4
IndentWrappedFunctionNames: false
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBlockIndentWidth: 4
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: true
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: true
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: true
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 4
UseTab: Never
...

2
CMakeLists.txt
View File

@ -150,7 +150,7 @@ IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" FALSE )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )

32
IO/IOHelpers.h
View File

@ -10,9 +10,9 @@ namespace IO {
// Find a character in a line
inline size_t find( const char *line, char token )
{
size_t i=0;
size_t i = 0;
while ( 1 ) {
if ( line[i]==token || line[i]<32 || line[i]==0 )
if ( line[i] == token || line[i] < 32 || line[i] == 0 )
break;
++i;
}
@ -21,17 +21,17 @@ inline size_t find( const char *line, char token )
// Remove preceeding/trailing whitespace
inline std::string deblank( const std::string& str )
inline std::string deblank( const std::string &str )
{
size_t i1 = str.size();
size_t i2 = 0;
for (size_t i=0; i<str.size(); i++) {
if ( str[i]!=' ' && str[i]>=32 ) {
i1 = std::min(i1,i);
i2 = std::max(i2,i);
for ( size_t i = 0; i < str.size(); i++ ) {
if ( str[i] != ' ' && str[i] >= 32 ) {
i1 = std::min( i1, i );
i2 = std::max( i2, i );
}
}
return str.substr(i1,i2-i1+1);
return str.substr( i1, i2 - i1 + 1 );
}
@ -42,14 +42,14 @@ inline std::vector<std::string> splitList( const char *line, const char token )
size_t i1 = 0;
size_t i2 = 0;
while ( 1 ) {
if ( line[i2]==token || line[i2]<32 ) {
std::string tmp(&line[i1],i2-i1);
tmp = deblank(tmp);
if ( line[i2] == token || line[i2] < 32 ) {
std::string tmp( &line[i1], i2 - i1 );
tmp = deblank( tmp );
if ( !tmp.empty() )
list.push_back(tmp);
i1 = i2+1;
list.push_back( tmp );
i1 = i2 + 1;
}
if ( line[i2]==0 )
if ( line[i2] == 0 )
break;
i2++;
}
@ -57,8 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
}
};
}; // namespace IO
#endif

629
IO/Mesh.cpp
View File

@ -1,4 +1,5 @@
#include "Mesh.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#include <limits>
@ -19,104 +20,110 @@ inline Point nullPoint()
/****************************************************
* Mesh *
****************************************************/
Mesh::Mesh( )
{
}
Mesh::~Mesh( )
{
}
* Mesh *
****************************************************/
Mesh::Mesh() {}
Mesh::~Mesh() {}
/****************************************************
* MeshDataStruct *
****************************************************/
bool MeshDataStruct::check() const
* MeshDataStruct *
****************************************************/
#define checkResult( pass, msg ) \
do { \
if ( !( pass ) ) { \
if ( abort ) \
ERROR( msg ); \
return false; \
} \
} while ( 0 )
bool MeshDataStruct::check( bool abort ) const
{
enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
bool pass = mesh != nullptr;
for ( const auto& var : vars ) {
pass = pass && static_cast<int>(var->type)>=1 && static_cast<int>(var->type)<=3;
pass = pass && !var->data.empty();
for ( const auto &var : vars ) {
checkResult( var->type == VariableType::NodeVariable ||
var->type == VariableType::EdgeVariable ||
var->type == VariableType::SurfaceVariable ||
var->type == VariableType::VolumeVariable,
"Invalid data type" );
checkResult( !var->data.empty(), "Variable data is empty" );
}
if ( !pass )
return false;
const std::string& meshClass = mesh->className();
const std::string &meshClass = mesh->className();
if ( meshClass == "PointList" ) {
const auto mesh2 = dynamic_cast<IO::PointList*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = dynamic_cast<IO::PointList *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
}
} else if ( meshClass == "TriMesh" || meshClass == "TriList" ) {
const auto mesh2 = getTriMesh( mesh );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = getTriMesh( mesh );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->vertices->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim;
checkResult( var->data.size( 0 ) == mesh2->A.size(), "sizeof VolumeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof VolumeVariable" );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
}
} else if ( meshClass == "DomainMesh" ) {
const auto mesh2 = dynamic_cast<IO::DomainMesh*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = dynamic_cast<IO::DomainMesh *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
ArraySize varSize;
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1)
&& (int) var->data.size(2)==(mesh2->nz+1) && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx + 1, mesh2->ny + 1, mesh2->nz + 1, var->dim );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny
&& (int) var->data.size(2)==mesh2->nz && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx, mesh2->ny, mesh2->nz, var->dim );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
if ( var->data.size( 0 ) == varSize[0] * varSize[1] * varSize[2] &&
var->data.size( 1 ) == varSize[3] )
var->data.resize( varSize );
for ( int d = 0; d < 4; d++ )
checkResult( var->data.size( d ) == varSize[d], "DomainMesh Variable" );
}
} else {
ERROR("Unknown mesh class: "+mesh->className());
ERROR( "Unknown mesh class: " + mesh->className() );
}
return pass;
return true;
}
/****************************************************
* PointList *
****************************************************/
PointList::PointList( )
{
}
* PointList *
****************************************************/
PointList::PointList() {}
PointList::PointList( size_t N )
{
Point tmp = nullPoint();
points.resize(N,tmp);
}
PointList::~PointList( )
{
points.resize( N, tmp );
}
PointList::~PointList() {}
size_t PointList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
@ -124,174 +131,168 @@ size_t PointList::numberPointsVar( VariableType type ) const
N = points.size();
return N;
}
std::pair<size_t,void*> PointList::pack( int level ) const
std::pair<size_t, void *> PointList::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
data_out.first = (2+3*points.size())*sizeof(double);
double *data_ptr = new double[2+3*points.size()];
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
data_out.first = ( 2 + 3 * points.size() ) * sizeof( double );
double *data_ptr = new double[2 + 3 * points.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
data_int[0] = level;
data_int[1] = points.size();
double *data = &data_ptr[2];
for (size_t i=0; i<points.size(); i++) {
data[3*i+0] = points[i].x;
data[3*i+1] = points[i].y;
data[3*i+2] = points[i].z;
for ( size_t i = 0; i < points.size(); i++ ) {
data[3 * i + 0] = points[i].x;
data[3 * i + 1] = points[i].y;
data[3 * i + 2] = points[i].z;
}
}
return data_out;
}
void PointList::unpack( const std::pair<size_t,void*>& data_in )
void PointList::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level==0 ) {
ASSERT((2+3*N)*sizeof(double)==data_in.first);
points.resize(N);
for (size_t i=0; i<points.size(); i++) {
points[i].x = data[3*i+0];
points[i].y = data[3*i+1];
points[i].z = data[3*i+2];
if ( level == 0 ) {
ASSERT( ( 2 + 3 * N ) * sizeof( double ) == data_in.first );
points.resize( N );
for ( size_t i = 0; i < points.size(); i++ ) {
points[i].x = data[3 * i + 0];
points[i].y = data[3 * i + 1];
points[i].z = data[3 * i + 2];
}
}
}
/****************************************************
* TriList *
****************************************************/
TriList::TriList( )
{
}
* TriList *
****************************************************/
TriList::TriList() {}
TriList::TriList( size_t N_tri )
{
Point tmp = nullPoint();
A.resize(N_tri,tmp);
B.resize(N_tri,tmp);
C.resize(N_tri,tmp);
A.resize( N_tri, tmp );
B.resize( N_tri, tmp );
C.resize( N_tri, tmp );
}
TriList::TriList( const TriMesh& mesh )
TriList::TriList( const TriMesh &mesh )
{
Point tmp = nullPoint();
A.resize(mesh.A.size(),tmp);
B.resize(mesh.B.size(),tmp);
C.resize(mesh.C.size(),tmp);
ASSERT(mesh.vertices.get()!=NULL);
const std::vector<Point>& P = mesh.vertices->points;
for (size_t i=0; i<A.size(); i++)
A.resize( mesh.A.size(), tmp );
B.resize( mesh.B.size(), tmp );
C.resize( mesh.C.size(), tmp );
ASSERT( mesh.vertices.get() != NULL );
const std::vector<Point> &P = mesh.vertices->points;
for ( size_t i = 0; i < A.size(); i++ )
A[i] = P[mesh.A[i]];
for (size_t i=0; i<B.size(); i++)
for ( size_t i = 0; i < B.size(); i++ )
B[i] = P[mesh.B[i]];
for (size_t i=0; i<C.size(); i++)
for ( size_t i = 0; i < C.size(); i++ )
C[i] = P[mesh.C[i]];
}
TriList::~TriList( )
{
}
TriList::~TriList() {}
size_t TriList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
N = 3*A.size();
else if ( type==VariableType::SurfaceVariable || type==VariableType::VolumeVariable )
if ( type == VariableType::NodeVariable )
N = 3 * A.size();
else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
N = A.size();
return N;
}
std::pair<size_t,void*> TriList::pack( int level ) const
std::pair<size_t, void *> TriList::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
data_out.first = (2+9*A.size())*sizeof(double);
double *data_ptr = new double[2+9*A.size()];
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
data_out.first = ( 2 + 9 * A.size() ) * sizeof( double );
double *data_ptr = new double[2 + 9 * A.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
data_int[0] = level;
data_int[1] = A.size();
double *data = &data_ptr[2];
for (size_t i=0; i<A.size(); i++) {
data[9*i+0] = A[i].x;
data[9*i+1] = A[i].y;
data[9*i+2] = A[i].z;
data[9*i+3] = B[i].x;
data[9*i+4] = B[i].y;
data[9*i+5] = B[i].z;
data[9*i+6] = C[i].x;
data[9*i+7] = C[i].y;
data[9*i+8] = C[i].z;
for ( size_t i = 0; i < A.size(); i++ ) {
data[9 * i + 0] = A[i].x;
data[9 * i + 1] = A[i].y;
data[9 * i + 2] = A[i].z;
data[9 * i + 3] = B[i].x;
data[9 * i + 4] = B[i].y;
data[9 * i + 5] = B[i].z;
data[9 * i + 6] = C[i].x;
data[9 * i + 7] = C[i].y;
data[9 * i + 8] = C[i].z;
}
}
return data_out;
}
void TriList::unpack( const std::pair<size_t,void*>& data_in )
void TriList::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level==0 ) {
ASSERT((2+9*N)*sizeof(double)==data_in.first);
A.resize(N);
B.resize(N);
C.resize(N);
for (size_t i=0; i<A.size(); i++) {
A[i].x = data[9*i+0];
A[i].y = data[9*i+1];
A[i].z = data[9*i+2];
B[i].x = data[9*i+3];
B[i].y = data[9*i+4];
B[i].z = data[9*i+5];
C[i].x = data[9*i+6];
C[i].y = data[9*i+7];
C[i].z = data[9*i+8];
if ( level == 0 ) {
ASSERT( ( 2 + 9 * N ) * sizeof( double ) == data_in.first );
A.resize( N );
B.resize( N );
C.resize( N );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i].x = data[9 * i + 0];
A[i].y = data[9 * i + 1];
A[i].z = data[9 * i + 2];
B[i].x = data[9 * i + 3];
B[i].y = data[9 * i + 4];
B[i].z = data[9 * i + 5];
C[i].x = data[9 * i + 6];
C[i].y = data[9 * i + 7];
C[i].z = data[9 * i + 8];
}
}
}
/****************************************************
* TriMesh *
****************************************************/
TriMesh::TriMesh( )
{
}
* TriMesh *
****************************************************/
TriMesh::TriMesh() {}
TriMesh::TriMesh( size_t N_tri, size_t N_point )
{
vertices.reset( new PointList(N_point) );
A.resize(N_tri,-1);
B.resize(N_tri,-1);
C.resize(N_tri,-1);
vertices.reset( new PointList( N_point ) );
A.resize( N_tri, -1 );
B.resize( N_tri, -1 );
C.resize( N_tri, -1 );
}
TriMesh::TriMesh( size_t N_tri, std::shared_ptr<PointList> points )
{
vertices = points;
A.resize(N_tri,-1);
B.resize(N_tri,-1);
C.resize(N_tri,-1);
A.resize( N_tri, -1 );
B.resize( N_tri, -1 );
C.resize( N_tri, -1 );
}
TriMesh::TriMesh( const TriList& mesh )
TriMesh::TriMesh( const TriList &mesh )
{
// For simlicity we will just create a mesh with ~3x the verticies for now
ASSERT(mesh.A.size()==mesh.B.size()&&mesh.A.size()==mesh.C.size());
A.resize(mesh.A.size());
B.resize(mesh.B.size());
C.resize(mesh.C.size());
vertices.reset( new PointList(3*mesh.A.size()) );
for (size_t i=0; i<A.size(); i++) {
A[i] = 3*i+0;
B[i] = 3*i+1;
C[i] = 3*i+2;
ASSERT( mesh.A.size() == mesh.B.size() && mesh.A.size() == mesh.C.size() );
A.resize( mesh.A.size() );
B.resize( mesh.B.size() );
C.resize( mesh.C.size() );
vertices.reset( new PointList( 3 * mesh.A.size() ) );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i] = 3 * i + 0;
B[i] = 3 * i + 1;
C[i] = 3 * i + 2;
vertices->points[A[i]] = mesh.A[i];
vertices->points[B[i]] = mesh.B[i];
vertices->points[C[i]] = mesh.C[i];
}
}
TriMesh::~TriMesh( )
TriMesh::~TriMesh()
{
vertices.reset();
A.clear();
@ -301,108 +302,122 @@ TriMesh::~TriMesh( )
size_t TriMesh::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
if ( type == VariableType::NodeVariable )
N = vertices->points.size();
else if ( type==VariableType::SurfaceVariable || type==VariableType::VolumeVariable )
else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
N = A.size();
return N;
}
std::pair<size_t,void*> TriMesh::pack( int level ) const
std::pair<size_t, void *> TriMesh::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
const std::vector<Point>& points = vertices->points;
data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int);
double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)];
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
const std::vector<Point> &points = vertices->points;
data_out.first =
( 3 + 3 * points.size() ) * sizeof( double ) + 3 * A.size() * sizeof( int );
double *data_ptr =
new double[4 + 3 * points.size() + ( 3 * A.size() * sizeof( int ) ) / sizeof( double )];
data_out.second = data_ptr;
uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_ptr);
uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_ptr );
data_int64[0] = level;
data_int64[1] = points.size();
data_int64[2] = A.size();
double *data = &data_ptr[3];
for (size_t i=0; i<points.size(); i++) {
data[3*i+0] = points[i].x;
data[3*i+1] = points[i].y;
data[3*i+2] = points[i].z;
for ( size_t i = 0; i < points.size(); i++ ) {
data[3 * i + 0] = points[i].x;
data[3 * i + 1] = points[i].y;
data[3 * i + 2] = points[i].z;
}
int *data_int = reinterpret_cast<int*>(&data[3*points.size()]);
for (size_t i=0; i<A.size(); i++) {
data_int[3*i+0] = A[i];
data_int[3*i+1] = B[i];
data_int[3*i+2] = C[i];
int *data_int = reinterpret_cast<int *>( &data[3 * points.size()] );
for ( size_t i = 0; i < A.size(); i++ ) {
data_int[3 * i + 0] = A[i];
data_int[3 * i + 1] = B[i];
data_int[3 * i + 2] = C[i];
}
}
return data_out;
}
void TriMesh::unpack( const std::pair<size_t,void*>& data_in )
void TriMesh::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int64[0];
uint64_t N_P = data_int64[1];
uint64_t N_A = data_int64[2];
data = &data[3];
if ( level==0 ) {
size_t size = (3+3*N_P)*sizeof(double)+3*N_A*sizeof(int);
ASSERT(size==data_in.first);
vertices.reset( new PointList(N_P) );
std::vector<Point>& points = vertices->points;
for (size_t i=0; i<points.size(); i++) {
points[i].x = data[3*i+0];
points[i].y = data[3*i+1];
points[i].z = data[3*i+2];
if ( level == 0 ) {
size_t size = ( 3 + 3 * N_P ) * sizeof( double ) + 3 * N_A * sizeof( int );
ASSERT( size == data_in.first );
vertices.reset( new PointList( N_P ) );
std::vector<Point> &points = vertices->points;
for ( size_t i = 0; i < points.size(); i++ ) {
points[i].x = data[3 * i + 0];
points[i].y = data[3 * i + 1];
points[i].z = data[3 * i + 2];
}
const int *data_int = reinterpret_cast<const int*>(&data[3*N_P]);
A.resize(N_A);
B.resize(N_A);
C.resize(N_A);
for (size_t i=0; i<A.size(); i++) {
A[i] = data_int[3*i+0];
B[i] = data_int[3*i+1];
C[i] = data_int[3*i+2];
const int *data_int = reinterpret_cast<const int *>( &data[3 * N_P] );
A.resize( N_A );
B.resize( N_A );
C.resize( N_A );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i] = data_int[3 * i + 0];
B[i] = data_int[3 * i + 1];
C[i] = data_int[3 * i + 2];
}
}
}
/****************************************************
* Domain mesh *
****************************************************/
DomainMesh::DomainMesh():
nprocx(0), nprocy(0), nprocz(0), rank(0),
nx(0), ny(0), nz(0),
Lx(0), Ly(0), Lz(0)
* Domain mesh *
****************************************************/
DomainMesh::DomainMesh()
: nprocx( 0 ),
nprocy( 0 ),
nprocz( 0 ),
rank( 0 ),
nx( 0 ),
ny( 0 ),
nz( 0 ),
Lx( 0 ),
Ly( 0 ),
Lz( 0 )
{
}
DomainMesh::DomainMesh( RankInfoStruct data,
int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ):
nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]),
nx(nx2), ny(ny2), nz(nz2),
Lx(Lx2), Ly(Ly2), Lz(Lz2)
{
}
DomainMesh::~DomainMesh()
DomainMesh::DomainMesh(
RankInfoStruct data, int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 )
: nprocx( data.nx ),
nprocy( data.ny ),
nprocz( data.nz ),
rank( data.rank[1][1][1] ),
nx( nx2 ),
ny( ny2 ),
nz( nz2 ),
Lx( Lx2 ),
Ly( Ly2 ),
Lz( Lz2 )
{
}
DomainMesh::~DomainMesh() {}
size_t DomainMesh::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
N = (nx+1)*(ny+1)*(nz+1);
else if ( type==VariableType::SurfaceVariable )
N = (nx+1)*ny*nz + nx*(ny+1)*nz + nx*ny*(nz+1);
else if ( type==VariableType::VolumeVariable )
N = nx*ny*nz;
if ( type == VariableType::NodeVariable )
N = ( nx + 1 ) * ( ny + 1 ) * ( nz + 1 );
else if ( type == VariableType::SurfaceVariable )
N = ( nx + 1 ) * ny * nz + nx * ( ny + 1 ) * nz + nx * ny * ( nz + 1 );
else if ( type == VariableType::VolumeVariable )
N = nx * ny * nz;
return N;
}
std::pair<size_t,void*> DomainMesh::pack( int level ) const
std::pair<size_t, void *> DomainMesh::pack( int level ) const
{
std::pair<size_t,void*> data(0,NULL);
data.first = 7*sizeof(double);
std::pair<size_t, void *> data( 0, NULL );
data.first = 7 * sizeof( double );
data.second = new double[7];
memset(data.second,0,7*sizeof(double));
int *data_int = reinterpret_cast<int*>(data.second);
double *data_double = &reinterpret_cast<double*>(data.second)[4];
memset( data.second, 0, 7 * sizeof( double ) );
int *data_int = reinterpret_cast<int *>( data.second );
double *data_double = &reinterpret_cast<double *>( data.second )[4];
data_int[0] = nprocx;
data_int[1] = nprocy;
data_int[2] = nprocz;
@ -415,10 +430,10 @@ std::pair<size_t,void*> DomainMesh::pack( int level ) const
data_double[2] = Lz;
return data;
}
void DomainMesh::unpack( const std::pair<size_t,void*>& data )
void DomainMesh::unpack( const std::pair<size_t, void *> &data )
{
const int *data_int = reinterpret_cast<const int*>(data.second);
const double *data_double = &reinterpret_cast<const double*>(data.second)[4];
const int *data_int = reinterpret_cast<const int *>( data.second );
const double *data_double = &reinterpret_cast<const double *>( data.second )[4];
nprocx = data_int[0];
nprocy = data_int[1];
nprocz = data_int[2];
@ -433,49 +448,173 @@ void DomainMesh::unpack( const std::pair<size_t,void*>& data )
/****************************************************
* Converters *
****************************************************/
* Converters *
****************************************************/
std::shared_ptr<PointList> getPointList( std::shared_ptr<Mesh> mesh )
{
return std::dynamic_pointer_cast<PointList>(mesh);
return std::dynamic_pointer_cast<PointList>( mesh );
}
std::shared_ptr<TriMesh> getTriMesh( std::shared_ptr<Mesh> mesh )
{
std::shared_ptr<TriMesh> mesh2;
if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriMesh>(mesh);
} else if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>(mesh);
ASSERT(trilist.get()!=NULL);
mesh2.reset( new TriMesh(*trilist) );
if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriMesh>( mesh );
} else if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>( mesh );
ASSERT( trilist.get() != NULL );
mesh2.reset( new TriMesh( *trilist ) );
}
return mesh2;
}
std::shared_ptr<TriList> getTriList( std::shared_ptr<Mesh> mesh )
{
std::shared_ptr<TriList> mesh2;
if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriList>(mesh);
} else if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>(mesh);
ASSERT(trimesh.get()!=NULL);
mesh2.reset( new TriList(*trimesh) );
if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriList>( mesh );
} else if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>( mesh );
ASSERT( trimesh.get() != NULL );
mesh2.reset( new TriList( *trimesh ) );
}
return mesh2;
}
std::shared_ptr<const PointList> getPointList( std::shared_ptr<const Mesh> mesh )
{
return getPointList( std::const_pointer_cast<Mesh>(mesh) );
return getPointList( std::const_pointer_cast<Mesh>( mesh ) );
}
std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh )
{
return getTriMesh( std::const_pointer_cast<Mesh>(mesh) );
return getTriMesh( std::const_pointer_cast<Mesh>( mesh ) );
}
std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh )
{
return getTriList( std::const_pointer_cast<Mesh>(mesh) );
return getTriList( std::const_pointer_cast<Mesh>( mesh ) );
}
} // IO namespace
/****************************************************
* Convert enum values *
****************************************************/
std::string getString( VariableType type )
{
if ( type == VariableType::NodeVariable )
return "node";
else if ( type == VariableType::EdgeVariable )
return "edge";
else if ( type == VariableType::SurfaceVariable )
return "face";
else if ( type == VariableType::VolumeVariable )
return "cell";
else if ( type == VariableType::NullVariable )
return "null";
else
ERROR( "Invalid type" );
return "";
}
VariableType getVariableType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "node" )
return VariableType::NodeVariable;
else if ( type == "edge" || type == "1" )
return VariableType::EdgeVariable;
else if ( type == "face" )
return VariableType::SurfaceVariable;
else if ( type == "cell" || type == "3" )
return VariableType::VolumeVariable;
else if ( type == "null" )
return VariableType::NullVariable;
else
ERROR( "Invalid type: " + type );
return VariableType::NullVariable;
}
std::string getString( DataType type )
{
if ( type == DataType::Double )
return "double";
else if ( type == DataType::Float )
return "float";
else if ( type == DataType::Int )
return "int";
else if ( type == DataType::Null )
return "null";
else
ERROR( "Invalid type" );
return "";
}
DataType getDataType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "double" )
return DataType::Double;
else if ( type == "float" )
return DataType::Float;
else if ( type == "int" )
return DataType::Int;
else if ( type == "null" )
return DataType::Null;
else
ERROR( "Invalid type: " + type );
return DataType::Null;
}
std::string getString( MeshType type )
{
if ( type == MeshType::PointMesh )
return "PointMesh";
else if ( type == MeshType::SurfaceMesh )
return "SurfaceMesh";
else if ( type == MeshType::VolumeMesh )
return "VolumeMesh";
else if ( type == MeshType::Unknown )
return "unknown";
else
ERROR( "Invalid type" );
return "";
}
MeshType getMeshType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "PointMesh" || type == "1" )
return MeshType::PointMesh;
else if ( type == "SurfaceMesh" || type == "2" )
return MeshType::SurfaceMesh;
else if ( type == "VolumeMesh" || type == "3" )
return MeshType::VolumeMesh;
else if ( type == "unknown" || type == "-1" )
return MeshType::Unknown;
else
ERROR( "Invalid type: " + type );
return MeshType::Unknown;
}
std::string getString( FileFormat type )
{
if ( type == FileFormat::OLD )
return "old";
else if ( type == FileFormat::NEW )
return "new";
else if ( type == FileFormat::NEW_SINGLE )
return "new(single)";
else if ( type == FileFormat::SILO )
return "silo";
else
ERROR( "Invalid type" );
return "";
}
FileFormat getFileFormat( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "old" || type == "1" )
return FileFormat::OLD;
else if ( type == "new" || type == "2" )
return FileFormat::NEW;
else if ( type == "new(single)" || type == "3" )
return FileFormat::NEW_SINGLE;
else if ( type == "silo" || type == "4" )
return FileFormat::SILO;
else
ERROR( "Invalid type: " + type );
return FileFormat::SILO;
}
} // namespace IO

107
IO/Mesh.h
View File

@ -6,17 +6,36 @@
#include <string.h>
#include <vector>
#include "analysis/PointList.h"
#include "common/Array.h"
#include "common/Communication.h"
#include "analysis/PointList.h"
namespace IO {
//! Possible variable types
enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 };
//! Enums to define types
enum class VariableType {
NodeVariable,
EdgeVariable,
SurfaceVariable,
VolumeVariable,
NullVariable
};
enum class DataType { Double, Float, Int, Null };
enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown };
enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO };
//! Convert enums to/from strings (more future-proof than static_cast<int>)
std::string getString( VariableType );
std::string getString( DataType );
std::string getString( MeshType );
std::string getString( FileFormat );
VariableType getVariableType( const std::string & );
DataType getDataType( const std::string & );
MeshType getMeshType( const std::string & );
FileFormat getFileFormat( const std::string & );
/*! \class Mesh
@ -32,21 +51,22 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const = 0;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const = 0;
virtual std::pair<size_t, void *> pack( int level ) const = 0;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data ) = 0;
virtual void unpack( const std::pair<size_t, void *> &data ) = 0;
protected:
//! Empty constructor
Mesh();
Mesh(const Mesh&);
Mesh& operator=(const Mesh&);
Mesh( const Mesh & );
Mesh &operator=( const Mesh & );
};
/*! \class PointList
\brief A class used to hold a list of verticies
*/
class PointList: public Mesh
class PointList : public Mesh
{
public:
//! Empty constructor
@ -60,11 +80,12 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
//! Access the points
const std::vector<Point>& getPoints() const { return points; }
const std::vector<Point> &getPoints() const { return points; }
public:
std::vector<Point> points; //!< List of points vertex
};
@ -74,7 +95,7 @@ public:
\brief A class used to hold a list of triangles specified by their vertex coordinates
*/
class TriMesh;
class TriList: public Mesh
class TriList : public Mesh
{
public:
//! Empty constructor
@ -82,7 +103,7 @@ public:
//! Constructor for N triangles
TriList( size_t N_tri );
//! Constructor from TriMesh
TriList( const TriMesh& );
TriList( const TriMesh & );
//! Destructor
virtual ~TriList();
//! Mesh class name
@ -90,9 +111,10 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::vector<Point> A; //!< First vertex
std::vector<Point> B; //!< Second vertex
@ -101,9 +123,10 @@ public:
/*! \class TriMesh
\brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes
\brief A class used to hold a list of trianges specified by their vertex number and list of
coordiantes
*/
class TriMesh: public Mesh
class TriMesh : public Mesh
{
public:
//! TriMesh constructor
@ -113,7 +136,7 @@ public:
//! Constructor for Nt triangles and the given points
TriMesh( size_t N_tri, std::shared_ptr<PointList> points );
//! Constructor from TriList
TriMesh( const TriList& );
TriMesh( const TriList & );
//! Destructor
virtual ~TriMesh();
//! Mesh class name
@ -121,9 +144,10 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::shared_ptr<PointList> vertices; //!< List of verticies
std::vector<int> A; //!< First vertex
@ -135,7 +159,7 @@ public:
/*! \class Domain
\brief A class used to hold the domain
*/
class DomainMesh: public Mesh
class DomainMesh : public Mesh
{
public:
//! Empty constructor
@ -149,9 +173,10 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
int nprocx, nprocy, nprocz, rank;
int nx, ny, nz;
@ -159,12 +184,10 @@ public:
};
/*! \class Variable
\brief A base class for variables
*/
struct Variable
{
struct Variable {
public:
// Internal variables
unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
@ -173,23 +196,28 @@ public:
std::string name; //!< Variable name
Array<double> data; //!< Variable data
//! Empty constructor
Variable(): dim(0), type(VariableType::NullVariable), precision(DataType::Double) {}
Variable() : dim( 0 ), type( VariableType::NullVariable ), precision( DataType::Double ) {}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_) {}
Variable( int dim_, IO::VariableType type_, const std::string &name_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ )
{
}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array<double>& data_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {}
Variable(
int dim_, IO::VariableType type_, const std::string &name_, const Array<double> &data_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ), data( data_ )
{
}
//! Destructor
virtual ~Variable() {}
protected:
//! Empty constructor
Variable(const Variable&);
Variable& operator=(const Variable&);
Variable( const Variable & );
Variable &operator=( const Variable & );
};
/*! \class MeshDataStruct
\brief A class used to hold database info for saving a mesh
*/
@ -197,11 +225,11 @@ struct MeshDataStruct {
DataType precision; //!< Precision to use for IO (mesh)
std::string meshName; //!< Mesh name
std::shared_ptr<Mesh> mesh; //!< Mesh data
std::vector<std::shared_ptr<Variable> > vars;
std::vector<std::shared_ptr<Variable>> vars;
//! Empty constructor
MeshDataStruct(): precision(DataType::Double) {}
MeshDataStruct() : precision( DataType::Double ) {}
//! Check the data
bool check() const;
bool check( bool abort = true ) const;
};
@ -214,7 +242,6 @@ std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh );
std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh );
} // IO namespace
} // namespace IO
#endif

495
IO/MeshDatabase.cpp
View File

@ -1,130 +1,144 @@
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/Mesh.h"
#include "IO/PackData.h"
#include "IO/IOHelpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <vector>
#include <cstdio>
#include <map>
#include <set>
#include <cstdio>
#include <vector>
#include <ProfilerApp.h>
// Default pack/unpack
// clang-format off
#define INSTANTIATE_PACK( TYPE ) \
template<> \
size_t packsize<TYPE>( const TYPE &rhs ) \
{ \
return sizeof( TYPE ); \
} \
template<> \
void pack<TYPE>( const TYPE &rhs, char *buffer ) \
{ \
memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); \
} \
template<> \
void unpack<TYPE>( TYPE &data, const char *buffer ) \
{ \
memcpy( &data, buffer, sizeof( IO::MeshType ) ); \
}
INSTANTIATE_PACK( IO::VariableType )
INSTANTIATE_PACK( IO::DataType )
INSTANTIATE_PACK( IO::MeshType )
INSTANTIATE_PACK( IO::FileFormat )
// clang-format on
// MeshType
template<>
size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
{
return sizeof(IO::MeshType);
}
template<>
void pack<IO::MeshType>( const IO::MeshType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::MeshType));
}
template<>
void unpack<IO::MeshType>( IO::MeshType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::MeshType));
}
// Variable::VariableType
template<>
size_t packsize<IO::VariableType>( const IO::VariableType& rhs )
{
return sizeof(IO::VariableType);
}
template<>
void pack<IO::VariableType>( const IO::VariableType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::VariableType));
}
template<>
void unpack<IO::VariableType>( IO::VariableType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::VariableType));
}
// DatabaseEntry
template<>
size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs )
size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs )
{
return packsize(rhs.name)+packsize(rhs.file)+packsize(rhs.offset);
return packsize( rhs.name ) + packsize( rhs.file ) + packsize( rhs.offset );
}
template<>
void pack<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs, char *buffer )
void pack<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs, char *buffer )
{
size_t i=0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.file,&buffer[i]); i+=packsize(rhs.file);
pack(rhs.offset,&buffer[i]); i+=packsize(rhs.offset);
size_t i = 0;
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.file, &buffer[i] );
i += packsize( rhs.file );
pack( rhs.offset, &buffer[i] );
i += packsize( rhs.offset );
}
template<>
void unpack<IO::DatabaseEntry>( IO::DatabaseEntry& data, const char *buffer )
void unpack<IO::DatabaseEntry>( IO::DatabaseEntry &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.file,&buffer[i]); i+=packsize(data.file);
unpack(data.offset,&buffer[i]); i+=packsize(data.offset);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.file, &buffer[i] );
i += packsize( data.file );
unpack( data.offset, &buffer[i] );
i += packsize( data.offset );
}
// VariableDatabase
template<>
size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase& rhs )
size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase &rhs )
{
return packsize(rhs.name)+packsize(rhs.type)+packsize(rhs.dim);
return packsize( rhs.name ) + packsize( rhs.type ) + packsize( rhs.dim );
}
template<>
void pack<IO::VariableDatabase>( const IO::VariableDatabase& rhs, char *buffer )
void pack<IO::VariableDatabase>( const IO::VariableDatabase &rhs, char *buffer )
{
size_t i=0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.dim,&buffer[i]); i+=packsize(rhs.dim);
size_t i = 0;
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.dim, &buffer[i] );
i += packsize( rhs.dim );
}
template<>
void unpack<IO::VariableDatabase>( IO::VariableDatabase& data, const char *buffer )
void unpack<IO::VariableDatabase>( IO::VariableDatabase &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.dim,&buffer[i]); i+=packsize(data.dim);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.dim, &buffer[i] );
i += packsize( data.dim );
}
// MeshDatabase
template<>
size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase& data )
size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase &data )
{
return packsize(data.name)
+ packsize(data.type)
+ packsize(data.meshClass)
+ packsize(data.format)
+ packsize(data.domains)
+ packsize(data.variables)
+ packsize(data.variable_data);
return packsize( data.name ) + packsize( data.type ) + packsize( data.meshClass ) +
packsize( data.format ) + packsize( data.domains ) + packsize( data.variables ) +
packsize( data.variable_data );
}
template<>
void pack<IO::MeshDatabase>( const IO::MeshDatabase& rhs, char *buffer )
void pack<IO::MeshDatabase>( const IO::MeshDatabase &rhs, char *buffer )
{
size_t i = 0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.meshClass,&buffer[i]); i+=packsize(rhs.meshClass);
pack(rhs.format,&buffer[i]); i+=packsize(rhs.format);
pack(rhs.domains,&buffer[i]); i+=packsize(rhs.domains);
pack(rhs.variables,&buffer[i]); i+=packsize(rhs.variables);
pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data);
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.meshClass, &buffer[i] );
i += packsize( rhs.meshClass );
pack( rhs.format, &buffer[i] );
i += packsize( rhs.format );
pack( rhs.domains, &buffer[i] );
i += packsize( rhs.domains );
pack( rhs.variables, &buffer[i] );
i += packsize( rhs.variables );
pack( rhs.variable_data, &buffer[i] );
i += packsize( rhs.variable_data );
}
template<>
void unpack<IO::MeshDatabase>( IO::MeshDatabase& data, const char *buffer )
void unpack<IO::MeshDatabase>( IO::MeshDatabase &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.meshClass,&buffer[i]); i+=packsize(data.meshClass);
unpack(data.format,&buffer[i]); i+=packsize(data.format);
unpack(data.domains,&buffer[i]); i+=packsize(data.domains);
unpack(data.variables,&buffer[i]); i+=packsize(data.variables);
unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.meshClass, &buffer[i] );
i += packsize( data.meshClass );
unpack( data.format, &buffer[i] );
i += packsize( data.format );
unpack( data.domains, &buffer[i] );
i += packsize( data.domains );
unpack( data.variables, &buffer[i] );
i += packsize( data.variables );
unpack( data.variable_data, &buffer[i] );
i += packsize( data.variable_data );
}
@ -132,56 +146,49 @@ namespace IO {
/****************************************************
* VariableDatabase *
****************************************************/
bool VariableDatabase::operator==(const VariableDatabase& rhs ) const
* VariableDatabase *
****************************************************/
bool VariableDatabase::operator==( const VariableDatabase &rhs ) const
{
return type==rhs.type && dim==rhs.dim && name==rhs.name;
return type == rhs.type && dim == rhs.dim && name == rhs.name;
}
bool VariableDatabase::operator!=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator!=( const VariableDatabase &rhs ) const
{
return type!=rhs.type || dim!=rhs.dim || name!=rhs.name;
return type != rhs.type || dim != rhs.dim || name != rhs.name;
}
bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator>=( const VariableDatabase &rhs ) const
{
return operator>(rhs) || operator==(rhs);
return operator>( rhs ) || operator==( rhs );
}
bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator<=( const VariableDatabase &rhs ) const { return !operator>( rhs ); }
bool VariableDatabase::operator>( const VariableDatabase &rhs ) const
{
return !operator>(rhs);
}
bool VariableDatabase::operator>(const VariableDatabase& rhs ) const
{
if ( name>rhs.name )
if ( name > rhs.name )
return true;
else if ( name<rhs.name )
else if ( name < rhs.name )
return false;
if ( type>rhs.type )
if ( type > rhs.type )
return true;
else if ( type<rhs.type )
else if ( type < rhs.type )
return false;
if ( dim>rhs.dim )
if ( dim > rhs.dim )
return true;
else if ( dim<rhs.dim )
else if ( dim < rhs.dim )
return false;
return false;
}
bool VariableDatabase::operator<(const VariableDatabase& rhs ) const
bool VariableDatabase::operator<( const VariableDatabase &rhs ) const
{
return !operator>(rhs) && operator!=(rhs);
return !operator>( rhs ) && operator!=( rhs );
}
/****************************************************
* MeshDatabase *
****************************************************/
MeshDatabase::MeshDatabase()
{
}
MeshDatabase::~MeshDatabase()
{
}
MeshDatabase::MeshDatabase(const MeshDatabase& rhs)
* MeshDatabase *
****************************************************/
MeshDatabase::MeshDatabase() {}
MeshDatabase::~MeshDatabase() {}
MeshDatabase::MeshDatabase( const MeshDatabase &rhs )
{
name = rhs.name;
type = rhs.type;
@ -191,7 +198,7 @@ MeshDatabase::MeshDatabase(const MeshDatabase& rhs)
variables = rhs.variables;
variable_data = rhs.variable_data;
}
MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs)
MeshDatabase &MeshDatabase::operator=( const MeshDatabase &rhs )
{
this->name = rhs.name;
this->type = rhs.type;
@ -202,9 +209,9 @@ MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs)
this->variable_data = rhs.variable_data;
return *this;
}
VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname ) const
VariableDatabase MeshDatabase::getVariableDatabase( const std::string &varname ) const
{
for (size_t i=0; i<variables.size(); i++) {
for ( size_t i = 0; i < variables.size(); i++ ) {
if ( variables[i].name == varname )
return variables[i];
}
@ -213,217 +220,219 @@ VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname )
/****************************************************
* DatabaseEntry *
****************************************************/
std::string DatabaseEntry::write( ) const
* DatabaseEntry *
****************************************************/
std::string DatabaseEntry::write() const
{
char tmp[1000];
sprintf(tmp,"%s; %s; %lu",name.c_str(),file.c_str(),offset);
return std::string(tmp);
sprintf( tmp, "%s; %s; %lu", name.c_str(), file.c_str(), offset );
return std::string( tmp );
}
DatabaseEntry::DatabaseEntry( const char* line )
DatabaseEntry::DatabaseEntry( const char *line )
{
std::vector<std::string> list = splitList(line,';');
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const char* line )
void DatabaseEntry::read( const char *line )
{
std::vector<std::string> list = splitList(line,';');
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const std::string& line )
void DatabaseEntry::read( const std::string &line )
{
std::vector<std::string> list = splitList(line.c_str(),';');
auto list = splitList( line.c_str(), ';' );
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
offset = atol( list[2].c_str() );
}
// Gather the mesh databases from all processors
inline int tod( int N ) { return (N+7)/sizeof(double); }
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm )
inline int tod( int N ) { return ( N + 7 ) / sizeof( double ); }
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm )
{
if ( comm.getSize() == 1 )
return meshes;
PROFILE_START("gatherAll");
PROFILE_START("gatherAll-pack",2);
PROFILE_START( "gatherAll" );
PROFILE_START( "gatherAll-pack", 2 );
int size = comm.getSize();
// First pack the mesh data to local buffers
int localsize = 0;
for (size_t i=0; i<meshes.size(); i++)
localsize += tod(packsize(meshes[i]));
for ( size_t i = 0; i < meshes.size(); i++ )
localsize += tod( packsize( meshes[i] ) );
auto localbuf = new double[localsize];
int pos = 0;
for (size_t i=0; i<meshes.size(); i++) {
pack( meshes[i], (char*) &localbuf[pos] );
pos += tod(packsize(meshes[i]));
for ( size_t i = 0; i < meshes.size(); i++ ) {
pack( meshes[i], (char *) &localbuf[pos] );
pos += tod( packsize( meshes[i] ) );
}
PROFILE_STOP("gatherAll-pack",2);
PROFILE_STOP( "gatherAll-pack", 2 );
// Get the number of bytes each processor will be sending/recieving
PROFILE_START("gatherAll-send1",2);
PROFILE_START( "gatherAll-send1", 2 );
auto recvsize = comm.allGather( localsize );
int globalsize = recvsize[0];
auto disp = new int[size];
disp[0] = 0;
for (int i=1; i<size; i++) {
disp[i] = disp[i-1] + recvsize[i];
for ( int i = 1; i < size; i++ ) {
disp[i] = disp[i - 1] + recvsize[i];
globalsize += recvsize[i];
}
PROFILE_STOP("gatherAll-send1",2);
PROFILE_STOP( "gatherAll-send1", 2 );
// Send/recv the global data
PROFILE_START("gatherAll-send2",2);
PROFILE_START( "gatherAll-send2", 2 );
auto globalbuf = new double[globalsize];
comm.allGather(localbuf,localsize,globalbuf,recvsize.data(),disp,true);
PROFILE_STOP("gatherAll-send2",2);
comm.allGather( localbuf, localsize, globalbuf, recvsize.data(), disp, true );
PROFILE_STOP( "gatherAll-send2", 2 );
// Unpack the data
PROFILE_START("gatherAll-unpack",2);
std::map<std::string,MeshDatabase> data;
PROFILE_START( "gatherAll-unpack", 2 );
std::map<std::string, MeshDatabase> data;
pos = 0;
while ( pos < globalsize ) {
MeshDatabase tmp;
unpack(tmp,(char*)&globalbuf[pos]);
pos += tod(packsize(tmp));
std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
if ( it==data.end() ) {
unpack( tmp, (char *) &globalbuf[pos] );
pos += tod( packsize( tmp ) );
std::map<std::string, MeshDatabase>::iterator it = data.find( tmp.name );
if ( it == data.end() ) {
data[tmp.name] = tmp;
} else {
for (size_t i=0; i<tmp.domains.size(); i++)
it->second.domains.push_back(tmp.domains[i]);
for (size_t i=0; i<tmp.variables.size(); i++)
it->second.variables.push_back(tmp.variables[i]);
it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
for ( size_t i = 0; i < tmp.domains.size(); i++ )
it->second.domains.push_back( tmp.domains[i] );
for ( size_t i = 0; i < tmp.variables.size(); i++ )
it->second.variables.push_back( tmp.variables[i] );
it->second.variable_data.insert( tmp.variable_data.begin(), tmp.variable_data.end() );
}
}
for (auto it=data.begin(); it!=data.end(); ++it) {
for ( auto it = data.begin(); it != data.end(); ++it ) {
// Get the unique variables
std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
std::set<VariableDatabase> data2(
it->second.variables.begin(), it->second.variables.end() );
it->second.variables = std::vector<VariableDatabase>( data2.begin(), data2.end() );
}
// Free temporary memory
delete [] localbuf;
delete [] disp;
delete [] globalbuf;
delete[] localbuf;
delete[] disp;
delete[] globalbuf;
// Return the results
std::vector<MeshDatabase> data2(data.size());
size_t i=0;
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
std::vector<MeshDatabase> data2( data.size() );
size_t i = 0;
for ( auto it = data.begin(); it != data.end(); ++it, ++i )
data2[i] = it->second;
PROFILE_STOP("gatherAll-unpack",2);
PROFILE_STOP("gatherAll");
PROFILE_STOP( "gatherAll-unpack", 2 );
PROFILE_STOP( "gatherAll" );
return data2;
}
//! Write the mesh databases to a file
void write( const std::vector<MeshDatabase>& meshes, const std::string& filename )
void write( const std::vector<MeshDatabase> &meshes, const std::string &filename )
{
PROFILE_START("write");
FILE *fid = fopen(filename.c_str(),"wb");
for (size_t i=0; i<meshes.size(); i++) {
fprintf(fid,"%s\n",meshes[i].name.c_str());
fprintf(fid," type: %i\n",static_cast<int>(meshes[i].type));
fprintf(fid," meshClass: %s\n",meshes[i].meshClass.c_str());
fprintf(fid," format: %i\n",static_cast<int>(meshes[i].format));
for (size_t j=0; j<meshes[i].domains.size(); j++)
fprintf(fid," domain: %s\n",meshes[i].domains[j].write().c_str());
fprintf(fid," variables: ");
for (size_t j=0; j<meshes[i].variables.size(); j++) {
const VariableDatabase& var = meshes[i].variables[j];
fprintf(fid,"%s|%i|%i; ",var.name.c_str(),static_cast<int>(var.type),var.dim);
PROFILE_START( "write" );
FILE *fid = fopen( filename.c_str(), "wb" );
for ( size_t i = 0; i < meshes.size(); i++ ) {
fprintf( fid, "%s\n", meshes[i].name.c_str() );
fprintf( fid, " type: %s\n", getString( meshes[i].type ).data() );
fprintf( fid, " meshClass: %s\n", meshes[i].meshClass.c_str() );
fprintf( fid, " format: %s\n", getString( meshes[i].format ).data() );
for ( size_t j = 0; j < meshes[i].domains.size(); j++ )
fprintf( fid, " domain: %s\n", meshes[i].domains[j].write().c_str() );
fprintf( fid, " variables: " );
for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) {
const VariableDatabase &var = meshes[i].variables[j];
fprintf( fid, "%s|%s|%i; ", var.name.data(), getString( var.type ).data(), var.dim );
}
fprintf(fid,"\n");
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) {
const char* domain = it->first.first.c_str();
const char* variable = it->first.second.c_str();
fprintf(fid," variable(%s,%s): %s\n",domain,variable,it->second.write().c_str());
fprintf( fid, "\n" );
for ( auto it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end();
++it ) {
const char *domain = it->first.first.c_str();
const char *variable = it->first.second.c_str();
fprintf(
fid, " variable(%s,%s): %s\n", domain, variable, it->second.write().c_str() );
}
}
fclose(fid);
PROFILE_STOP("write");
fclose( fid );
PROFILE_STOP( "write" );
}
//! Read the mesh databases from a file
std::vector<MeshDatabase> read( const std::string& filename )
std::vector<MeshDatabase> read( const std::string &filename )
{
std::vector<MeshDatabase> meshes;
PROFILE_START("read");
FILE *fid = fopen(filename.c_str(),"rb");
if ( fid==NULL )
ERROR("Error opening file");
PROFILE_START( "read" );
FILE *fid = fopen( filename.c_str(), "rb" );
if ( fid == NULL )
ERROR( "Error opening file" );
char *line = new char[10000];
while ( std::fgets(line,1000,fid) != NULL ) {
if ( line[0]<32 ) {
while ( std::fgets( line, 1000, fid ) != NULL ) {
if ( line[0] < 32 ) {
// Empty line
continue;
} else if ( line[0] != ' ' ) {
meshes.resize(meshes.size()+1);
std::string name(line);
name.resize(name.size()-1);
meshes.resize( meshes.size() + 1 );
std::string name( line );
name.resize( name.size() - 1 );
meshes.back().name = name;
} else if ( strncmp(line," format:",10)==0 ) {
meshes.back().format = static_cast<unsigned char>(atoi(&line[10]));
} else if ( strncmp(line," type:",8)==0 ) {
meshes.back().type = static_cast<MeshType>(atoi(&line[8]));
} else if ( strncmp(line," meshClass:",13)==0 ) {
meshes.back().meshClass = deblank(std::string(&line[13]));
} else if ( strncmp(line," domain:",10)==0 ) {
DatabaseEntry data(&line[10]);
meshes.back().domains.push_back(data);
} else if ( strncmp(line," variables:",13)==0 ) {
MeshDatabase& mesh = meshes.back();
std::vector<std::string> variables = splitList(&line[13],';');
mesh.variables.resize(variables.size());
for (size_t i=0; i<variables.size(); i++) {
std::vector<std::string> tmp = splitList(variables[i].c_str(),'|');
ASSERT(tmp.size()==3);
} else if ( strncmp( line, " format:", 10 ) == 0 ) {
meshes.back().format = getFileFormat( &line[10] );
} else if ( strncmp( line, " type:", 8 ) == 0 ) {
meshes.back().type = getMeshType( &line[8] );
} else if ( strncmp( line, " meshClass:", 13 ) == 0 ) {
meshes.back().meshClass = deblank( std::string( &line[13] ) );
} else if ( strncmp( line, " domain:", 10 ) == 0 ) {
DatabaseEntry data( &line[10] );
meshes.back().domains.push_back( data );
} else if ( strncmp( line, " variables:", 13 ) == 0 ) {
MeshDatabase &mesh = meshes.back();
std::vector<std::string> variables = splitList( &line[13], ';' );
mesh.variables.resize( variables.size() );
for ( size_t i = 0; i < variables.size(); i++ ) {
std::vector<std::string> tmp = splitList( variables[i].c_str(), '|' );
ASSERT( tmp.size() == 3 );
mesh.variables[i].name = tmp[0];
mesh.variables[i].type = static_cast<VariableType>(atoi(tmp[1].c_str()));
mesh.variables[i].dim = atoi(tmp[2].c_str());
mesh.variables[i].type = getVariableType( tmp[1] );
mesh.variables[i].dim = atoi( tmp[2].c_str() );
}
} else if ( strncmp(line," variable(",12)==0 ) {
size_t i1 = find(line,',');
size_t i2 = find(line,':');
std::string domain = deblank(std::string(line,12,i1-12));
std::string variable = deblank(std::string(line,i1+1,i2-i1-2));
std::pair<std::string,std::string> key(domain,variable);
DatabaseEntry data(&line[i2+1]);
} else if ( strncmp( line, " variable(", 12 ) == 0 ) {
size_t i1 = find( line, ',' );
size_t i2 = find( line, ':' );
std::string domain = deblank( std::string( line, 12, i1 - 12 ) );
std::string variable = deblank( std::string( line, i1 + 1, i2 - i1 - 2 ) );
std::pair<std::string, std::string> key( domain, variable );
DatabaseEntry data( &line[i2 + 1] );
meshes.back().variable_data.insert(
std::pair<std::pair<std::string,std::string>,DatabaseEntry>(key,data) );
std::pair<std::pair<std::string, std::string>, DatabaseEntry>( key, data ) );
} else {
ERROR("Error reading line");
ERROR( "Error reading line" );
}
}
fclose(fid);
delete [] line;
PROFILE_STOP("read");
fclose( fid );
delete[] line;
PROFILE_STOP( "read" );
return meshes;
}
// Return the mesh type
IO::MeshType meshType( const IO::Mesh& mesh )
IO::MeshType meshType( const IO::Mesh &mesh )
{
IO::MeshType type = IO::Unknown;
IO::MeshType type = IO::MeshType::Unknown;
const std::string meshClass = mesh.className();
if ( meshClass=="PointList" ) {
type = IO::PointMesh;
} else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
type = IO::SurfaceMesh;
} else if ( meshClass=="DomainMesh" ) {
type = IO::VolumeMesh;
if ( meshClass == "PointList" ) {
type = IO::MeshType::PointMesh;
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
type = IO::MeshType::SurfaceMesh;
} else if ( meshClass == "DomainMesh" ) {
type = IO::MeshType::VolumeMesh;
} else {
ERROR("Unknown mesh");
ERROR( "Unknown mesh" );
}
return type;
}
} // IO namespace
} // namespace IO

55
IO/MeshDatabase.h
View File

@ -5,32 +5,25 @@
#include "common/MPI.h"
#include <iostream>
#include <map>
#include <memory>
#include <string.h>
#include <vector>
#include <map>
namespace IO {
class Mesh;
//! Enum to identify mesh type
//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
//! Helper struct for containing offsets for the mesh info
struct DatabaseEntry {
std::string name; //!< Name of the entry
std::string file; //!< Name of the file containing the entry
size_t offset; //!< Offset in the file to start reading
std::string write( ) const; //!< Convert the data to a string
void read( const char* line ); //!< Convert the string to data
void read( const std::string& line ); //!< Convert the string to data
DatabaseEntry( ) {} //!< Empty constructor
DatabaseEntry( const char* line ); //!< Convert the string to data
std::string write() const; //!< Convert the data to a string
void read( const char *line ); //!< Convert the string to data
void read( const std::string &line ); //!< Convert the string to data
DatabaseEntry() {} //!< Empty constructor
DatabaseEntry( const char *line ); //!< Convert the string to data
~DatabaseEntry() {} //!< Destructor
};
@ -41,50 +34,52 @@ struct VariableDatabase {
IO::VariableType type; //!< Variable
unsigned int dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
// Overload key operators
bool operator==(const VariableDatabase& rhs ) const;
bool operator!=(const VariableDatabase& rhs ) const;
bool operator>=(const VariableDatabase& rhs ) const;
bool operator<=(const VariableDatabase& rhs ) const;
bool operator> (const VariableDatabase& rhs ) const;
bool operator< (const VariableDatabase& rhs ) const;
bool operator==( const VariableDatabase &rhs ) const;
bool operator!=( const VariableDatabase &rhs ) const;
bool operator>=( const VariableDatabase &rhs ) const;
bool operator<=( const VariableDatabase &rhs ) const;
bool operator>( const VariableDatabase &rhs ) const;
bool operator<( const VariableDatabase &rhs ) const;
};
//! Structure to hold the info about the meshes
struct MeshDatabase {
typedef std::pair<std::string,std::string> variable_id;
typedef std::pair<std::string, std::string> variable_id;
std::string name; //!< Name of the mesh
MeshType type; //!< Mesh type
std::string meshClass; //!< Mesh class
unsigned char format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
FileFormat format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
std::vector<DatabaseEntry> domains; //!< List of the domains
std::vector<VariableDatabase> variables; //!< List of the variables
std::map<variable_id,DatabaseEntry> variable_data; //!< Data for the variables
VariableDatabase getVariableDatabase( const std::string& varname ) const;
std::map<variable_id, DatabaseEntry> variable_data; //!< Data for the variables
VariableDatabase getVariableDatabase( const std::string &varname ) const;
public:
MeshDatabase();
~MeshDatabase();
MeshDatabase(const MeshDatabase&);
MeshDatabase& operator=(const MeshDatabase&);
MeshDatabase( const MeshDatabase & );
MeshDatabase &operator=( const MeshDatabase & );
};
//! Gather the mesh databases from all processors
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm );
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm );
//! Write the mesh databases to a file
void write( const std::vector<MeshDatabase>& meshes, const std::string& filename );
void write( const std::vector<MeshDatabase> &meshes, const std::string &filename );
//! Read the mesh databases from a file
std::vector<MeshDatabase> read( const std::string& filename );
std::vector<MeshDatabase> read( const std::string &filename );
//! Return the mesh type
IO::MeshType meshType( const IO::Mesh& mesh );
IO::MeshType meshType( const IO::Mesh &mesh );
} // IO namespace
} // namespace IO
#endif

121
IO/PIO.cpp
View File

@ -1,10 +1,10 @@
#include "IO/PIO.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <cstring>
#include <fstream>
#include <string>
#include <cstring>
namespace IO {
@ -15,19 +15,18 @@ static ParallelStreamBuffer perr_buffer;
static ParallelStreamBuffer plog_buffer;
std::ostream pout(&pout_buffer);
std::ostream perr(&perr_buffer);
std::ostream plog(&plog_buffer);
std::ostream pout( &pout_buffer );
std::ostream perr( &perr_buffer );
std::ostream plog( &plog_buffer );
/****************************************************************************
* Functions to control logging *
****************************************************************************/
std::ofstream *global_filestream=NULL;
static void shutdownFilestream( )
* Functions to control logging *
****************************************************************************/
std::ofstream *global_filestream = NULL;
static void shutdownFilestream()
{
if ( global_filestream!=NULL ) {
if ( global_filestream != NULL ) {
global_filestream->flush();
global_filestream->close();
delete global_filestream;
@ -37,16 +36,16 @@ static void shutdownFilestream( )
void Utilities::logOnlyNodeZero( const std::string &filename )
{
int rank = 0;
#ifdef USE_MPI
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
#endif
if ( rank == 0 )
logAllNodes(filename,true);
logAllNodes( filename, true );
}
void Utilities::logAllNodes( const std::string &filename, bool singleStream )
{
if ( singleStream )
ERROR("Not implimented yet");
ERROR( "Not implimented yet" );
// If the filestream was open, then close it and reset streams
shutdownFilestream();
@ -55,33 +54,33 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream )
std::string full_filename = filename;
if ( !singleStream ) {
int rank = 0;
#ifdef USE_MPI
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
#endif
char tmp[100];
sprintf(tmp,".%04i",rank);
full_filename += std::string(tmp);
sprintf( tmp, ".%04i", rank );
full_filename += std::string( tmp );
}
global_filestream = new std::ofstream(full_filename.c_str());
global_filestream = new std::ofstream( full_filename.c_str() );
if ( !(*global_filestream) ) {
if ( !( *global_filestream ) ) {
delete global_filestream;
global_filestream = NULL;
perr << "PIO: Could not open log file ``" << full_filename << "''\n";
} else {
pout_buffer.setOutputStream(global_filestream);
pout_buffer.setOutputStream(&std::cout);
perr_buffer.setOutputStream(global_filestream);
perr_buffer.setOutputStream(&std::cerr);
plog_buffer.setOutputStream(global_filestream);
pout_buffer.setOutputStream( global_filestream );
pout_buffer.setOutputStream( &std::cout );
perr_buffer.setOutputStream( global_filestream );
perr_buffer.setOutputStream( &std::cerr );
plog_buffer.setOutputStream( global_filestream );
}
}
/****************************************************************************
* ParallelStreamBuffer class *
****************************************************************************/
void Utilities::stopLogging( )
* ParallelStreamBuffer class *
****************************************************************************/
void Utilities::stopLogging()
{
pout_buffer.reset();
perr_buffer.reset();
@ -93,77 +92,71 @@ void Utilities::stopLogging( )
/****************************************************************************
* ParallelStreamBuffer class *
****************************************************************************/
ParallelStreamBuffer::ParallelStreamBuffer( ):
d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL)
* ParallelStreamBuffer class *
****************************************************************************/
ParallelStreamBuffer::ParallelStreamBuffer()
: d_rank( 0 ), d_size( 0 ), d_buffer_size( 0 ), d_buffer( NULL )
{
}
ParallelStreamBuffer:: ~ParallelStreamBuffer()
{
delete [] d_buffer;
}
void ParallelStreamBuffer::setOutputStream( std::ostream *stream )
{
d_stream.push_back( stream );
}
ParallelStreamBuffer::~ParallelStreamBuffer() { delete[] d_buffer; }
void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) { d_stream.push_back( stream ); }
int ParallelStreamBuffer::sync()
{
for (size_t i=0; i<d_stream.size(); i++) {
std::ostream& stream = *d_stream[i];
for ( size_t i = 0; i < d_stream.size(); i++ ) {
std::ostream &stream = *d_stream[i];
stream << d_buffer;
}
d_size = 0;
memset(d_buffer,0,d_buffer_size);
memset( d_buffer, 0, d_buffer_size );
return 0;
}
void ParallelStreamBuffer::reserve( size_t size )
{
if ( size > d_buffer_size ) {
if ( d_buffer_size==0 ) {
if ( d_buffer_size == 0 ) {
d_buffer_size = 1024;
d_buffer = new char[d_buffer_size];
memset(d_buffer,0,d_buffer_size);
memset( d_buffer, 0, d_buffer_size );
}
while ( size > d_buffer_size ) {
char *tmp = d_buffer;
d_buffer_size *= 2;
d_buffer = new char[d_buffer_size];
memset(d_buffer,0,d_buffer_size);
memcpy(d_buffer,tmp,d_size);
delete [] tmp;
memset( d_buffer, 0, d_buffer_size );
memcpy( d_buffer, tmp, d_size );
delete[] tmp;
}
}
}
std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize n )
std::streamsize ParallelStreamBuffer::xsputn( const char *text, std::streamsize n )
{
reserve(d_size+n);
memcpy(&d_buffer[d_size],text,n);
reserve( d_size + n );
memcpy( &d_buffer[d_size], text, n );
d_size += n;
if ( text[n-1]==0 || text[n-1]==10 ) { sync(); }
if ( text[n - 1] == 0 || text[n - 1] == 10 ) {
sync();
}
return n;
}
int ParallelStreamBuffer::overflow(int ch)
int ParallelStreamBuffer::overflow( int ch )
{
reserve(d_size+1);
reserve( d_size + 1 );
d_buffer[d_size] = ch;
d_size++;
if ( ch==0 || ch==10 ) { sync(); }
return std::char_traits<char>::to_int_type(ch);
}
int ParallelStreamBuffer::underflow()
{
return -1;
if ( ch == 0 || ch == 10 ) {
sync();
}
return std::char_traits<char>::to_int_type( ch );
}
int ParallelStreamBuffer::underflow() { return -1; }
void ParallelStreamBuffer::reset()
{
sync();
d_stream.clear();
delete [] d_buffer;
delete[] d_buffer;
d_buffer = NULL;
d_buffer_size = 0;
}
} // IO namespace
} // namespace IO

21
IO/PIO.h
View File

@ -45,12 +45,11 @@ inline int printp( const char *format, ... );
class ParallelStreamBuffer : public std::streambuf
{
public:
/*!
* Create a parallel buffer class. The object will require further
* initialization to set up the I/O streams and prefix string.
*/
ParallelStreamBuffer( );
ParallelStreamBuffer();
/*!
* Set the output file stream (multiple output streams are supported)
@ -73,13 +72,13 @@ public:
* Write the specified number of characters into the output stream (called
* from streambuf).
*/
virtual std::streamsize xsputn(const char* text, std::streamsize n);
virtual std::streamsize xsputn( const char *text, std::streamsize n );
/*!
* Write an overflow character into the parallel buffer (called from
* streambuf).
*/
virtual int overflow(int ch);
virtual int overflow( int ch );
/*!
* Read an overflow character from the parallel buffer (called from
@ -98,30 +97,30 @@ private:
size_t d_size;
size_t d_buffer_size;
char *d_buffer;
std::vector<std::ostream*> d_stream;
std::vector<std::ostream *> d_stream;
inline void reserve( size_t size );
};
namespace Utilities {
/*!
/*!
* Log messages for node zero only to the specified filename. All output
* to pout, perr, and plog on node zero will go to the log file.
*/
void logOnlyNodeZero( const std::string &filename );
void logOnlyNodeZero( const std::string &filename );
/*!
/*!
* Log messages from all nodes. The diagnostic data for processor XXXXX
* will be sent to a file with the name filename.XXXXX, where filename is
* the function argument.
*/
void logAllNodes( const std::string &filename, bool singleStream=false );
void logAllNodes( const std::string &filename, bool singleStream = false );
/*!
/*!
* Stop logging messages, flush buffers, and reset memory.
*/
void stopLogging( );
void stopLogging();
} // namespace Utilities

10
IO/PIO.hpp
View File

@ -3,9 +3,9 @@
#include "IO/PIO.h"
#include <cstdio>
#include <iostream>
#include <stdarg.h>
#include <cstdio>
namespace IO {
@ -14,16 +14,16 @@ namespace IO {
inline int printp( const char *format, ... )
{
va_list ap;
va_start(ap,format);
va_start( ap, format );
char tmp[1024];
int n = vsprintf(tmp,format,ap);
va_end(ap);
int n = vsprintf( tmp, format, ap );
va_end( ap );
pout << tmp;
pout.flush();
return n;
}
} // IO namespace
} // namespace IO
#endif

77
IO/PackData.cpp
View File

@ -4,102 +4,101 @@
/********************************************************
* Concrete implimentations for packing/unpacking *
********************************************************/
* Concrete implimentations for packing/unpacking *
********************************************************/
// unsigned char
template<>
size_t packsize<unsigned char>( const unsigned char& rhs )
size_t packsize<unsigned char>( const unsigned char &rhs )
{
return sizeof(unsigned char);
return sizeof( unsigned char );
}
template<>
void pack<unsigned char>( const unsigned char& rhs, char *buffer )
void pack<unsigned char>( const unsigned char &rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(unsigned char));
memcpy( buffer, &rhs, sizeof( unsigned char ) );
}
template<>
void unpack<unsigned char>( unsigned char& data, const char *buffer )
void unpack<unsigned char>( unsigned char &data, const char *buffer )
{
memcpy(&data,buffer,sizeof(unsigned char));
memcpy( &data, buffer, sizeof( unsigned char ) );
}
// char
template<>
size_t packsize<char>( const char& rhs )
size_t packsize<char>( const char &rhs )
{
return sizeof(char);
return sizeof( char );
}
template<>
void pack<char>( const char& rhs, char *buffer )
void pack<char>( const char &rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(char));
memcpy( buffer, &rhs, sizeof( char ) );
}
template<>
void unpack<char>( char& data, const char *buffer )
void unpack<char>( char &data, const char *buffer )
{
memcpy(&data,buffer,sizeof(char));
memcpy( &data, buffer, sizeof( char ) );
}
// int
template<>
size_t packsize<int>( const int& rhs )
size_t packsize<int>( const int &rhs )
{
return sizeof(int);
return sizeof( int );
}
template<>
void pack<int>( const int& rhs, char *buffer )
void pack<int>( const int &rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
memcpy( buffer, &rhs, sizeof( int ) );
}
template<>
void unpack<int>( int& data, const char *buffer )
void unpack<int>( int &data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
memcpy( &data, buffer, sizeof( int ) );
}
// unsigned int
template<>
size_t packsize<unsigned int>( const unsigned int& rhs )
size_t packsize<unsigned int>( const unsigned int &rhs )
{
return sizeof(unsigned int);
return sizeof( unsigned int );
}
template<>
void pack<unsigned int>( const unsigned int& rhs, char *buffer )
void pack<unsigned int>( const unsigned int &rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
memcpy( buffer, &rhs, sizeof( int ) );
}
template<>
void unpack<unsigned int>( unsigned int& data, const char *buffer )
void unpack<unsigned int>( unsigned int &data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
memcpy( &data, buffer, sizeof( int ) );
}
// size_t
template<>
size_t packsize<size_t>( const size_t& rhs )
size_t packsize<size_t>( const size_t &rhs )
{
return sizeof(size_t);
return sizeof( size_t );
}
template<>
void pack<size_t>( const size_t& rhs, char *buffer )
void pack<size_t>( const size_t &rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(size_t));
memcpy( buffer, &rhs, sizeof( size_t ) );
}
template<>
void unpack<size_t>( size_t& data, const char *buffer )
void unpack<size_t>( size_t &data, const char *buffer )
{
memcpy(&data,buffer,sizeof(size_t));
memcpy( &data, buffer, sizeof( size_t ) );
}
// std::string
template<>
size_t packsize<std::string>( const std::string& rhs )
size_t packsize<std::string>( const std::string &rhs )
{
return rhs.size()+1;
return rhs.size() + 1;
}
template<>
void pack<std::string>( const std::string& rhs, char *buffer )
void pack<std::string>( const std::string &rhs, char *buffer )
{
memcpy(buffer,rhs.c_str(),rhs.size()+1);
memcpy( buffer, rhs.c_str(), rhs.size() + 1 );
}
template<>
void unpack<std::string>( std::string& data, const char *buffer )
void unpack<std::string>( std::string &data, const char *buffer )
{
data = std::string(buffer);
data = std::string( buffer );
}

35
IO/PackData.h
View File

@ -2,77 +2,76 @@
#ifndef included_PackData
#define included_PackData
#include <vector>
#include <set>
#include <map>
#include <set>
#include <vector>
//! Template function to return the buffer size required to pack a class
template<class TYPE>
size_t packsize( const TYPE& rhs );
size_t packsize( const TYPE &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const TYPE& rhs, char *buffer );
void pack( const TYPE &rhs, char *buffer );
//! Template function to unpack a class from a buffer
template<class TYPE>
void unpack( TYPE& data, const char *buffer );
void unpack( TYPE &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::vector
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs );
size_t packsize( const std::vector<TYPE> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer );
void pack( const std::vector<TYPE> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer );
void unpack( std::vector<TYPE> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::pair
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
size_t packsize( const std::pair<TYPE1, TYPE2> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::map
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
size_t packsize( const std::map<TYPE1, TYPE2> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::set
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs );
size_t packsize( const std::set<TYPE> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer );
void pack( const std::set<TYPE> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer );
void unpack( std::set<TYPE> &data, const char *buffer );
#include "IO/PackData.hpp"
#endif

150
IO/PackData.hpp
View File

@ -4,152 +4,156 @@
#include "IO/PackData.h"
#include <map>
#include <set>
#include <string.h>
#include <vector>
#include <set>
#include <map>
/********************************************************
* Default instantiations for std::vector *
********************************************************/
* Default instantiations for std::vector *
********************************************************/
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs )
size_t packsize( const std::vector<TYPE> &rhs )
{
size_t bytes = sizeof(size_t);
for (size_t i=0; i<rhs.size(); i++)
bytes += packsize(rhs[i]);
size_t bytes = sizeof( size_t );
for ( size_t i = 0; i < rhs.size(); i++ )
bytes += packsize( rhs[i] );
return bytes;
}
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer )
void pack( const std::vector<TYPE> &rhs, char *buffer )
{
size_t size = rhs.size();
memcpy(buffer,&size,sizeof(size_t));
size_t pos = sizeof(size_t);
for (size_t i=0; i<rhs.size(); i++) {
pack(rhs[i],&buffer[pos]);
pos += packsize(rhs[i]);
memcpy( buffer, &size, sizeof( size_t ) );
size_t pos = sizeof( size_t );
for ( size_t i = 0; i < rhs.size(); i++ ) {
pack( rhs[i], &buffer[pos] );
pos += packsize( rhs[i] );
}
}
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer )
void unpack( std::vector<TYPE> &data, const char *buffer )
{
size_t size;
memcpy(&size,buffer,sizeof(size_t));
memcpy( &size, buffer, sizeof( size_t ) );
data.clear();
data.resize(size);
size_t pos = sizeof(size_t);
for (size_t i=0; i<data.size(); i++) {
unpack(data[i],&buffer[pos]);
pos += packsize(data[i]);
data.resize( size );
size_t pos = sizeof( size_t );
for ( size_t i = 0; i < data.size(); i++ ) {
unpack( data[i], &buffer[pos] );
pos += packsize( data[i] );
}
}
/********************************************************
* Default instantiations for std::pair *
********************************************************/
* Default instantiations for std::pair *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs )
size_t packsize( const std::pair<TYPE1, TYPE2> &rhs )
{
return packsize(rhs.first)+packsize(rhs.second);
return packsize( rhs.first ) + packsize( rhs.second );
}
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer )
void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer )
{
pack(rhs.first,buffer);
pack(rhs.second,&buffer[packsize(rhs.first)]);
pack( rhs.first, buffer );
pack( rhs.second, &buffer[packsize( rhs.first )] );
}
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer )
void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer )
{
unpack(data.first,buffer);
unpack(data.second,&buffer[packsize(data.first)]);
unpack( data.first, buffer );
unpack( data.second, &buffer[packsize( data.first )] );
}
/********************************************************
* Default instantiations for std::map *
********************************************************/
* Default instantiations for std::map *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs )
size_t packsize( const std::map<TYPE1, TYPE2> &rhs )
{
size_t bytes = sizeof(size_t);
typename std::map<TYPE1,TYPE2>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
bytes += packsize(it->first);
bytes += packsize(it->second);
size_t bytes = sizeof( size_t );
typename std::map<TYPE1, TYPE2>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
bytes += packsize( it->first );
bytes += packsize( it->second );
}
return bytes;
}
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer )
void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer )
{
size_t N = rhs.size();
pack(N,buffer);
size_t pos = sizeof(size_t);
typename std::map<TYPE1,TYPE2>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
pack(it->first,&buffer[pos]); pos+=packsize(it->first);
pack(it->second,&buffer[pos]); pos+=packsize(it->second);
pack( N, buffer );
size_t pos = sizeof( size_t );
typename std::map<TYPE1, TYPE2>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack( it->first, &buffer[pos] );
pos += packsize( it->first );
pack( it->second, &buffer[pos] );
pos += packsize( it->second );
}
}
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer )
void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer )
{
size_t N = 0;
unpack(N,buffer);
size_t pos = sizeof(size_t);
unpack( N, buffer );
size_t pos = sizeof( size_t );
data.clear();
for (size_t i=0; i<N; i++) {
std::pair<TYPE1,TYPE2> tmp;
unpack(tmp.first,&buffer[pos]); pos+=packsize(tmp.first);
unpack(tmp.second,&buffer[pos]); pos+=packsize(tmp.second);
data.insert(tmp);
for ( size_t i = 0; i < N; i++ ) {
std::pair<TYPE1, TYPE2> tmp;
unpack( tmp.first, &buffer[pos] );
pos += packsize( tmp.first );
unpack( tmp.second, &buffer[pos] );
pos += packsize( tmp.second );
data.insert( tmp );
}
}
/********************************************************
* Default instantiations for std::set *
********************************************************/
* Default instantiations for std::set *
********************************************************/
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs )
size_t packsize( const std::set<TYPE> &rhs )
{
size_t bytes = sizeof(size_t);
size_t bytes = sizeof( size_t );
typename std::set<TYPE>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
bytes += packsize(*it);
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
bytes += packsize( *it );
}
return bytes;
}
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer )
void pack( const std::set<TYPE> &rhs, char *buffer )
{
size_t N = rhs.size();
pack(N,buffer);
size_t pos = sizeof(size_t);
pack( N, buffer );
size_t pos = sizeof( size_t );
typename std::set<TYPE>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
pack(*it); pos+=packsize(*it);
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack( *it );
pos += packsize( *it );
}
}
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer )
void unpack( std::set<TYPE> &data, const char *buffer )
{
size_t N = 0;
unpack(N,buffer);
size_t pos = sizeof(size_t);
unpack( N, buffer );
size_t pos = sizeof( size_t );
data.clear();
for (size_t i=0; i<N; i++) {
for ( size_t i = 0; i < N; i++ ) {
TYPE tmp;
unpack(tmp,&buffer[pos]); pos+=packsize(tmp);
data.insert(tmp);
unpack( tmp, &buffer[pos] );
pos += packsize( tmp );
data.insert( tmp );
}
}
#endif

438
IO/Reader.cpp
View File

@ -1,7 +1,7 @@
#include "IO/Reader.h"
#include "IO/IOHelpers.h"
#include "IO/Mesh.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#ifdef USE_SILO
@ -10,60 +10,139 @@
#include <ProfilerApp.h>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <map>
#include <string.h>
#include <vector>
#include <map>
#include <cstdio>
// Inline function to read line without a return argument
static inline void fgetl( char * str, int num, FILE * stream )
static inline void fgetl( char *str, int num, FILE *stream )
{
char* ptr = fgets( str, num, stream );
if ( 0 ) {char *temp = (char *)&ptr; temp++;}
char *ptr = fgets( str, num, stream );
if ( 0 ) {
char *temp = (char *) &ptr;
temp++;
}
}
// Check if the file exists
bool fileExists( const std::string &filename )
{
std::ifstream ifile( filename.c_str() );
return ifile.good();
}
// Get the path to a file
std::string IO::getPath( const std::string& filename )
std::string IO::getPath( const std::string &filename )
{
std::string file(filename);
size_t k1 = file.rfind(47);
size_t k2 = file.rfind(92);
if ( k1==std::string::npos ) { k1=0; }
if ( k2==std::string::npos ) { k2=0; }
return file.substr(0,std::max(k1,k2));
std::string file( filename );
size_t k1 = file.rfind( 47 );
size_t k2 = file.rfind( 92 );
if ( k1 == std::string::npos ) {
k1 = 0;
}
if ( k2 == std::string::npos ) {
k2 = 0;
}
return file.substr( 0, std::max( k1, k2 ) );
}
// List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string& filename )
// List the timesteps in the given directory (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string &path, const std::string &format )
{
PROFILE_START("readTimesteps");
FILE *fid= fopen(filename.c_str(),"rb");
if ( fid==NULL )
ERROR("Error opening file");
// Get the name of the summary filename
std::string filename = path + "/";
if ( format == "old" || format == "new" ) {
filename += "summary.LBM";
} else if ( format == "silo" ) {
filename += "LBM.visit";
} else if ( format == "auto" ) {
bool test_old = fileExists( path + "/summary.LBM" );
bool test_silo = fileExists( path + "/LBM.visit" );
if ( test_old && test_silo ) {
ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" );
} else if ( test_old ) {
filename += "summary.LBM";
} else if ( test_silo ) {
filename += "LBM.visit";
} else {
ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" );
}
} else {
ERROR( "Unknown format: " + format );
}
PROFILE_START( "readTimesteps" );
// Read the data
FILE *fid = fopen( filename.c_str(), "rb" );
if ( fid == NULL )
ERROR( "Error opening file" );
std::vector<std::string> timesteps;
char buf[1000];
while (fgets(buf,sizeof(buf),fid) != NULL) {
std::string line(buf);
line.resize(line.size()-1);
while ( fgets( buf, sizeof( buf ), fid ) != NULL ) {
std::string line( buf );
line.resize( line.size() - 1 );
auto pos = line.find( "summary.silo" );
if ( pos != std::string::npos )
line.resize(pos);
line.resize( pos );
if ( line.empty() )
continue;
timesteps.push_back(line);
timesteps.push_back( line );
}
fclose(fid);
PROFILE_STOP("readTimesteps");
fclose( fid );
PROFILE_STOP( "readTimesteps" );
return timesteps;
return timesteps;
}
// Get the maximum number of domains
int IO::maxDomains( const std::string &path, const std::string &format, const Utilities::MPI &comm )
{
int rank = comm.getRank();
int n_domains = 0;
if ( rank == 0 ) {
// Get the timesteps
auto timesteps = IO::readTimesteps( path, format );
ASSERT( !timesteps.empty() );
// Get the database for the first domain
auto db = IO::getMeshList( path, timesteps[0] );
for ( size_t i = 0; i < db.size(); i++ )
n_domains = std::max<int>( n_domains, db[i].domains.size() );
}
return comm.bcast( n_domains, 0 );
}
// Read the data for the given timestep
std::vector<IO::MeshDataStruct> IO::readData(
const std::string &path, const std::string &timestep, int rank )
{
// Get the mesh databases
auto db = IO::getMeshList( path, timestep );
// Create the data
std::vector<IO::MeshDataStruct> data( db.size() );
for ( size_t i = 0; i < data.size(); i++ ) {
data[i].precision = IO::DataType::Double;
data[i].meshName = db[i].name;
data[i].mesh = getMesh( path, timestep, db[i], rank );
data[i].vars.resize( db[i].variables.size() );
for ( size_t j = 0; j < db[i].variables.size(); j++ )
data[i].vars[j] = getVariable( path, timestep, db[i], rank, db[i].variables[j].name );
INSIST( data[i].check(), "Failed check of " + data[i].meshName );
}
return data;
}
// Read the list of variables for the given timestep
std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const std::string& timestep )
std::vector<IO::MeshDatabase> IO::getMeshList(
const std::string &path, const std::string &timestep )
{
std::string filename = path + "/" + timestep + "/LBM.summary";
return IO::read( filename );
@ -71,270 +150,271 @@ std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const st
// Read the given mesh domain
std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::string& timestep,
const IO::MeshDatabase& meshDatabase, int domain )
std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::string &timestep,
const IO::MeshDatabase &meshDatabase, int domain )
{
PROFILE_START("getMesh");
PROFILE_START( "getMesh" );
std::shared_ptr<IO::Mesh> mesh;
if ( meshDatabase.format==1 ) {
if ( meshDatabase.format == FileFormat::OLD ) {
// Old format (binary doubles)
std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file;
FILE *fid = fopen(filename.c_str(),"rb");
INSIST(fid!=NULL,"Error opening file");
FILE *fid = fopen( filename.c_str(), "rb" );
INSIST( fid != NULL, "Error opening file" );
fseek( fid, 0, SEEK_END );
size_t bytes = ftell(fid);
size_t N_max = bytes/sizeof(double)+1000;
size_t bytes = ftell( fid );
size_t N_max = bytes / sizeof( double ) + 1000;
double *data = new double[N_max];
fseek(fid,0,SEEK_SET);
size_t count = fread(data,sizeof(double),N_max,fid);
fclose(fid);
if ( count%3 != 0 )
ERROR("Error reading file");
if ( meshDatabase.type==IO::PointMesh ) {
size_t N = count/3;
std::shared_ptr<PointList> pointlist( new PointList(N) );
std::vector<Point>& P = pointlist->points;
for (size_t i=0; i<N; i++) {
P[i].x = data[3*i+0];
P[i].y = data[3*i+1];
P[i].z = data[3*i+2];
fseek( fid, 0, SEEK_SET );
size_t count = fread( data, sizeof( double ), N_max, fid );
fclose( fid );
if ( count % 3 != 0 )
ERROR( "Error reading file" );
if ( meshDatabase.type == IO::MeshType::PointMesh ) {
size_t N = count / 3;
std::shared_ptr<PointList> pointlist( new PointList( N ) );
std::vector<Point> &P = pointlist->points;
for ( size_t i = 0; i < N; i++ ) {
P[i].x = data[3 * i + 0];
P[i].y = data[3 * i + 1];
P[i].z = data[3 * i + 2];
}
mesh = pointlist;
} else if ( meshDatabase.type==IO::SurfaceMesh ) {
if ( count%9 != 0 )
ERROR("Error reading file (2)");
size_t N_tri = count/9;
std::shared_ptr<TriList> trilist( new TriList(N_tri) );
std::vector<Point>& A = trilist->A;
std::vector<Point>& B = trilist->B;
std::vector<Point>& C = trilist->C;
for (size_t i=0; i<N_tri; i++) {
A[i].x = data[9*i+0];
A[i].y = data[9*i+1];
A[i].z = data[9*i+2];
B[i].x = data[9*i+3];
B[i].y = data[9*i+4];
B[i].z = data[9*i+5];
C[i].x = data[9*i+6];
C[i].y = data[9*i+7];
C[i].z = data[9*i+8];
} else if ( meshDatabase.type == IO::MeshType::SurfaceMesh ) {
if ( count % 9 != 0 )
ERROR( "Error reading file (2)" );
size_t N_tri = count / 9;
std::shared_ptr<TriList> trilist( new TriList( N_tri ) );
std::vector<Point> &A = trilist->A;
std::vector<Point> &B = trilist->B;
std::vector<Point> &C = trilist->C;
for ( size_t i = 0; i < N_tri; i++ ) {
A[i].x = data[9 * i + 0];
A[i].y = data[9 * i + 1];
A[i].z = data[9 * i + 2];
B[i].x = data[9 * i + 3];
B[i].y = data[9 * i + 4];
B[i].z = data[9 * i + 5];
C[i].x = data[9 * i + 6];
C[i].y = data[9 * i + 7];
C[i].z = data[9 * i + 8];
}
mesh = trilist;
} else if ( meshDatabase.type==IO::VolumeMesh ) {
} else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) {
// this was never supported in the old format
mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
} else {
ERROR("Unknown mesh type");
ERROR( "Unknown mesh type" );
}
delete [] data;
} else if ( meshDatabase.format==2 ) {
const DatabaseEntry& database = meshDatabase.domains[domain];
delete[] data;
} else if ( meshDatabase.format == FileFormat::NEW ||
meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen(filename.c_str(),"rb");
fseek(fid,database.offset,SEEK_SET);
FILE *fid = fopen( filename.c_str(), "rb" );
fseek( fid, database.offset, SEEK_SET );
char line[1000];
fgetl(line,1000,fid);
size_t i1 = find(line,':');
size_t i2 = find(&line[i1+1],':')+i1+1;
size_t bytes = atol(&line[i2+1]);
fgetl( line, 1000, fid );
size_t i1 = find( line, ':' );
size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1;
size_t bytes = atol( &line[i2 + 1] );
char *data = new char[bytes];
size_t count = fread(data,1,bytes,fid);
fclose(fid);
ASSERT(count==bytes);
if ( meshDatabase.meshClass=="PointList" ) {
size_t count = fread( data, 1, bytes, fid );
fclose( fid );
ASSERT( count == bytes );
if ( meshDatabase.meshClass == "PointList" ) {
mesh.reset( new IO::PointList() );
} else if ( meshDatabase.meshClass=="TriMesh" ) {
} else if ( meshDatabase.meshClass == "TriMesh" ) {
mesh.reset( new IO::TriMesh() );
} else if ( meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriList" ) {
mesh.reset( new IO::TriList() );
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
mesh.reset( new IO::DomainMesh() );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
mesh->unpack( std::pair<size_t,void*>(bytes,data) );
delete [] data;
} else if ( meshDatabase.format==4 ) {
mesh->unpack( std::pair<size_t, void *>( bytes, data ) );
delete[] data;
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const DatabaseEntry& database = meshDatabase.domains[domain];
const DatabaseEntry &database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
if ( meshDatabase.meshClass=="PointList" ) {
if ( meshDatabase.meshClass == "PointList" ) {
Array<double> coords = silo::readPointMesh<double>( fid, database.name );
ASSERT(coords.size(1)==3);
std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size(0) ) );
for (size_t i=0; i<coords.size(1); i++) {
mesh2->points[i].x = coords(i,0);
mesh2->points[i].y = coords(i,1);
mesh2->points[i].z = coords(i,2);
ASSERT( coords.size( 1 ) == 3 );
std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size( 0 ) ) );
for ( size_t i = 0; i < coords.size( 1 ); i++ ) {
mesh2->points[i].x = coords( i, 0 );
mesh2->points[i].y = coords( i, 1 );
mesh2->points[i].z = coords( i, 2 );
}
mesh = mesh2;
} else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
Array<double> coords;
Array<int> tri;
silo::readTriMesh( fid, database.name, coords, tri );
ASSERT( tri.size(1)==3 && coords.size(1)==3 );
int N_tri = tri.size(0);
int N_point = coords.size(0);
ASSERT( tri.size( 1 ) == 3 && coords.size( 1 ) == 3 );
int N_tri = tri.size( 0 );
int N_point = coords.size( 0 );
std::shared_ptr<IO::TriMesh> mesh2( new IO::TriMesh( N_tri, N_point ) );
for (int i=0; i<N_point; i++) {
mesh2->vertices->points[i].x = coords(i,0);
mesh2->vertices->points[i].y = coords(i,1);
mesh2->vertices->points[i].z = coords(i,2);
for ( int i = 0; i < N_point; i++ ) {
mesh2->vertices->points[i].x = coords( i, 0 );
mesh2->vertices->points[i].y = coords( i, 1 );
mesh2->vertices->points[i].z = coords( i, 2 );
}
for (int i=0; i<N_tri; i++) {
mesh2->A[i] = tri(i,0);
mesh2->B[i] = tri(i,1);
mesh2->C[i] = tri(i,2);
for ( int i = 0; i < N_tri; i++ ) {
mesh2->A[i] = tri( i, 0 );
mesh2->B[i] = tri( i, 1 );
mesh2->C[i] = tri( i, 2 );
}
if ( meshDatabase.meshClass=="TriMesh" ) {
if ( meshDatabase.meshClass == "TriMesh" ) {
mesh = mesh2;
} else if ( meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriList" ) {
auto trilist = IO::getTriList( std::dynamic_pointer_cast<IO::Mesh>( mesh2 ) );
mesh = trilist;
}
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
std::vector<double> range;
std::vector<int> N;
silo::readUniformMesh( fid, database.name, range, N );
auto rankinfo = silo::read<int>( fid, database.name+"_rankinfo" );
auto rankinfo = silo::read<int>( fid, database.name + "_rankinfo" );
RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0],
range[3] - range[2], range[5] - range[4] ) );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
silo::close( fid );
#else
ERROR("Build without silo support");
ERROR( "Build without silo support" );
#endif
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
PROFILE_STOP("getMesh");
PROFILE_STOP( "getMesh" );
return mesh;
}
// Read the given variable for the given mesh domain
std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain, const std::string& variable )
std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain, const std::string &variable )
{
std::pair<std::string,std::string> key(meshDatabase.domains[domain].name,variable);
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
it = meshDatabase.variable_data.find(key);
if ( it==meshDatabase.variable_data.end() )
std::pair<std::string, std::string> key( meshDatabase.domains[domain].name, variable );
auto it = meshDatabase.variable_data.find( key );
if ( it == meshDatabase.variable_data.end() )
return std::shared_ptr<IO::Variable>();
std::shared_ptr<IO::Variable> var;
if ( meshDatabase.format == 2 ) {
const DatabaseEntry& database = it->second;
if ( meshDatabase.format == FileFormat::NEW || meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = it->second;
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen(filename.c_str(),"rb");
fseek(fid,database.offset,SEEK_SET);
FILE *fid = fopen( filename.c_str(), "rb" );
fseek( fid, database.offset, SEEK_SET );
char line[1000];
fgetl(line,1000,fid);
size_t i1 = find(line,':');
size_t i2 = find(&line[i1+1],':')+i1+1;
std::vector<std::string> values = splitList(&line[i2+1],',');
ASSERT(values.size()==5);
int dim = atoi(values[0].c_str());
int type = atoi(values[1].c_str());
size_t N = atol(values[2].c_str());
size_t bytes = atol(values[3].c_str());
fgetl( line, 1000, fid );
size_t i1 = find( line, ':' );
size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1;
std::vector<std::string> values = splitList( &line[i2 + 1], ',' );
ASSERT( values.size() == 5 );
int dim = atoi( values[0].c_str() );
auto type = values[1];
size_t N = atol( values[2].c_str() );
size_t bytes = atol( values[3].c_str() );
std::string precision = values[4];
var = std::shared_ptr<IO::Variable>( new IO::Variable() );
var->dim = dim;
var->type = static_cast<IO::VariableType>(type);
var->type = getVariableType( type );
var->name = variable;
var->data.resize(N*dim);
if ( precision=="double" ) {
size_t count = fread(var->data.data(),sizeof(double),N*dim,fid);
ASSERT(count*sizeof(double)==bytes);
var->data.resize( N, dim );
if ( precision == "double" ) {
size_t count = fread( var->data.data(), sizeof( double ), N * dim, fid );
ASSERT( count * sizeof( double ) == bytes );
} else {
ERROR("Format not implimented");
ERROR( "Format not implimented" );
}
fclose(fid);
} else if ( meshDatabase.format == 4 ) {
fclose( fid );
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const auto& database = meshDatabase.domains[domain];
const auto &database = meshDatabase.domains[domain];
auto variableDatabase = meshDatabase.getVariableDatabase( variable );
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
if ( meshDatabase.meshClass=="PointList" ) {
if ( meshDatabase.meshClass == "PointList" ) {
var->data = silo::readPointMeshVariable<double>( fid, variable );
} else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
var->data = silo::readTriMeshVariable<double>( fid, variable );
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
var->data = silo::readUniformMeshVariable<double>( fid, variable );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
silo::close( fid );
#else
ERROR("Build without silo support");
ERROR( "Build without silo support" );
#endif
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
return var;
}
/****************************************************
* Reformat the variable to match the mesh *
****************************************************/
void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
* Reformat the variable to match the mesh *
****************************************************/
void IO::reformatVariable( const IO::Mesh &mesh, IO::Variable &var )
{
if ( mesh.className() == "DomainMesh" ) {
const IO::DomainMesh& mesh2 = dynamic_cast<const IO::DomainMesh&>( mesh );
const IO::DomainMesh &mesh2 = dynamic_cast<const IO::DomainMesh &>( mesh );
if ( var.type == VariableType::NodeVariable ) {
size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1));
ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() );
var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } );
size_t N2 =
var.data.length() / ( ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) );
ASSERT(
( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) * N2 == var.data.length() );
var.data.reshape(
{ (size_t) mesh2.nx + 1, (size_t) mesh2.ny + 1, (size_t) mesh2.nz + 1, N2 } );
} else if ( var.type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::VolumeVariable ) {
size_t N2 = var.data.length() / (mesh2.nx*mesh2.ny*mesh2.nz);
ASSERT( mesh2.nx*mesh2.ny*mesh2.nz*N2 == var.data.length() );
size_t N2 = var.data.length() / ( mesh2.nx * mesh2.ny * mesh2.nz );
ASSERT( mesh2.nx * mesh2.ny * mesh2.nz * N2 == var.data.length() );
var.data.reshape( { (size_t) mesh2.nx, (size_t) mesh2.ny, (size_t) mesh2.nz, N2 } );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
} else if ( mesh.className() == "PointList" ) {
const IO::PointList& mesh2 = dynamic_cast<const IO::PointList&>( mesh );
const IO::PointList &mesh2 = dynamic_cast<const IO::PointList &>( mesh );
size_t N = mesh2.points.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else if ( mesh.className()=="TriMesh" || mesh.className() == "TriList" ) {
std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh*>(&mesh), []( void* ) {} );
} else if ( mesh.className() == "TriMesh" || mesh.className() == "TriList" ) {
std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh *>( &mesh ), []( void * ) {} );
std::shared_ptr<TriMesh> mesh2 = getTriMesh( mesh_ptr );
if ( var.type == VariableType::NodeVariable ) {
size_t N = mesh2->vertices->points.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else if ( var.type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::VolumeVariable ) {
size_t N = mesh2->A.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
} else {
ERROR("Unknown mesh type");
ERROR( "Unknown mesh type" );
}
}

59
IO/Reader.h
View File

@ -14,20 +14,59 @@ namespace IO {
//! Get the path to a file
std::string getPath( const std::string& filename );
std::string getPath( const std::string &filename );
//! List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> readTimesteps( const std::string& filename );
/*!
* @brief Get the maximum number of domains written
* @details This function reads the summary files to determine the maximum
* number of domains in the output.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @param[in] comm Optional comm to use to reduce IO load by
* reading on rank 0 and then communicating the result
*/
int maxDomains( const std::string &path, const std::string &format = "auto",
const Utilities::MPI &comm = MPI_COMM_SELF );
/*!
* @brief Read the timestep list
* @details This function reads the timestep list from the summary file.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @return append Append any existing data (default is false)
*/
std::vector<std::string> readTimesteps(
const std::string &path, const std::string &format = "auto" );
/*!
* @brief Read the data for the timestep
* @details This function reads the mesh and variable data provided for the given timestep.
* @param[in] path The path to use for reading
* @param[in] timestep The timestep iteration
* @param[in] domain The desired domain to read
*/
std::vector<IO::MeshDataStruct> readData(
const std::string &path, const std::string &timestep, int domain );
//! Read the list of mesh databases for the given timestep
std::vector<IO::MeshDatabase> getMeshList( const std::string& path, const std::string& timestep );
std::vector<IO::MeshDatabase> getMeshList( const std::string &path, const std::string &timestep );
//! Read the given mesh domain
std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain );
std::shared_ptr<IO::Mesh> getMesh( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain );
/*!
@ -40,8 +79,8 @@ std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& t
* @param[in] variable The variable name to read
* @return Returns the variable data as a linear array
*/
std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain, const std::string& variable );
std::shared_ptr<IO::Variable> getVariable( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain, const std::string &variable );
/*!
@ -50,9 +89,9 @@ std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::s
* @param[in] mesh The underlying mesh
* @param[in/out] variable The variable name to read
*/
void reformatVariable( const IO::Mesh& mesh, IO::Variable& var );
void reformatVariable( const IO::Mesh &mesh, IO::Variable &var );
} // IO namespace
} // namespace IO
#endif

482
IO/Writer.cpp
View File

@ -1,28 +1,69 @@
#include "IO/Writer.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/MeshDatabase.h"
#include "IO/silo.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <sys/stat.h>
#include <algorithm>
#include <vector>
#include <set>
#include <memory>
#include <set>
#include <sys/stat.h>
#include <vector>
enum class Format { OLD, NEW, SILO, UNKNOWN };
/****************************************************
* Recursively create the subdirectory *
****************************************************/
static void recursiveMkdir( const std::string &path, mode_t mode )
{
// Iterate through the root directories until we create the desired path
for ( size_t pos = 0; pos < path.size(); ) {
// slide backwards in string until next slash found
pos++;
for ( ; pos < path.size(); pos++ ) {
if ( path[pos] == '/' || path[pos] == 92 )
break;
}
// Create the temporary path
auto path2 = path.substr( 0, pos );
// Check if the temporary path exists
struct stat status;
int result = stat( path2.data(), &status );
if ( result == 0 ) {
// if there is a part of the path that already exists make sure it is really a directory
if ( !S_ISDIR( status.st_mode ) ) {
ERROR(
"Error in recursiveMkdir...\n"
" Cannot create directories in path = " +
path +
"\n because some intermediate item in path exists and is NOT a directory" );
}
continue;
}
// Create the directory and test the result
result = mkdir( path2.data(), mode );
if ( result != 0 ) {
// Maybe another rank created the directory, check
int result = stat( path2.data(), &status );
if ( result != 0 && !S_ISDIR( status.st_mode ) )
ERROR( "Error in Utilities::recursiveMkdir...\n"
" Cannot create directory = " +
path2 );
}
}
}
/****************************************************
* Initialize the writer *
****************************************************/
* Initialize the writer *
****************************************************/
static std::string global_IO_path;
static Format global_IO_format = Format::UNKNOWN;
void IO::initialize( const std::string& path, const std::string& format, bool append )
void IO::initialize( const std::string &path, const std::string &format, bool append )
{
if ( path.empty() )
global_IO_path = ".";
@ -35,98 +76,109 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
else if ( format == "silo" )
global_IO_format = Format::SILO;
else
ERROR("Unknown format");
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
if ( !append && rank==0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
ERROR( "Unknown format" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( !append && rank == 0 ) {
recursiveMkdir( path, S_IRWXU | S_IRGRP );
std::string filename;
if ( global_IO_format==Format::OLD || global_IO_format==Format::NEW )
if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW )
filename = global_IO_path + "/summary.LBM";
else if ( global_IO_format==Format::SILO )
else if ( global_IO_format == Format::SILO )
filename = global_IO_path + "/LBM.visit";
else
ERROR("Unknown format");
auto fid = fopen(filename.c_str(),"wb");
fclose(fid);
ERROR( "Unknown format" );
auto fid = fopen( filename.c_str(), "wb" );
fclose( fid );
}
}
// Write the mesh data in the original format
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
for (size_t i=0; i<meshData.size(); i++) {
for ( size_t i = 0; i < meshData.size(); i++ ) {
char domainname[100], filename[100], fullpath[200];
sprintf(domainname,"%05i",rank);
sprintf(filename,"%s.%05i",meshData[i].meshName.c_str(),rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
FILE *fid = fopen(fullpath,"wb");
INSIST(fid!=NULL,std::string("Error opening file: ")+fullpath);
sprintf( domainname, "%05i", rank );
sprintf( filename, "%s.%05i", meshData[i].meshName.c_str(), rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
FILE *fid = fopen( fullpath, "wb" );
INSIST( fid != NULL, std::string( "Error opening file: " ) + fullpath );
std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
IO::MeshDatabase mesh_entry;
mesh_entry.name = meshData[i].meshName;
mesh_entry.type = meshType(*mesh);
mesh_entry.type = meshType( *mesh );
mesh_entry.meshClass = meshData[i].mesh->className();
mesh_entry.format = 1;
mesh_entry.format = IO::FileFormat::OLD;
IO::DatabaseEntry domain;
domain.name = domainname;
domain.file = filename;
domain.offset = 0;
mesh_entry.domains.push_back(domain);
mesh_entry.domains.push_back( domain );
if ( !meshData[i].vars.empty() ) {
printf("Warning: variables are not supported with this format\n");
//for (size_t j=0; j<meshData[i].vars.size(); j++)
printf( "Warning: variables are not supported with this format (original)\n" );
// for (size_t j=0; j<meshData[i].vars.size(); j++)
// mesh_entry.variables.push_back( meshData[i].vars[j]->name );
}
const std::string meshClass = mesh->className();
if ( meshClass=="PointList" ) {
if ( meshClass == "PointList" ) {
// List of points
std::shared_ptr<IO::PointList> pointlist = std::dynamic_pointer_cast<IO::PointList>(mesh);
const std::vector<Point>& P = pointlist->points;
for (size_t i=0; i<P.size(); i++) {
std::shared_ptr<IO::PointList> pointlist =
std::dynamic_pointer_cast<IO::PointList>( mesh );
const std::vector<Point> &P = pointlist->points;
for ( size_t i = 0; i < P.size(); i++ ) {
double x[3];
x[0] = P[i].x; x[1] = P[i].y; x[2] = P[i].z;
fwrite(x,sizeof(double),3,fid);
x[0] = P[i].x;
x[1] = P[i].y;
x[2] = P[i].z;
fwrite( x, sizeof( double ), 3, fid );
}
} else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
// Triangle mesh
std::shared_ptr<IO::TriList> trilist = IO::getTriList(mesh);
const std::vector<Point>& A = trilist->A;
const std::vector<Point>& B = trilist->B;
const std::vector<Point>& C = trilist->C;
for (size_t i=0; i<A.size(); i++) {
std::shared_ptr<IO::TriList> trilist = IO::getTriList( mesh );
const std::vector<Point> &A = trilist->A;
const std::vector<Point> &B = trilist->B;
const std::vector<Point> &C = trilist->C;
for ( size_t i = 0; i < A.size(); i++ ) {
double tri[9];
tri[0] = A[i].x; tri[1] = A[i].y; tri[2] = A[i].z;
tri[3] = B[i].x; tri[4] = B[i].y; tri[5] = B[i].z;
tri[6] = C[i].x; tri[7] = C[i].y; tri[8] = C[i].z;
fwrite(tri,sizeof(double),9,fid);
tri[0] = A[i].x;
tri[1] = A[i].y;
tri[2] = A[i].z;
tri[3] = B[i].x;
tri[4] = B[i].y;
tri[5] = B[i].z;
tri[6] = C[i].x;
tri[7] = C[i].y;
tri[8] = C[i].z;
fwrite( tri, sizeof( double ), 9, fid );
}
} else if ( meshClass=="DomainMesh" ) {
} else if ( meshClass == "DomainMesh" ) {
// This format was never supported with the old format
} else {
ERROR("Unknown mesh");
ERROR( "Unknown mesh" );
}
fclose(fid);
fclose( fid );
std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() );
mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() );
meshes_written.push_back(mesh_entry);
mesh_entry.variables.erase(
std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ),
mesh_entry.variables.end() );
meshes_written.push_back( mesh_entry );
}
return meshes_written;
}
// Create the database entry for the mesh data
static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase getDatabase(
const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
char domainname[100];
sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank);
sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank );
// Create the MeshDatabase
IO::MeshDatabase database;
database.name = mesh.meshName;
database.type = meshType(*(mesh.mesh));
database.type = meshType( *( mesh.mesh ) );
database.meshClass = mesh.mesh->className();
database.format = format;
// Write the mesh
@ -134,62 +186,58 @@ static IO::MeshDatabase getDatabase( const std::string& filename, const IO::Mesh
domain.name = domainname;
domain.file = filename;
domain.offset = -1;
database.domains.push_back(domain);
database.domains.push_back( domain );
// Write the variables
for (size_t i=0; i<mesh.vars.size(); i++) {
for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
// Add basic variable info
IO::VariableDatabase info;
info.name = mesh.vars[i]->name;
info.type = mesh.vars[i]->type;
info.dim = mesh.vars[i]->dim;
database.variables.push_back(info);
database.variables.push_back( info );
// Add domain variable info
IO::DatabaseEntry variable;
variable.name = mesh.vars[i]->name;
variable.file = filename;
variable.offset = -1;
std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
database.variable_data.insert(
std::pair<std::pair<std::string,std::string>,IO::DatabaseEntry>(key,variable) );
std::pair<std::pair<std::string, std::string>, IO::DatabaseEntry>( key, variable ) );
}
return database;
}
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
const int level = 0;
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
// Create the MeshDatabase
IO::MeshDatabase database = getDatabase( filename, mesh, format );
IO::MeshDatabase database = getDatabase( filename, mesh, format, rank );
// Write the mesh
IO::DatabaseEntry& domain = database.domains[0];
domain.offset = ftell(fid);
std::pair<size_t,void*> data = mesh.mesh->pack(level);
fprintf(fid,"Mesh: %s-%05i: %lu\n",mesh.meshName.c_str(),rank,data.first);
fwrite(data.second,1,data.first,fid);
fprintf(fid,"\n");
delete [] (char*) data.second;
IO::DatabaseEntry &domain = database.domains[0];
domain.offset = ftell( fid );
std::pair<size_t, void *> data = mesh.mesh->pack( level );
fprintf( fid, "Mesh: %s-%05i: %lu\n", mesh.meshName.c_str(), rank, data.first );
fwrite( data.second, 1, data.first, fid );
fprintf( fid, "\n" );
delete[]( char * ) data.second;
// Write the variables
for (size_t i=0; i<mesh.vars.size(); i++) {
std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
IO::DatabaseEntry& variable = database.variable_data[key];
variable.offset = ftell(fid);
for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
ASSERT( mesh.vars[i]->type != IO::VariableType::NullVariable );
std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
auto &variable = database.variable_data[key];
variable.offset = ftell( fid );
int dim = mesh.vars[i]->dim;
int type = static_cast<int>(mesh.vars[i]->type);
auto type = getString( mesh.vars[i]->type );
size_t N = mesh.vars[i]->data.length();
if ( type == static_cast<int>(IO::VariableType::NullVariable) ) {
ERROR("Variable type not set");
}
size_t N_mesh = mesh.mesh->numberPointsVar(mesh.vars[i]->type);
ASSERT(N==dim*N_mesh);
fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n",
database.name.c_str(), rank, variable.name.c_str(),
dim, type, N_mesh, N*sizeof(double) );
fwrite(mesh.vars[i]->data.data(),sizeof(double),N,fid);
fprintf(fid,"\n");
size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type );
ASSERT( N == dim * N_mesh );
fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank,
variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) );
fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid );
fprintf( fid, "\n" );
}
return database;
}
@ -198,72 +246,74 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
#ifdef USE_SILO
// Write a PointList mesh (and variables) to a file
template<class TYPE>
static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname )
static void writeSiloPointMesh(
DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
{
const auto& points = mesh.getPoints();
std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.getPoints();
std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
}
static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloPointList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::PointList& mesh = dynamic_cast<IO::PointList&>( *meshData.mesh );
const IO::PointList &mesh = dynamic_cast<IO::PointList &>( *meshData.mesh );
const std::string meshname = database.domains[0].name;
if ( meshData.precision == IO::DataType::Double ) {
writeSiloPointMesh<double>( fid, mesh, meshname );
} else if ( meshData.precision == IO::DataType::Float ) {
writeSiloPointMesh<float>( fid, mesh, meshname );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
const auto& points = mesh.getPoints();
std::vector<double> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.getPoints();
std::vector<double> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const double *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh( fid, meshname, 3, points.size(), coords );
for (size_t i=0; i<meshData.vars.size(); i++) {
const IO::Variable& var = *meshData.vars[i];
IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writePointMeshVariable( fid, meshname, var.name, var.data );
IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
// Write a TriMesh mesh (and variables) to a file
template<class TYPE>
static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::string& meshname )
static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname )
{
const auto& points = mesh.vertices->getPoints();
std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.vertices->getPoints();
std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
}
static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
const IO::TriMesh& mesh, IO::MeshDatabase database )
static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
const IO::TriMesh &mesh, IO::MeshDatabase database )
{
const std::string meshname = database.domains[0].name;
if ( meshData.precision == IO::DataType::Double ) {
@ -271,238 +321,240 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
} else if ( meshData.precision == IO::DataType::Float ) {
writeSiloTriMesh<float>( fid, mesh, meshname );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
for (size_t i=0; i<meshData.vars.size(); i++) {
const IO::Variable& var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::TriMesh& mesh = dynamic_cast<IO::TriMesh&>( *meshData.mesh );
const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
writeSiloTriMesh2( fid, meshData, mesh, database );
}
static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
auto mesh = getTriMesh( meshData.mesh );
writeSiloTriMesh2( fid, meshData, *mesh, database );
}
// Write a DomainMesh mesh (and variables) to a file
static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloDomainMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::DomainMesh& mesh = dynamic_cast<IO::DomainMesh&>( *meshData.mesh );
const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
std::array<double,6> range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx,
info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny,
info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz };
std::array<int,3> N = { mesh.nx, mesh.ny, mesh.nz };
std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
( info.kz + 1 ) * mesh.Lz / info.nz };
std::array<int, 3> N = { mesh.nx, mesh.ny, mesh.nz };
auto meshname = database.domains[0].name;
silo::writeUniformMesh<3>( fid, meshname, range, N );
silo::write<int>( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
for (size_t i=0; i<meshData.vars.size(); i++) {
const auto& var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
IO::silo::write<int>(
fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const auto &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename,
const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
// Create the MeshDatabase
auto database = getDatabase( filename, mesh, format );
if ( database.meshClass=="PointList" ) {
auto database = getDatabase( filename, mesh, format, rank );
if ( database.meshClass == "PointList" ) {
writeSiloPointList( fid, mesh, database );
} else if ( database.meshClass=="TriMesh" ) {
} else if ( database.meshClass == "TriMesh" ) {
writeSiloTriMesh( fid, mesh, database );
} else if ( database.meshClass=="TriList" ) {
} else if ( database.meshClass == "TriList" ) {
writeSiloTriList( fid, mesh, database );
} else if ( database.meshClass=="DomainMesh" ) {
} else if ( database.meshClass == "DomainMesh" ) {
writeSiloDomainMesh( fid, mesh, database );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
return database;
}
// Write the summary file for silo
std::pair<int,int> getSiloMeshType( const std::string& meshClass )
std::pair<int, int> getSiloMeshType( const std::string &meshClass )
{
int meshType = 0;
int varType = 0;
if ( meshClass=="PointList" ) {
if ( meshClass == "PointList" ) {
meshType = DB_POINTMESH;
varType = DB_POINTVAR;
} else if ( meshClass=="TriMesh" ) {
} else if ( meshClass == "TriMesh" ) {
meshType = DB_UCDMESH;
varType = DB_UCDVAR;
} else if ( meshClass=="TriList" ) {
} else if ( meshClass == "TriList" ) {
meshType = DB_UCDMESH;
varType = DB_UCDVAR;
} else if ( meshClass=="DomainMesh" ) {
} else if ( meshClass == "DomainMesh" ) {
meshType = DB_QUAD_RECT;
varType = DB_QUADVAR;
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
return std::make_pair( meshType, varType );
}
void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, const std::string& filename )
void writeSiloSummary(
const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
{
auto fid = silo::open( filename, silo::CREATE );
for ( const auto& data : meshes_written ) {
auto fid = IO::silo::open( filename, IO::silo::CREATE );
for ( const auto &data : meshes_written ) {
auto type = getSiloMeshType( data.meshClass );
std::vector<int> meshTypes( data.domains.size(), type.first );
std::vector<int> varTypes( data.domains.size(), type.second );
std::vector<std::string> meshNames;
for ( const auto& tmp : data.domains )
for ( const auto &tmp : data.domains )
meshNames.push_back( tmp.file + ":" + tmp.name );
silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
for (const auto& variable : data.variables ) {
IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
for ( const auto &variable : data.variables ) {
std::vector<std::string> varnames;
for ( const auto& tmp : data.domains )
for ( const auto &tmp : data.domains )
varnames.push_back( tmp.file + ":" + variable.name );
silo::writeMultiVar( fid, variable.name, varnames, varTypes );
IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
}
}
silo::close( fid );
IO::silo::close( fid );
}
#endif
// Write the mesh data in the new format
static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i",rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
FILE *fid = fopen(fullpath,"wb");
for (size_t i=0; i<meshData.size(); i++) {
sprintf( filename, "%05i", rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
FILE *fid = fopen( fullpath, "wb" );
for ( size_t i = 0; i < meshData.size(); i++ ) {
std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
meshes_written.push_back( write_domain(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) );
}
fclose(fid);
fclose( fid );
return meshes_written;
}
// Write the mesh data to silo
static std::vector<IO::MeshDatabase> writeMeshesSilo(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
#ifdef USE_SILO
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i.silo",rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
auto fid = silo::open( fullpath, silo::CREATE );
for (size_t i=0; i<meshData.size(); i++) {
sprintf( filename, "%05i.silo", rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
for ( size_t i = 0; i < meshData.size(); i++ ) {
auto mesh = meshData[i].mesh;
meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
}
silo::close( fid );
IO::silo::close( fid );
return meshes_written;
#else
ERROR("Application built without silo support");
NULL_USE( meshData );
NULL_USE( path );
NULL_USE( format );
NULL_USE( rank );
ERROR( "Application built without silo support" );
return std::vector<IO::MeshDatabase>();
#endif
}
/****************************************************
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm )
{
if ( global_IO_path.empty() )
IO::initialize( );
PROFILE_START("writeData");
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
IO::initialize();
PROFILE_START( "writeData" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
// Check the meshData before writing
for ( const auto& data : meshData ) {
if ( !data.check() )
ERROR("Error in meshData");
}
for ( const auto &data : meshData )
ASSERT( data.check() );
// Create the output directory
std::string path = global_IO_path + "/" + subdir;
if ( rank == 0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
}
comm.barrier();
recursiveMkdir( path, S_IRWXU | S_IRGRP );
// Write the mesh files
std::vector<IO::MeshDatabase> meshes_written;
if ( global_IO_format == Format::OLD ) {
// Write the original triangle format
meshes_written = writeMeshesOrigFormat( meshData, path );
meshes_written = writeMeshesOrigFormat( meshData, path, rank );
} else if ( global_IO_format == Format::NEW ) {
// Write the new format (double precision)
meshes_written = writeMeshesNewFormat( meshData, path, 2 );
meshes_written = writeMeshesNewFormat( meshData, path, IO::FileFormat::NEW, rank );
} else if ( global_IO_format == Format::SILO ) {
// Write silo
meshes_written = writeMeshesSilo( meshData, path, 4 );
meshes_written = writeMeshesSilo( meshData, path, IO::FileFormat::SILO, rank );
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
// Gather a complete list of files on rank 0
meshes_written = gatherAll(meshes_written,comm);
meshes_written = gatherAll( meshes_written, comm );
// Write the summary files
if ( rank == 0 ) {
// Write the summary file for the current timestep
char filename[200];
sprintf(filename,"%s/LBM.summary",path.c_str());
write(meshes_written,filename);
// Write summary silo file if needed
#ifdef USE_SILO
sprintf( filename, "%s/LBM.summary", path.c_str() );
write( meshes_written, filename );
// Write summary silo file if needed
#ifdef USE_SILO
if ( global_IO_format == Format::SILO ) {
sprintf(filename,"%s/summary.silo",path.c_str());
writeSiloSummary(meshes_written,filename);
sprintf( filename, "%s/summary.silo", path.c_str() );
writeSiloSummary( meshes_written, filename );
}
#endif
#endif
// Add the timestep to the global summary file
if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) {
auto filename = global_IO_path+"/summary.LBM";
FILE *fid = fopen(filename.c_str(),"ab");
fprintf(fid,"%s/\n",subdir.c_str());
fclose(fid);
auto filename = global_IO_path + "/summary.LBM";
FILE *fid = fopen( filename.c_str(), "ab" );
fprintf( fid, "%s/\n", subdir.c_str() );
fclose( fid );
} else if ( global_IO_format == Format::SILO ) {
auto filename = global_IO_path+"/LBM.visit";
FILE *fid = fopen(filename.c_str(),"ab");
fprintf(fid,"%s/summary.silo\n",subdir.c_str());
fclose(fid);
auto filename = global_IO_path + "/LBM.visit";
FILE *fid = fopen( filename.c_str(), "ab" );
fprintf( fid, "%s/summary.silo\n", subdir.c_str() );
fclose( fid );
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
}
PROFILE_STOP("writeData");
PROFILE_STOP( "writeData" );
}

25
IO/Writer.h
View File

@ -14,17 +14,18 @@ namespace IO {
/*!
* @brief Initialize the writer
* @details This function initializes the writer to the given path. All subsequent
* writes will occur in this directory. If this is not called, then it will default
* to the current path.
* @details This function initializes the writer to the given path.
* All subsequent writes will occur in this directory.
* If this is not called, then it will default to the current path.
* @param[in] path The path to use for writes
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process
* silo - Silo
* old - Old mesh format
* (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process silo - Silo
* @param[in] append Append any existing data (default is false)
*/
void initialize( const std::string& path="", const std::string& format="silo", bool append=false );
void initialize(
const std::string &path = "", const std::string &format = "silo", bool append = false );
/*!
@ -34,7 +35,8 @@ void initialize( const std::string& path="", const std::string& format="silo", b
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm );
void writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm );
/*!
@ -44,14 +46,15 @@ void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
inline void writeData(
int timestep, const std::vector<IO::MeshDataStruct> &meshData, const Utilities::MPI &comm )
{
char subdir[100];
sprintf(subdir,"vis%03i",timestep);
sprintf( subdir, "vis%03i", timestep );
writeData( subdir, meshData, comm );
}
} // IO namespace
} // namespace IO
#endif

347
IO/netcdf.cpp
View File

@ -1,6 +1,6 @@
#include "IO/netcdf.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -19,7 +19,7 @@
msg += nc_strerror( ERR ); \
ERROR( msg ); \
} \
} while (0)
} while ( 0 )
namespace netcdf {
@ -50,43 +50,64 @@ static inline VariableType convertType( nc_type type )
else if ( type == NC_DOUBLE )
type2 = DOUBLE;
else
ERROR("Unknown type");
ERROR( "Unknown type" );
return type2;
}
// Get nc_type from the template
template<class T> inline nc_type getType();
template<> inline nc_type getType<char>() { return NC_CHAR; }
template<> inline nc_type getType<short>() { return NC_SHORT; }
template<> inline nc_type getType<int>() { return NC_INT; }
template<> inline nc_type getType<float>() { return NC_FLOAT; }
template<> inline nc_type getType<double>() { return NC_DOUBLE; }
template<class T>
inline nc_type getType();
template<>
inline nc_type getType<char>()
{
return NC_CHAR;
}
template<>
inline nc_type getType<short>()
{
return NC_SHORT;
}
template<>
inline nc_type getType<int>()
{
return NC_INT;
}
template<>
inline nc_type getType<float>()
{
return NC_FLOAT;
}
template<>
inline nc_type getType<double>()
{
return NC_DOUBLE;
}
// Function to reverse an array
template<class TYPE>
inline std::vector<TYPE> reverse( const std::vector<TYPE>& x )
inline std::vector<TYPE> reverse( const std::vector<TYPE> &x )
{
std::vector<TYPE> y(x.size());
for (size_t i=0; i<x.size(); i++)
y[i] = x[x.size()-i-1];
std::vector<TYPE> y( x.size() );
for ( size_t i = 0; i < x.size(); i++ )
y[i] = x[x.size() - i - 1];
return y;
}
// Function to reverse an array
template<class TYPE1, class TYPE2>
inline std::vector<TYPE2> convert( const std::vector<TYPE1>& x )
inline std::vector<TYPE2> convert( const std::vector<TYPE1> &x )
{
std::vector<TYPE2> y(x.size());
for (size_t i=0; i<x.size(); i++)
y[i] = static_cast<TYPE2>(x[i]);
std::vector<TYPE2> y( x.size() );
for ( size_t i = 0; i < x.size(); i++ )
y[i] = static_cast<TYPE2>( x[i] );
return y;
}
/****************************************************
* Convert the VariableType to a string *
****************************************************/
* Convert the VariableType to a string *
****************************************************/
std::string VariableTypeName( VariableType type )
{
if ( type == BYTE )
@ -114,9 +135,9 @@ std::string VariableTypeName( VariableType type )
/****************************************************
* Open/close a file *
****************************************************/
int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm )
* Open/close a file *
****************************************************/
int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm )
{
int fid = 0;
if ( comm.isNull() ) {
@ -127,23 +148,26 @@ int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm
int err = nc_open( filename.c_str(), NC_WRITE, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create( filename.c_str(), NC_SHARE|NC_64BIT_OFFSET, &fid );
int err = nc_create( filename.c_str(), NC_SHARE | NC_64BIT_OFFSET, &fid );
CHECK_NC_ERR( err );
} else {
ERROR("Unknown file mode");
ERROR( "Unknown file mode" );
}
} else {
if ( mode == READ ) {
int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_open_par(
filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == WRITE ) {
int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_open_par( filename.c_str(), NC_WRITE | NC_MPIPOSIX, comm.getCommunicator(),
MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_create_par( filename.c_str(), NC_NETCDF4 | NC_MPIIO,
comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else {
ERROR("Unknown file mode");
ERROR( "Unknown file mode" );
}
}
return fid;
@ -152,42 +176,42 @@ void close( int fid )
{
int err = nc_close( fid );
if ( err != NC_NOERR )
ERROR("Error closing file");
ERROR( "Error closing file" );
}
/****************************************************
* Query basic properties *
****************************************************/
* Query basic properties *
****************************************************/
static std::vector<size_t> getDimVar( int fid, int varid )
{
int ndim = 0;
int err = nc_inq_varndims( fid, varid, &ndim );
CHECK_NC_ERR( err );
std::vector<size_t> dims(ndim,0);
int dimid[64] = {-1};
std::vector<size_t> dims( ndim, 0 );
int dimid[64] = { -1 };
err = nc_inq_vardimid( fid, varid, dimid );
CHECK_NC_ERR( err );
for (int i=0; i<ndim; i++) {
for ( int i = 0; i < ndim; i++ ) {
err = nc_inq_dimlen( fid, dimid[i], &dims[i] );
CHECK_NC_ERR( err );
}
return dims;
}
static int getVarID( int fid, const std::string& var )
static int getVarID( int fid, const std::string &var )
{
int id = -1;
int err = nc_inq_varid( fid, var.c_str(), &id );
CHECK_NC_ERR( err );
return id;
}
std::vector<size_t> getVarDim( int fid, const std::string& var )
std::vector<size_t> getVarDim( int fid, const std::string &var )
{
return getDimVar( fid, getVarID( fid, var ) );
}
std::vector<size_t> getAttDim( int fid, const std::string& att )
std::vector<size_t> getAttDim( int fid, const std::string &att )
{
std::vector<size_t> dim(1,0);
std::vector<size_t> dim( 1, 0 );
int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
CHECK_NC_ERR( err );
return dim;
@ -197,9 +221,9 @@ std::vector<std::string> getVarNames( int fid )
int nvar;
int err = nc_inq( fid, NULL, &nvar, NULL, NULL );
CHECK_NC_ERR( err );
std::vector<std::string> vars(nvar);
for (int i=0; i<nvar; i++) {
char name[NC_MAX_NAME+1];
std::vector<std::string> vars( nvar );
for ( int i = 0; i < nvar; i++ ) {
char name[NC_MAX_NAME + 1];
err = nc_inq_varname( fid, i, name );
CHECK_NC_ERR( err );
vars[i] = name;
@ -211,234 +235,239 @@ std::vector<std::string> getAttNames( int fid )
int natt;
int err = nc_inq( fid, NULL, NULL, &natt, NULL );
CHECK_NC_ERR( err );
std::vector<std::string> att(natt);
for (int i=0; i<natt; i++) {
char name[NC_MAX_NAME+1];
std::vector<std::string> att( natt );
for ( int i = 0; i < natt; i++ ) {
char name[NC_MAX_NAME + 1];
err = nc_inq_attname( fid, NC_GLOBAL, i, name );
CHECK_NC_ERR( err );
att[i] = name;
}
return att;
}
VariableType getVarType( int fid, const std::string& var )
VariableType getVarType( int fid, const std::string &var )
{
int varid = -1;
int err = nc_inq_varid( fid, var.c_str(), &varid );
CHECK_NC_ERR( err );
nc_type type=0;
nc_type type = 0;
err = nc_inq_vartype( fid, varid, &type );
CHECK_NC_ERR( err );
return convertType(type);
return convertType( type );
}
VariableType getAttType( int fid, const std::string& att )
VariableType getAttType( int fid, const std::string &att )
{
nc_type type=0;
nc_type type = 0;
int err = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type );
CHECK_NC_ERR( err );
return convertType(type);
return convertType( type );
}
/****************************************************
* Read a variable *
****************************************************/
* Read a variable *
****************************************************/
template<>
Array<unsigned short> getVar<unsigned short>( int fid, const std::string& var )
Array<unsigned short> getVar<unsigned short>( int fid, const std::string &var )
{
PROFILE_START("getVar<unsigned short>");
Array<unsigned short> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_ushort( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<unsigned short>" );
Array<unsigned short> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_ushort( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<unsigned short>");
PROFILE_STOP( "getVar<unsigned short>" );
return x.reverseDim();
}
template<>
Array<short> getVar<short>( int fid, const std::string& var )
Array<short> getVar<short>( int fid, const std::string &var )
{
PROFILE_START("getVar<short>");
Array<short> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_short( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<short>" );
Array<short> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_short( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<short>");
PROFILE_STOP( "getVar<short>" );
return x.reverseDim();
}
template<>
Array<unsigned int> getVar<unsigned int>( int fid, const std::string& var )
Array<unsigned int> getVar<unsigned int>( int fid, const std::string &var )
{
PROFILE_START("getVar<unsigned int>");
Array<unsigned int> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_uint( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<unsigned int>" );
Array<unsigned int> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_uint( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<unsigned int>");
PROFILE_STOP( "getVar<unsigned int>" );
return x.reverseDim();
}
template<>
Array<int> getVar<int>( int fid, const std::string& var )
Array<int> getVar<int>( int fid, const std::string &var )
{
PROFILE_START("getVar<int>");
Array<int> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_int( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<int>" );
Array<int> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_int( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<int>");
PROFILE_STOP( "getVar<int>" );
return x.reverseDim();
}
template<>
Array<float> getVar<float>( int fid, const std::string& var )
Array<float> getVar<float>( int fid, const std::string &var )
{
PROFILE_START("getVar<float>");
Array<float> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_float( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<float>" );
Array<float> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_float( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<float>");
PROFILE_STOP( "getVar<float>" );
return x.reverseDim();
}
template<>
Array<double> getVar<double>( int fid, const std::string& var )
Array<double> getVar<double>( int fid, const std::string &var )
{
PROFILE_START("getVar<double>");
Array<double> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_double( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<double>" );
Array<double> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_double( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<double>");
PROFILE_STOP( "getVar<double>" );
return x.reverseDim();
}
template<>
Array<char> getVar<char>( int fid, const std::string& var )
Array<char> getVar<char>( int fid, const std::string &var )
{
PROFILE_START("getVar<char>");
Array<char> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_text( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<char>" );
Array<char> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_text( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<char>");
PROFILE_STOP( "getVar<char>" );
return x.reverseDim();
}
template<>
Array<std::string> getVar<std::string>( int fid, const std::string& var )
Array<std::string> getVar<std::string>( int fid, const std::string &var )
{
PROFILE_START("getVar<std::string>");
PROFILE_START( "getVar<std::string>" );
Array<char> tmp = getVar<char>( fid, var );
std::vector<size_t> dim = {tmp.size(0), tmp.size(1), tmp.size(2) };
std::vector<size_t> dim = { tmp.size( 0 ), tmp.size( 1 ), tmp.size( 2 ) };
if ( dim.size() == 1 )
dim[0] = 1;
else
dim.erase( dim.begin() );
Array<std::string> text(dim);
for (size_t i=0; i<text.length(); i++)
text(i) = &(tmp(0,i));
PROFILE_STOP("getVar<std::string>");
Array<std::string> text( dim );
for ( size_t i = 0; i < text.length(); i++ )
text( i ) = &( tmp( 0, i ) );
PROFILE_STOP( "getVar<std::string>" );
return text;
}
static inline void get_stride_args( const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride,
size_t *startp, size_t *countp, ptrdiff_t *stridep )
static inline void get_stride_args( const std::vector<int> &start, const std::vector<int> &count,
const std::vector<int> &stride, size_t *startp, size_t *countp, ptrdiff_t *stridep )
{
for (size_t i=0; i<start.size(); i++)
for ( size_t i = 0; i < start.size(); i++ )
startp[i] = start[i];
for (size_t i=0; i<count.size(); i++)
for ( size_t i = 0; i < count.size(); i++ )
countp[i] = count[i];
for (size_t i=0; i<stride.size(); i++)
for ( size_t i = 0; i < stride.size(); i++ )
stridep[i] = stride[i];
}
template<class TYPE>
int nc_get_vars_TYPE( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], TYPE *ptr );
int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], TYPE *ptr );
template<>
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], short *ptr )
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], short *ptr )
{
return nc_get_vars_short( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], int *ptr )
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], int *ptr )
{
return nc_get_vars_int( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], float *ptr )
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], float *ptr )
{
return nc_get_vars_float( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], double *ptr )
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], double *ptr )
{
return nc_get_vars_double( fid, varid, start, count, stride, ptr );
}
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride )
Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
const std::vector<int> &count, const std::vector<int> &stride )
{
PROFILE_START("getVar<> (strided)");
PROFILE_START( "getVar<> (strided)" );
std::vector<size_t> var_size = getVarDim( fid, var );
for (int d=0; d<(int)var_size.size(); d++) {
if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) {
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
for ( int d = 0; d < (int) var_size.size(); d++ ) {
if ( start[d] < 0 || start[d] + stride[d] * ( count[d] - 1 ) > (int) var_size[d] ) {
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
char tmp[1000];
sprintf(tmp,"%i: Range exceeded array dimension:\n"
sprintf( tmp,
"%i: Range exceeded array dimension:\n"
" start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",
rank,d,start[d],d,count[d],d,stride[d],d,(int)var_size[d]);
ERROR(tmp);
rank, d, start[d], d, count[d], d, stride[d], d, (int) var_size[d] );
ERROR( tmp );
}
}
Array<TYPE> x( reverse(convert<int,size_t>(count)) );
Array<TYPE> x( reverse( convert<int, size_t>( count ) ) );
size_t startp[10], countp[10];
ptrdiff_t stridep[10];
get_stride_args( start, count, stride, startp, countp, stridep );
int err = nc_get_vars_TYPE<TYPE>( fid, getVarID(fid,var), startp, countp, stridep, x.data() );
int err =
nc_get_vars_TYPE<TYPE>( fid, getVarID( fid, var ), startp, countp, stridep, x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<> (strided)");
PROFILE_STOP( "getVar<> (strided)" );
return x.reverseDim();
}
template Array<short> getVar<short>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<int> getVar<int>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<float> getVar<float>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<double> getVar<double>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<short> getVar<short>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<int> getVar<int>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<float> getVar<float>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<double> getVar<double>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
/****************************************************
* Read an attribute *
****************************************************/
* Read an attribute *
****************************************************/
template<>
Array<double> getAtt<double>( int fid, const std::string& att )
Array<double> getAtt<double>( int fid, const std::string &att )
{
PROFILE_START("getAtt<double>");
Array<double> x( getAttDim(fid,att) );
PROFILE_START( "getAtt<double>" );
Array<double> x( getAttDim( fid, att ) );
int err = nc_get_att_double( fid, NC_GLOBAL, att.c_str(), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getAtt<double>");
PROFILE_STOP( "getAtt<double>" );
return x;
}
template<>
Array<std::string> getAtt<std::string>( int fid, const std::string& att )
Array<std::string> getAtt<std::string>( int fid, const std::string &att )
{
PROFILE_START("getAtt<std::string>");
char *tmp = new char[getAttDim(fid,att)[0]];
Array<std::string> x(1);
x(0) = tmp;
delete [] tmp;
PROFILE_STOP("getAtt<std::string>");
PROFILE_START( "getAtt<std::string>" );
char *tmp = new char[getAttDim( fid, att )[0]];
Array<std::string> x( 1 );
x( 0 ) = tmp;
delete[] tmp;
PROFILE_STOP( "getAtt<std::string>" );
return x;
}
/****************************************************
* Write an array to a file *
****************************************************/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims )
* Write an array to a file *
****************************************************/
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims )
{
std::vector<int> dimid(names.size(),0);
for (size_t i=0; i<names.size(); i++) {
int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i]);
std::vector<int> dimid( names.size(), 0 );
for ( size_t i = 0; i < names.size(); i++ ) {
int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i] );
CHECK_NC_ERR( err );
}
return dimid;
}
template<class TYPE>
void write( int fid, const std::string& var, const std::vector<int>& dimids,
const Array<TYPE>& data, const RankInfoStruct& info )
void write( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<TYPE> &data, const RankInfoStruct &info )
{
// Define the variable
int varid = 0;
@ -452,21 +481,23 @@ void write( int fid, const std::string& var, const std::vector<int>& dimids,
CHECK_NC_ERR( err );
// parallel write: each process writes its subarray to the file
auto x = data.reverseDim();
std::vector<size_t> count = { data.size(0), data.size(1), data.size(2) };
std::vector<size_t> start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) };
std::vector<size_t> count = { data.size( 0 ), data.size( 1 ), data.size( 2 ) };
std::vector<size_t> start = { info.ix * data.size( 0 ), info.jy * data.size( 1 ),
info.kz * data.size( 2 ) };
nc_put_vara( fid, varid, start.data(), count.data(), x.data() );
}
template void write<short>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<short>& data, const RankInfoStruct& info );
template void write<int>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<int>& data, const RankInfoStruct& info );
template void write<float>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<float>& data, const RankInfoStruct& info );
template void write<double>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<double>& data, const RankInfoStruct& info );
template void write<short>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<short> &data, const RankInfoStruct &info );
template void write<int>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<int> &data, const RankInfoStruct &info );
template void write<float>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<float> &data, const RankInfoStruct &info );
template void write<double>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<double> &data, const RankInfoStruct &info );
}; // netcdf namespace
}; // namespace netcdf
#else
#endif

51
IO/netcdf.h
View File

@ -5,9 +5,8 @@
#include <vector>
#include "common/Array.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/MPI.h"
namespace netcdf {
@ -31,15 +30,15 @@ std::string VariableTypeName( VariableType type );
* @param filename File to open
* @param mode Open the file for reading or writing
* @param comm MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf)
*/
int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm=MPI_COMM_NULL );
*/
int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm = MPI_COMM_NULL );
/*!
* @brief Close netcdf file
* @details This function closes a netcdf file
* @param fid Handle to the open file
*/
*/
void close( int fid );
@ -47,7 +46,7 @@ void close( int fid );
* @brief Read the variable names
* @details This function reads a list of the variable names in the file
* @param fid Handle to the open file
*/
*/
std::vector<std::string> getVarNames( int fid );
@ -55,7 +54,7 @@ std::vector<std::string> getVarNames( int fid );
* @brief Read the attribute names
* @details This function reads a list of the attribute names in the file
* @param fid Handle to the open file
*/
*/
std::vector<std::string> getAttNames( int fid );
@ -64,8 +63,8 @@ std::vector<std::string> getAttNames( int fid );
* @details This function returns the type for a variable
* @param fid Handle to the open file
* @param var Variable to read
*/
VariableType getVarType( int fid, const std::string& var );
*/
VariableType getVarType( int fid, const std::string &var );
/*!
@ -73,8 +72,8 @@ VariableType getVarType( int fid, const std::string& var );
* @details This function returns the type for an attribute
* @param fid Handle to the open file
* @param att Attribute to read
*/
VariableType getAttType( int fid, const std::string& att );
*/
VariableType getAttType( int fid, const std::string &att );
/*!
@ -82,8 +81,8 @@ VariableType getAttType( int fid, const std::string& att );
* @details This function returns the die for a variable
* @param fid Handle to the open file
* @param var Variable to read
*/
std::vector<size_t> getVarDim( int fid, const std::string& var );
*/
std::vector<size_t> getVarDim( int fid, const std::string &var );
/*!
@ -91,9 +90,9 @@ std::vector<size_t> getVarDim( int fid, const std::string& var );
* @details This function reads a variable with the given name from the file
* @param fid Handle to the open file
* @param var Variable to read
*/
*/
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var );
Array<TYPE> getVar( int fid, const std::string &var );
/*!
@ -104,10 +103,10 @@ Array<TYPE> getVar( int fid, const std::string& var );
* @param start Starting corner for the read
* @param count Number of elements to read
* @param stride Stride size for the read
*/
*/
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride );
Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
const std::vector<int> &count, const std::vector<int> &stride );
/*!
@ -115,27 +114,29 @@ Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& sta
* @details This function reads an attribute with the given name from the file
* @param fid Handle to the open file
* @param att Attribute to read
*/
*/
template<class TYPE>
Array<TYPE> getAtt( int fid, const std::string& att );
Array<TYPE> getAtt( int fid, const std::string &att );
/*!
* @brief Write the dimensions
* @details This function writes the grid dimensions to netcdf.
* @param fid Handle to the open file
*/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims );
*/
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims );
/*!
* @brief Write a variable
* @details This function writes a variable to netcdf.
* @param fid Handle to the open file
*/
*/
template<class TYPE>
void write( int fid, const std::string& var, const std::vector<int>& dimids, const Array<TYPE>& data, const RankInfoStruct& rank_info );
void write( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<TYPE> &data, const RankInfoStruct &rank_info );
}; // netcdf namespace
}; // namespace netcdf
#endif

87
IO/silo.cpp
View File

@ -1,6 +1,6 @@
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -10,14 +10,13 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
* Open/close a file *
****************************************************/
DBfile* open( const std::string& filename, FileMode mode )
* Open/close a file *
****************************************************/
DBfile *open( const std::string &filename, FileMode mode )
{
DBfile *fid = nullptr;
if ( mode == CREATE ) {
@ -29,82 +28,78 @@ DBfile* open( const std::string& filename, FileMode mode )
}
return fid;
}
void close( DBfile* fid )
{
DBClose( fid );
}
void close( DBfile *fid ) { DBClose( fid ); }
/****************************************************
* Helper functions *
****************************************************/
VariableDataType varDataType( DBfile *fid, const std::string& name )
* Helper functions *
****************************************************/
DataType varDataType( DBfile *fid, const std::string &name )
{
auto type = DBGetVarType( fid, name.c_str() );
VariableDataType type2 = VariableDataType::UNKNOWN;
DataType type2 = DataType::Null;
if ( type == DB_DOUBLE )
type2 = VariableDataType::DOUBLE;
type2 = DataType::Double;
else if ( type == DB_FLOAT )
type2 = VariableDataType::FLOAT;
type2 = DataType::Float;
else if ( type == DB_INT )
type2 = VariableDataType::INT;
type2 = DataType::Int;
return type2;
}
/****************************************************
* Write/read a uniform mesh to silo *
****************************************************/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N )
* Write/read a uniform mesh to silo *
****************************************************/
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N )
{
DBquadmesh* mesh = DBGetQuadmesh( fid, meshname.c_str() );
DBquadmesh *mesh = DBGetQuadmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
range.resize(2*ndim);
N.resize(ndim);
for (int d=0; d<ndim; d++) {
N[d] = mesh->dims[d]-1;
range[2*d+0] = mesh->min_extents[d];
range[2*d+1] = mesh->max_extents[d];
range.resize( 2 * ndim );
N.resize( ndim );
for ( int d = 0; d < ndim; d++ ) {
N[d] = mesh->dims[d] - 1;
range[2 * d + 0] = mesh->min_extents[d];
range[2 * d + 1] = mesh->max_extents[d];
}
DBFreeQuadmesh( mesh );
}
/****************************************************
* Write a multimesh *
****************************************************/
void writeMultiMesh( DBfile* fid, const std::string& meshname,
const std::vector<std::string>& meshNames,
const std::vector<int>& meshTypes )
* Write a multimesh *
****************************************************/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string> &meshNames, const std::vector<int> &meshTypes )
{
std::vector<char*> meshnames(meshNames.size());
std::vector<char *> meshnames( meshNames.size() );
for ( size_t i = 0; i < meshNames.size(); ++i )
meshnames[i] = (char *) meshNames[i].c_str();
std::string tree_name = meshname + "_tree";
DBoptlist *optList = DBMakeOptlist( 1 );
DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(),
(int *) meshTypes.data(), nullptr );
DBFreeOptlist( optList );
}
/****************************************************
* Write a multivariable *
****************************************************/
void writeMultiVar( DBfile* fid, const std::string& varname,
const std::vector<std::string>& varNames,
const std::vector<int>& varTypes )
* Write a multivariable *
****************************************************/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &varNames, const std::vector<int> &varTypes )
{
std::vector<char*> varnames(varNames.size(),nullptr);
for (size_t j=0; j<varNames.size(); j++)
varnames[j] = const_cast<char*>(varNames[j].c_str());
DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr );
std::vector<char *> varnames( varNames.size(), nullptr );
for ( size_t j = 0; j < varNames.size(); j++ )
varnames[j] = const_cast<char *>( varNames[j].c_str() );
DBPutMultivar(
fid, varname.c_str(), varNames.size(), varnames.data(), (int *) varTypes.data(), nullptr );
}
}; // silo namespace
}; // namespace IO::silo
#else

118
IO/silo.h
View File

@ -1,32 +1,28 @@
#ifndef SILO_INTERFACE
#define SILO_INTERFACE
#include <array>
#include <string>
#include <vector>
#include <array>
#include "IO/Mesh.h"
#include "common/Array.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/MPI.h"
#ifdef USE_SILO
#include <silo.h>
#include <silo.h>
#else
typedef int DBfile;
typedef int DBfile;
#endif
namespace silo {
namespace IO::silo {
enum FileMode { READ, WRITE, CREATE };
enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
/*!
* @brief Open silo file
@ -34,16 +30,16 @@ enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
* @param[in] filename File to open
* @param[in] mode Open the file for reading or writing
* @return This function returns a handle to the file
*/
DBfile* open( const std::string& filename, FileMode mode );
*/
DBfile *open( const std::string &filename, FileMode mode );
/*!
* @brief Close silo file
* @details This function closes a silo file
* @param[in] fid Handle to the open file
*/
void close( DBfile* fid );
*/
void close( DBfile *fid );
/*!
@ -51,8 +47,8 @@ void close( DBfile* fid );
* @details This function returns the type of variable data
* @param[in] fid Handle to the open file
* @param[in] name Name of variable
*/
VariableDataType varDataType( DBfile *dbfile, const std::string& name );
*/
DataType varDataType( DBfile *dbfile, const std::string &name );
/*!
@ -61,9 +57,9 @@ VariableDataType varDataType( DBfile *dbfile, const std::string& name );
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @param[in] data Data to write
*/
*/
template<class TYPE>
void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data );
void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data );
/*!
@ -72,9 +68,9 @@ void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& da
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Data read
*/
*/
template<class TYPE>
std::vector<TYPE> read( DBfile* fid, const std::string& varname );
std::vector<TYPE> read( DBfile *fid, const std::string &varname );
/*!
@ -84,10 +80,10 @@ std::vector<TYPE> read( DBfile* fid, const std::string& varname );
* @param[in] meshname Mesh name
* @param[in] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
* @param[in] N Number of cells in each direction
*/
*/
template<int NDIM>
void writeUniformMesh( DBfile* fid, const std::string& meshname,
const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N );
void writeUniformMesh( DBfile *fid, const std::string &meshname,
const std::array<double, 2 * NDIM> &range, const std::array<int, NDIM> &N );
/*!
@ -97,9 +93,9 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname,
* @param[in] meshname Mesh name
* @param[out] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
* @param[out] N Number of cells in each direction
*/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N );
*/
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N );
/*!
@ -111,10 +107,11 @@ void readUniformMesh( DBfile* fid, const std::string& meshname,
* @param[in] varname Variable name
* @param[in] data Variable data
* @param[in] type Variable type
*/
template< int NDIM, class TYPE >
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type );
*/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type );
/*!
@ -123,9 +120,9 @@ void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const s
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname );
/*!
@ -136,10 +133,10 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] ndim Number of dimensions
* @param[in] N Number of points
* @param[in] coords Coordinates of the points
*/
*/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] );
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] );
/*!
@ -148,9 +145,9 @@ void writePointMesh( DBfile* fid, const std::string& meshname,
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @return Returns the coordinates as a N x ndim array
*/
*/
template<class TYPE>
Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
Array<TYPE> readPointMesh( DBfile *fid, const std::string &meshname );
/*!
@ -160,10 +157,10 @@ Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
* @param[in] meshname Mesh name
* @param[in] varname Variable name
* @param[in] data Variable data
*/
*/
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data );
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data );
/*!
@ -172,9 +169,9 @@ void writePointMeshVariable( DBfile* fid, const std::string& meshname,
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readPointMeshVariable( DBfile *fid, const std::string &varname );
/*!
@ -188,10 +185,10 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] coords Coordinates of the points
* @param[in] N_tri Number of triangles
* @param[in] tri Coordinates of the points
*/
*/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshname,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] );
void writeTriMesh( DBfile *fid, const std::string &meshname, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] );
/*!
@ -201,9 +198,9 @@ void writeTriMesh( DBfile* fid, const std::string& meshname,
* @param[in] meshname Mesh name
* @param[in] coords Coordinates of the points
* @param[in] tri Coordinates of the points
*/
*/
template<class TYPE>
void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri );
void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri );
/*!
@ -215,10 +212,10 @@ void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords,
* @param[in] varname Variable name
* @param[in] data Variable data
* @param[in] type Variable type
*/
*/
template<class TYPE>
void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data, VariableType type );
void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname,
const std::string &varname, const Array<TYPE> &data, VariableType type );
/*!
@ -227,9 +224,9 @@ void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname );
/*!
@ -239,10 +236,9 @@ Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] meshname Mesh name
* @param[in] subMeshNames Names of the sub meshes in the form "filename:meshname"
* @param[in] subMeshTypes Type of each submesh
*/
void writeMultiMesh( DBfile* fid, const std::string& meshname,
const std::vector<std::string>& subMeshNames,
const std::vector<int>& subMeshTypes );
*/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string> &subMeshNames, const std::vector<int> &subMeshTypes );
/*!
@ -255,14 +251,12 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
* @param[in] subVarTypes Type of each submesh
* @param[in] ndim Dimension of variable (used to determine suffix)
* @param[in] nvar Number of subvariables (used to determine suffix)
*/
void writeMultiVar( DBfile* fid, const std::string& varname,
const std::vector<std::string>& subVarNames,
const std::vector<int>& subVarTypes );
*/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
}; // silo namespace
}; // namespace IO::silo
#endif
#include "IO/silo.hpp"

349
IO/silo.hpp
View File

@ -2,8 +2,8 @@
#define SILO_INTERFACE_HPP
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -13,52 +13,77 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
* Helper functions *
****************************************************/
template<class TYPE> static constexpr int getType();
template<> constexpr int getType<double>() { return DB_DOUBLE; }
template<> constexpr int getType<float>() { return DB_FLOAT; }
template<> constexpr int getType<int>() { return DB_INT; }
* Helper functions *
****************************************************/
template<class TYPE>
inline void copyData( Array<TYPE>& data, int type, const void *src )
static constexpr int getType();
template<>
constexpr int getType<double>()
{
return DB_DOUBLE;
}
template<>
constexpr int getType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getType<int>()
{
return DB_INT;
}
template<class TYPE>
inline void copyData( Array<TYPE> &data, int type, const void *src )
{
if ( type == getType<TYPE>() )
memcpy( data.data(), src, data.length()*sizeof(TYPE) );
memcpy( data.data(), src, data.length() * sizeof( TYPE ) );
else if ( type == DB_DOUBLE )
data.copy( static_cast<const double*>(src) );
data.copy( static_cast<const double *>( src ) );
else if ( type == DB_FLOAT )
data.copy( static_cast<const float*>(src) );
data.copy( static_cast<const float *>( src ) );
else if ( type == DB_INT )
data.copy( static_cast<const int*>(src) );
data.copy( static_cast<const int *>( src ) );
else
ERROR("Unknown type");
ERROR( "Unknown type" );
}
/****************************************************
* Write/read an arbitrary vector *
****************************************************/
template<class TYPE> constexpr int getSiloType();
template<> constexpr int getSiloType<int>() { return DB_INT; }
template<> constexpr int getSiloType<float>() { return DB_FLOAT; }
template<> constexpr int getSiloType<double>() { return DB_DOUBLE; }
* Write/read an arbitrary vector *
****************************************************/
template<class TYPE>
void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data )
constexpr int getSiloType();
template<>
constexpr int getSiloType<int>()
{
return DB_INT;
}
template<>
constexpr int getSiloType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getSiloType<double>()
{
return DB_DOUBLE;
}
template<class TYPE>
void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data )
{
int dims = data.size();
int err = DBWrite( fid, varname.c_str(), (void*) data.data(), &dims, 1, getSiloType<TYPE>() );
int err = DBWrite( fid, varname.c_str(), (void *) data.data(), &dims, 1, getSiloType<TYPE>() );
ASSERT( err == 0 );
}
template<class TYPE>
std::vector<TYPE> read( DBfile* fid, const std::string& varname )
std::vector<TYPE> read( DBfile *fid, const std::string &varname )
{
int N = DBGetVarLength( fid, varname.c_str() );
std::vector<TYPE> data(N);
std::vector<TYPE> data( N );
int err = DBReadVar( fid, varname.c_str(), data.data() );
ASSERT( err == 0 );
return data;
@ -66,31 +91,31 @@ std::vector<TYPE> read( DBfile* fid, const std::string& varname )
/****************************************************
* Helper function to get variable suffixes *
****************************************************/
* Helper function to get variable suffixes *
****************************************************/
inline std::vector<std::string> getVarSuffix( int ndim, int nvars )
{
std::vector<std::string> suffix(nvars);
std::vector<std::string> suffix( nvars );
if ( nvars == 1 ) {
suffix[0] = "";
} else if ( nvars == ndim ) {
if ( ndim==2 ) {
if ( ndim == 2 ) {
suffix[0] = "_x";
suffix[1] = "_y";
} else if ( ndim==3 ) {
} else if ( ndim == 3 ) {
suffix[0] = "_x";
suffix[1] = "_y";
suffix[2] = "_z";
} else {
ERROR("Not finished");
ERROR( "Not finished" );
}
} else if ( nvars == ndim*ndim ) {
if ( ndim==2 ) {
} else if ( nvars == ndim * ndim ) {
if ( ndim == 2 ) {
suffix[0] = "_xx";
suffix[1] = "_xy";
suffix[2] = "_yx";
suffix[3] = "_yy";
} else if ( ndim==3 ) {
} else if ( ndim == 3 ) {
suffix[0] = "_xx";
suffix[1] = "_xy";
suffix[2] = "_xz";
@ -101,122 +126,127 @@ inline std::vector<std::string> getVarSuffix( int ndim, int nvars )
suffix[7] = "_zy";
suffix[8] = "_zz";
} else {
ERROR("Not finished");
ERROR( "Not finished" );
}
} else {
for (int i=0; i<nvars; i++)
suffix[i] = "_" + std::to_string(i+1);
for ( int i = 0; i < nvars; i++ )
suffix[i] = "_" + std::to_string( i + 1 );
}
return suffix;
}
/****************************************************
* Write/read a uniform mesh to silo *
****************************************************/
* Write/read a uniform mesh to silo *
****************************************************/
template<int NDIM>
void writeUniformMesh( DBfile* fid, const std::string& meshname,
const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N )
void writeUniformMesh( DBfile *fid, const std::string &meshname,
const std::array<double, 2 * NDIM> &range, const std::array<int, NDIM> &N )
{
PROFILE_START("writeUniformMesh",2);
PROFILE_START( "writeUniformMesh", 2 );
int dims[NDIM];
for (size_t d=0; d<N.size(); d++)
dims[d] = N[d]+1;
for ( size_t d = 0; d < N.size(); d++ )
dims[d] = N[d] + 1;
float *x = nullptr;
if ( NDIM >= 1 ) {
x = new float[dims[0]];
for (int i=0; i<N[0]; i++)
x[i] = range[0] + i*(range[1]-range[0])/N[0];
for ( int i = 0; i < N[0]; i++ )
x[i] = range[0] + i * ( range[1] - range[0] ) / N[0];
x[N[0]] = range[1];
}
float *y = nullptr;
if ( NDIM >= 2 ) {
y = new float[dims[1]];
for (int i=0; i<N[1]; i++)
y[i] = range[2] + i*(range[3]-range[2])/N[1];
for ( int i = 0; i < N[1]; i++ )
y[i] = range[2] + i * ( range[3] - range[2] ) / N[1];
y[N[1]] = range[3];
}
float *z = nullptr;
if ( NDIM >= 3 ) {
z = new float[dims[2]];
for (int i=0; i<N[2]; i++)
z[i] = range[4] + i*(range[5]-range[4])/N[2];
for ( int i = 0; i < N[2]; i++ )
z[i] = range[4] + i * ( range[5] - range[4] ) / N[2];
z[N[2]] = range[5];
}
float *coords[] = { x, y, z };
int err = DBPutQuadmesh( fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
int err = DBPutQuadmesh(
fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
delete[] x;
delete[] y;
delete[] z;
ASSERT( err == 0 );
PROFILE_STOP("writeUniformMesh",2);
PROFILE_STOP( "writeUniformMesh", 2 );
}
/****************************************************
* Write a vector/tensor quad variable *
****************************************************/
template<int NDIM,class TYPE>
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type )
* Write a vector/tensor quad variable *
****************************************************/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type )
{
PROFILE_START("writeUniformMeshVariable",2);
int nvars=1, dims[NDIM]={1};
PROFILE_START( "writeUniformMeshVariable", 2 );
int nvars = 1, dims[NDIM] = { 1 };
const TYPE *vars[NDIM] = { nullptr };
int vartype = 0;
if ( type == VariableType::NodeVariable ) {
ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
for (int d=0; d<NDIM; d++)
ASSERT(N[d]+1==(int)data.size(d));
ASSERT( data.ndim() == NDIM || data.ndim() == NDIM + 1 );
for ( int d = 0; d < NDIM; d++ )
ASSERT( N[d] + 1 == (int) data.size( d ) );
vartype = DB_NODECENT;
nvars = data.size(NDIM);
size_t N = data.length()/nvars;
for (int d=0; d<NDIM; d++)
dims[d] = data.size(d);
for (int i=0; i<nvars; i++)
vars[i] = &data(i*N);
nvars = data.size( NDIM );
size_t N = data.length() / nvars;
for ( int d = 0; d < NDIM; d++ )
dims[d] = data.size( d );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( i * N );
} else if ( type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::VolumeVariable ) {
ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
for (int d=0; d<NDIM; d++)
ASSERT(N[d]==(int)data.size(d));
ASSERT( data.ndim() == NDIM || data.ndim() == NDIM + 1 );
for ( int d = 0; d < NDIM; d++ )
ASSERT( N[d] == (int) data.size( d ) );
vartype = DB_ZONECENT;
nvars = data.size(NDIM);
size_t N = data.length()/nvars;
for (int d=0; d<NDIM; d++)
dims[d] = data.size(d);
for (int i=0; i<nvars; i++)
vars[i] = &data(i*N);
nvars = data.size( NDIM );
size_t N = data.length() / nvars;
for ( int d = 0; d < NDIM; d++ )
dims[d] = data.size( d );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( i * N );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
auto suffix = getVarSuffix( NDIM, nvars );
std::vector<std::string> var_names(nvars);
for (int i=0; i<nvars; i++)
std::vector<std::string> var_names( nvars );
for ( int i = 0; i < nvars; i++ )
var_names[i] = varname + suffix[i];
std::vector<char*> varnames(nvars,nullptr);
for (int i=0; i<nvars; i++)
varnames[i] = const_cast<char*>(var_names[i].c_str());
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
ASSERT( err == 0 );
PROFILE_STOP("writeUniformMeshVariable",2);
PROFILE_STOP( "writeUniformMeshVariable", 2 );
}
template<class TYPE>
Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetQuadvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeQuadvar( var );
std::vector<size_t> dims( var->ndims+1, var->nvals );
for (int d=0; d<var->ndims; d++)
std::vector<size_t> dims( var->ndims + 1, var->nvals );
for ( int d = 0; d < var->ndims; d++ )
dims[d] = var->dims[d];
data.reshape( dims );
return data;
@ -224,54 +254,55 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
/****************************************************
* Read/write a point mesh/variable to silo *
****************************************************/
* Read/write a point mesh/variable to silo *
****************************************************/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] )
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] )
{
int err = DBPutPointmesh( fid, meshname.c_str(), ndim, coords, N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
}
template<class TYPE>
Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname )
Array<TYPE> readPointMesh( DBfile *fid, const std::string &meshname )
{
auto mesh = DBGetPointmesh( fid, meshname.c_str() );
int N = mesh->nels;
int ndim = mesh->ndims;
Array<TYPE> coords(N,ndim);
Array<TYPE> coords( N, ndim );
int type = mesh->datatype;
for (int d=0; d<ndim; d++) {
for ( int d = 0; d < ndim; d++ ) {
Array<TYPE> data2( N );
copyData<TYPE>( data2, type, mesh->coords[d] );
memcpy( &coords(0,d), data2.data(), N*sizeof(TYPE) );
memcpy( &coords( 0, d ), data2.data(), N * sizeof( TYPE ) );
}
DBFreePointmesh( mesh );
return coords;
}
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data )
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data )
{
int N = data.size(0);
int nvars = data.size(1);
std::vector<const TYPE*> vars(nvars);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
int err = DBPutPointvar( fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
int N = data.size( 0 );
int nvars = data.size( 1 );
std::vector<const TYPE *> vars( nvars );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
int err = DBPutPointvar(
fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
}
template<class TYPE>
Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
Array<TYPE> readPointMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetPointvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeMeshvar( var );
return data;
@ -279,110 +310,110 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
/****************************************************
* Read/write a triangle mesh *
****************************************************/
* Read/write a triangle mesh *
****************************************************/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshName,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] )
void writeTriMesh( DBfile *fid, const std::string &meshName, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] )
{
auto zoneName = meshName + "_zones";
std::vector<int> nodelist( (ndim_tri+1)*N_tri );
for (int i=0, j=0; i<N_tri; i++) {
for (int d=0; d<ndim_tri+1; d++, j++)
std::vector<int> nodelist( ( ndim_tri + 1 ) * N_tri );
for ( int i = 0, j = 0; i < N_tri; i++ ) {
for ( int d = 0; d < ndim_tri + 1; d++, j++ )
nodelist[j] = tri[d][i];
}
int shapetype = 0;
if ( ndim_tri==1 )
if ( ndim_tri == 1 )
shapetype = DB_ZONETYPE_BEAM;
else if ( ndim_tri==2 )
else if ( ndim_tri == 2 )
shapetype = DB_ZONETYPE_TRIANGLE;
else if ( ndim_tri==3 )
else if ( ndim_tri == 3 )
shapetype = DB_ZONETYPE_PYRAMID;
else
ERROR("Unknown shapetype");
int shapesize = ndim_tri+1;
ERROR( "Unknown shapetype" );
int shapesize = ndim_tri + 1;
int shapecnt = N_tri;
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(),
nodelist.size(), 0, 0, 0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N,
nodelist.size(), zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(), nodelist.size(), 0, 0,
0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N, nodelist.size(),
zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
}
template<class TYPE>
void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri )
void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri )
{
auto mesh = DBGetUcdmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
int N_nodes = mesh->nnodes;
coords.resize(N_nodes,ndim);
coords.resize( N_nodes, ndim );
int mesh_type = mesh->datatype;
for (int d=0; d<ndim; d++) {
for ( int d = 0; d < ndim; d++ ) {
Array<TYPE> data2( N_nodes );
copyData<TYPE>( data2, mesh_type, mesh->coords[d] );
memcpy( &coords(0,d), data2.data(), N_nodes*sizeof(TYPE) );
memcpy( &coords( 0, d ), data2.data(), N_nodes * sizeof( TYPE ) );
}
auto zones = mesh->zones;
int N_zones = zones->nzones;
ASSERT( zones->nshapes==1 );
ASSERT( zones->nshapes == 1 );
int shapesize = zones->shapesize[0];
tri.resize(N_zones,shapesize);
for (int i=0; i<N_zones; i++) {
for (int j=0; j<shapesize; j++)
tri(i,j) = zones->nodelist[i*shapesize+j];
tri.resize( N_zones, shapesize );
for ( int i = 0; i < N_zones; i++ ) {
for ( int j = 0; j < shapesize; j++ )
tri( i, j ) = zones->nodelist[i * shapesize + j];
}
DBFreeUcdmesh( mesh );
}
template<class TYPE>
void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data, VariableType type )
void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname,
const std::string &varname, const Array<TYPE> &data, VariableType type )
{
int nvars = 0;
int vartype = 0;
const TYPE *vars[10] = { nullptr };
if ( type == VariableType::NodeVariable ) {
vartype = DB_NODECENT;
nvars = data.size(1);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
nvars = data.size( 1 );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
} else if ( type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::VolumeVariable ) {
vartype = DB_ZONECENT;
nvars = data.size(1);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
nvars = data.size( 1 );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
auto suffix = getVarSuffix( ndim, nvars );
std::vector<std::string> var_names(nvars);
for (int i=0; i<nvars; i++)
std::vector<std::string> var_names( nvars );
for ( int i = 0; i < nvars; i++ )
var_names[i] = varname + suffix[i];
std::vector<char*> varnames(nvars,nullptr);
for (int i=0; i<nvars; i++)
varnames[i] = const_cast<char*>(var_names[i].c_str());
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, data.size(0), nullptr, 0, getType<TYPE>(), vartype, nullptr );
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
data.size( 0 ), nullptr, 0, getType<TYPE>(), vartype, nullptr );
}
template<class TYPE>
Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname )
Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetUcdvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeUcdvar( var );
return data;
}
}; // silo namespace
}; // namespace IO::silo
#endif

1
README.titan
View File

@ -26,7 +26,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CFLAGS="-DCBUB" \
-D CXXFLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

3
StackTrace/ErrorHandlers.h
View File

@ -3,11 +3,10 @@
#include "StackTrace/StackTrace.h"
#include "common/MPI.h"
#include <functional>
#include "mpi.h"
namespace StackTrace
{

181
analysis/FreeEnergy.cpp Normal file
View File

@ -0,0 +1,181 @@
#include "analysis/FreeEnergy.h"
FreeEnergyAnalyzer::FreeEnergyAnalyzer(std::shared_ptr <Domain> dm):
Dm(dm)
{
Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0);
Phi.resize(Nx,Ny,Nz); Phi.fill(0);
Pressure.resize(Nx,Ny,Nz); Pressure.fill(0);
Rho.resize(Nx,Ny,Nz); Rho.fill(0);
Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field
Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0);
Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0);
SDs.resize(Nx,Ny,Nz); SDs.fill(0);
if (Dm->rank()==0){
bool WriteHeader=false;
TIMELOG = fopen("free.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("free.csv","a+");
if (WriteHeader)
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"timestep\n");
}
}
}
FreeEnergyAnalyzer::~FreeEnergyAnalyzer(){
if (Dm->rank()==0){
fclose(TIMELOG);
}
}
void FreeEnergyAnalyzer::SetParams(){
}
void FreeEnergyAnalyzer::Basic(ScaLBL_FreeLeeModel &LeeModel, int timestep){
int i,j,k;
if (Dm->rank()==0){
fprintf(TIMELOG,"%i ",timestep);
/*for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
}
*/
fprintf(TIMELOG,"\n");
fflush(TIMELOG);
}
/* else{
fprintf(TIMELOG,"%i ",timestep);
for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
}
fflush(TIMELOG);
} */
}
void FreeEnergyAnalyzer::WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep){
auto vis_db = input_db->getDatabase( "Visualization" );
char VisName[40];
std::vector<IO::MeshDataStruct> visData;
fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
IO::initialize("","silo","false");
// Create the MeshDataStruct
visData.resize(1);
visData[0].meshName = "domain";
visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
auto VisPhase = std::make_shared<IO::Variable>();
auto VisPressure = std::make_shared<IO::Variable>();
auto VisChemicalPotential = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
auto VyVar = std::make_shared<IO::Variable>();
auto VzVar = std::make_shared<IO::Variable>();
if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
VisPhase->name = "Phase";
VisPhase->type = IO::VariableType::VolumeVariable;
VisPhase->dim = 1;
VisPhase->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPhase);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
VisPressure->name = "Pressure";
VisPressure->type = IO::VariableType::VolumeVariable;
VisPressure->dim = 1;
VisPressure->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPressure);
VisChemicalPotential->name = "ChemicalPotential";
VisChemicalPotential->type = IO::VariableType::VolumeVariable;
VisChemicalPotential->dim = 1;
VisChemicalPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisChemicalPotential);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VxVar);
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VyVar);
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VzVar);
}
if (vis_db->getWithDefault<bool>( "save_phase", true )){
ASSERT(visData[0].vars[0]->name=="Phase");
LeeModel.getPhase(Phi);
Array<double>& PhaseData = visData[0].vars[0]->data;
fillData.copy(Phi,PhaseData);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
ASSERT(visData[0].vars[1]->name=="Pressure");
LeeModel.getPotential(Pressure, ChemicalPotential);
Array<double>& PressureData = visData[0].vars[1]->data;
fillData.copy(Pressure,PressureData);
ASSERT(visData[0].vars[2]->name=="ChemicalPotential");
Array<double>& ChemicalPotentialData = visData[0].vars[2]->data;
fillData.copy(ChemicalPotential,ChemicalPotentialData);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
ASSERT(visData[0].vars[3]->name=="Velocity_x");
ASSERT(visData[0].vars[4]->name=="Velocity_y");
ASSERT(visData[0].vars[5]->name=="Velocity_z");
LeeModel.getVelocity(Vel_x,Vel_y,Vel_z);
Array<double>& VelxData = visData[0].vars[3]->data;
Array<double>& VelyData = visData[0].vars[4]->data;
Array<double>& VelzData = visData[0].vars[5]->data;
fillData.copy(Vel_x,VelxData);
fillData.copy(Vel_y,VelyData);
fillData.copy(Vel_z,VelzData);
}
if (vis_db->getWithDefault<bool>( "write_silo", true ))
IO::writeData( timestep, visData, Dm->Comm );
/* if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
char CurrentIDFilename[40];
sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
Averages.AggregateLabels(CurrentIDFilename);
}
*/
}

54
analysis/FreeEnergy.h Normal file
View File

@ -0,0 +1,54 @@
/*
* averaging tools for electrochemistry
*/
#ifndef FreeEnergyAnalyzer_INC
#define FreeEnergyAnalyzer_INC
#include <vector>
#include "common/Domain.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "analysis/SubPhase.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
#include "models/FreeLeeModel.h"
class FreeEnergyAnalyzer{
public:
std::shared_ptr <Domain> Dm;
double Volume;
// input variables
double rho_n, rho_w;
double nu_n, nu_w;
double gamma_wn, beta;
double Fx, Fy, Fz;
//...........................................................................
int Nx,Ny,Nz;
DoubleArray Rho;
DoubleArray Phi;
DoubleArray ChemicalPotential;
DoubleArray Pressure;
DoubleArray Vel_x;
DoubleArray Vel_y;
DoubleArray Vel_z;
DoubleArray SDs;
FreeEnergyAnalyzer(std::shared_ptr <Domain> Dm);
~FreeEnergyAnalyzer();
void SetParams();
void Basic( ScaLBL_FreeLeeModel &LeeModel, int timestep);
void WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep);
private:
FILE *TIMELOG;
};
#endif

59
analysis/SubPhase.cpp
View File

@ -40,7 +40,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -50,7 +50,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
fprintf(SUBPHASE,"Vwc Awc Hwc Xwc "); // wc region
fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd "); // wd region
fprintf(SUBPHASE,"Vnc Anc Hnc Xnc "); // nc region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd regionin
fprintf(SUBPHASE,"Vi Ai Hi Xi "); // interface region
fprintf(SUBPHASE,"Vic Aic Hic Xic Nic\n"); // interface region
@ -65,7 +65,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
sprintf(LocalRankFilename,"%s%s","subphase.csv.",LocalRankString);
SUBPHASE = fopen(LocalRankFilename,"a+");
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -93,7 +93,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn wet\n");
}
}
}
@ -109,7 +109,7 @@ SubPhase::~SubPhase()
void SubPhase::Write(int timestep)
{
if (Dm->rank()==0){
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction_global);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
@ -125,7 +125,7 @@ void SubPhase::Write(int timestep)
fflush(SUBPHASE);
}
else{
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
@ -172,6 +172,21 @@ void SubPhase::Basic(){
double count_w = 0.0;
double count_n = 0.0;
/* compute the laplacian */
Dm->CommunicateMeshHalo(Phi);
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
// Compute all of the derivatives using finite differences
double fx = 0.5*(Phi(i+1,j,k) - Phi(i-1,j,k));
double fy = 0.5*(Phi(i,j+1,k) - Phi(i,j-1,k));
double fz = 0.5*(Phi(i,j,k+1) - Phi(i,j,k-1));
DelPhi(i,j,k) = sqrt(fx*fx+fy*fy+fz*fz);
}
}
}
Dm->CommunicateMeshHalo(DelPhi);
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
@ -184,6 +199,11 @@ void SubPhase::Basic(){
double phi = (nA-nB)/(nA+nB);
Phi(n) = phi;
}
if (Phi(n) != Phi(n)){
// check for NaN
Phi(n) = 0.0;
//printf("Nan at %i %i %i \n",i,j,k);
}
}
}
}
@ -229,6 +249,31 @@ void SubPhase::Basic(){
}
}
}
total_wetting_interaction = count_wetting_interaction = 0.0;
total_wetting_interaction_global = count_wetting_interaction_global=0.0;
for (k=kmin; k<kmax; k++){
for (j=jmin; j<Ny-1; j++){
for (i=imin; i<Nx-1; i++){
n = k*Nx*Ny + j*Nx + i;
// compute contribution of wetting terms (within two voxels of solid)
if ( Dm->id[n] > 0 && SDs(i,j,k) < 2.0 ){
count_wetting_interaction += 1.0;
total_wetting_interaction += DelPhi(i,j,k);
}
}
}
}
//printf("wetting interaction = %f, count = %f\n",total_wetting_interaction,count_wetting_interaction);
total_wetting_interaction_global=Dm->Comm.sumReduce( total_wetting_interaction);
count_wetting_interaction_global=Dm->Comm.sumReduce( count_wetting_interaction);
/* normalize wetting interactions <-- Don't do this if normalizing laplacian (use solid surface area)
if (count_wetting_interaction > 0.0)
total_wetting_interaction /= count_wetting_interaction;
if (count_wetting_interaction_global > 0.0)
total_wetting_interaction_global /= count_wetting_interaction_global;
*/
gwb.V=Dm->Comm.sumReduce( wb.V);
gnb.V=Dm->Comm.sumReduce( nb.V);
gwb.M=Dm->Comm.sumReduce( wb.M);
@ -303,7 +348,7 @@ void SubPhase::Basic(){
double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
double krw = h*h*nu_w*water_flow_rate / force_mag;
//printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p);
fprintf(TIMELOG,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p, total_wetting_interaction_global);
fflush(TIMELOG);
}
if (err==true){

8
analysis/SubPhase.h
View File

@ -68,12 +68,16 @@ public:
* b - bulk (total)
*/
// local entities
phase wc,wd,wb,nc,nd,nb;
phase wc,wd,wb,nc,nd,nb,solid;
interface iwn,iwnc;
// global entities
phase gwc,gwd,gwb,gnc,gnd,gnb;
phase gwc,gwd,gwb,gnc,gnd,gnb,gsolid;
interface giwn,giwnc;
/* fluid-solid wetting interaction */
double total_wetting_interaction, count_wetting_interaction;
double total_wetting_interaction_global, count_wetting_interaction_global;
//...........................................................................
int Nx,Ny,Nz;
IntArray PhaseID; // Phase ID array (solid=0, non-wetting=1, wetting=2)

3
analysis/morphology.cpp
View File

@ -542,7 +542,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
// Extract only the connected part of NWP
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
Dm->Comm.barrier();
@ -703,10 +702,12 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
if (rank == 0) printf(" delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT);
// Now adjust morph_delta
if (fabs(GrowthEstimate - GrowthPrevious) > 0.0) {
double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
GrowthPrevious = GrowthEstimate;
morph_delta_previous = morph_delta;
morph_delta += step_size;
}
if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0;
//MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25);

1117
analysis/runAnalysis.cpp
View File

File diff suppressed because it is too large Load Diff

64
analysis/runAnalysis.h
View File

@ -1,41 +1,51 @@
#ifndef RunAnalysis_H_INC
#define RunAnalysis_H_INC
#include "analysis/analysis.h"
#include "analysis/TwoPhase.h"
#include "analysis/SubPhase.h"
#include "analysis/TwoPhase.h"
#include "analysis/analysis.h"
#include "common/Communication.h"
#include "common/ScaLBL.h"
#include "threadpool/thread_pool.h"
#include "models/ColorModel.h"
#include <limits.h>
typedef std::shared_ptr<std::pair<int,IntArray>> BlobIDstruct;
typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
// Types of analysis
enum class AnalysisType : uint64_t { AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02,
CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20, ComputeSubphase=0x40 };
enum class AnalysisType : uint64_t {
AnalyzeNone = 0,
IdentifyBlobs = 0x01,
CopyPhaseIndicator = 0x02,
CopySimState = 0x04,
ComputeAverages = 0x08,
CreateRestart = 0x10,
WriteVis = 0x20,
ComputeSubphase = 0x40
};
//! Class to run the analysis in multiple threads
class runAnalysis
{
public:
//! Constructor
runAnalysis(std::shared_ptr<Database> db, const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool Regular, IntArray Map );
runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct &rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> dm, int Np,
bool Regular, IntArray Map );
runAnalysis( ScaLBL_ColorModel &ColorModel);
//! Destructor
~runAnalysis();
//! Run the next analysis
void run(int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
void run( int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData(int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den);
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData( int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages,
const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
//! Finish all active analysis
void finish();
@ -44,7 +54,8 @@ public:
* \brief Set the affinities
* \details This function will create the analysis threads and set the affinity
* of this thread and all analysis threads. If MPI_THREAD_MULTIPLE is not
* enabled, the analysis threads will be disabled and the analysis will run in the current thread.
* enabled, the analysis threads will be disabled and the analysis will run in the current
* thread.
* @param[in] method Method used to control the affinities:
* none - Don't use threads (runs all analysis in the current thread)
* default - Create the specified number of threads, but don't load balance
@ -53,39 +64,36 @@ public:
* that all threads run on independent cores
* @param[in] N_threads Number of threads, only used by some of the methods
*/
void createThreads( const std::string& method = "default", int N_threads = 4 );
void createThreads( const std::string &method = "default", int N_threads = 4 );
private:
runAnalysis();
// Determine the analysis to perform
AnalysisType computeAnalysisType( int timestep );
public:
class commWrapper
{
public:
Utilities::MPI comm;
int tag;
runAnalysis *analysis;
commWrapper( int tag, const Utilities::MPI& comm, runAnalysis *analysis );
commWrapper( ) = delete;
commWrapper( int tag, const Utilities::MPI &comm, runAnalysis *analysis );
commWrapper() = delete;
commWrapper( const commWrapper &rhs ) = delete;
commWrapper& operator=( const commWrapper &rhs ) = delete;
commWrapper &operator=( const commWrapper &rhs ) = delete;
commWrapper( commWrapper &&rhs );
~commWrapper();
};
// Get a comm (not thread safe)
commWrapper getComm( );
commWrapper getComm();
private:
std::array<int,3> d_n; // Number of local cells
std::array<int,3> d_N; // NNumber of local cells with ghosts
std::array<int, 3> d_n; // Number of local cells
std::array<int, 3> d_N; // Number of local cells with ghosts
int d_Np;
int d_rank;
int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
@ -95,9 +103,9 @@ private:
ThreadPool d_tpool;
RankInfoStruct d_rank_info;
IntArray d_Map;
BlobIDstruct d_last_ids;
BlobIDstruct d_last_index;
BlobIDList d_last_id_map;
std::shared_ptr<std::pair<int, IntArray>> d_last_ids;
std::shared_ptr<std::pair<int, IntArray>> d_last_index;
std::shared_ptr<std::vector<BlobIDType>> d_last_id_map;
std::vector<IO::MeshDataStruct> d_meshData;
std::string d_restartFile;
Utilities::MPI d_comm;
@ -114,8 +122,6 @@ private:
// Friends
friend commWrapper::~commWrapper();
};
#endif

362
cmake/FindMPI.cmake
View File

@ -1,362 +0,0 @@
# - Message Passing Interface (MPI) module.
#
# The Message Passing Interface (MPI) is a library used to write
# high-performance parallel applications that use message passing, and
# is typically deployed on a cluster. MPI is a standard interface
# (defined by the MPI forum) for which many implementations are
# available. All of these implementations have somewhat different
# compilation approaches (different include paths, libraries to link
# against, etc.), and this module tries to smooth out those differences.
#
# This module will set the following variables:
# MPI_FOUND TRUE if we have found MPI
# MPI_COMPILE_FLAGS Compilation flags for MPI programs
# MPI_INCLUDE_PATH Include path(s) for MPI header
# MPI_LINK_FLAGS Linking flags for MPI programs
# MPI_LIBRARY First MPI library to link against (cached)
# MPI_EXTRA_LIBRARY Extra MPI libraries to link against (cached)
# MPI_LIBRARIES All libraries to link MPI programs against
# MPIEXEC Executable for running MPI programs
# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving it the
# number of processors to run on
# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly before the
# executable to run.
# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after all other flags.
#
# This module will attempt to auto-detect these settings, first by
# looking for a MPI compiler, which many MPI implementations provide
# as a pass-through to the native compiler to simplify the compilation
# of MPI programs. The MPI compiler is stored in the cache variable
# MPI_COMPILER, and will attempt to look for commonly-named drivers
# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
# recognized, it will be used to set all of the module variables. To
# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
# the CMake cache.
#
# If no compiler driver is found or the compiler driver is not
# recognized, this module will then search for common include paths
# and library names to try to detect MPI.
#
# If CMake initially finds a different MPI than was intended, and you
# want to use the MPI compiler auto-detection for a different MPI
# implementation, set MPI_COMPILER to the MPI compiler driver you want
# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
# will run again with the newly-specified MPI_COMPILER.
#
# When using MPIEXEC to execute MPI applications, you should typically
# use all of the MPIEXEC flags as follows:
# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
# ${MPIEXEC_POSTFLAGS} ARGS
# where PROCS is the number of processors on which to execute the program,
# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
# MPI program.
#=============================================================================
# Copyright 2001-2009 Kitware, Inc.
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
# This module is maintained by David Partyka <dave.partyka@kitware.com>.
# A set of directories to search through in addition to the standard system paths
# that find_program will search through.
# Microsoft HPC SDK is automatically added to the system path
# Argonne National Labs MPICH2 sets a registry key that we can use.
set(_MPI_PACKAGE_DIR
mpi
mpich
openmpi
lib/mpi
lib/mpich
lib/openmpi
"MPICH/SDK"
"Microsoft Compute Cluster Pack"
"Microsoft HPC Pack 2008 R2"
)
set(_MPI_PREFIX_PATH)
if(WIN32)
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
endif()
foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
endif()
endforeach(MpiPackageDir)
endforeach(SystemPrefixDir)
# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
find_program(MPIEXEC
NAMES mpiexec mpirun lamexec
PATHS ${_MPI_PREFIX_PATH}
PATH_SUFFIXES bin
DOC "Executable for running MPI programs."
)
# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
# If there is an mpi compiler find it and interogate (farther below) it for the include
# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
find_program(MPI_COMPILER
NAMES mpic++ mpicxx mpiCC mpicc
HINTS "${_MPI_BASE_DIR}"
PATH_SUFFIXES bin
DOC "MPI compiler. Used only to detect MPI compilation flags.")
mark_as_advanced(MPI_COMPILER)
set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILER)
# Check whether the -showme:compile option works. This indicates
# that we have either Open MPI or a newer version of LAM-MPI, and
# implies that -showme:link will also work.
# Note that Windows distros do not have an mpi compiler to interogate.
exec_program(${MPI_COMPILER}
ARGS -showme:compile
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
if (MPI_COMPILER_RETURN EQUAL 0)
# If we appear to have -showme:compile, then we should also have
# -showme:link. Try it.
exec_program(${MPI_COMPILER}
ARGS -showme:link
OUTPUT_VARIABLE MPI_LINK_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
# Note that we probably have -showme:incdirs and -showme:libdirs
# as well.
set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# Older versions of LAM-MPI have "-showme". Try it.
exec_program(${MPI_COMPILER}
ARGS -showme
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# MPICH uses "-show". Try it.
exec_program(${MPI_COMPILER}
ARGS -show
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# We have our command lines, but we might need to copy
# MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
if (NOT MPI_LINK_CMDLINE)
SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
endif (NOT MPI_LINK_CMDLINE)
else (MPI_COMPILER_RETURN EQUAL 0)
message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
endif (MPI_COMPILER_RETURN EQUAL 0)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILE_CMDLINE)
# Extract compile flags from the compile command line.
string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
set(MPI_COMPILE_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
if (MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
else(MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK ${FLAG})
endif(MPI_COMPILE_FLAGS_WORK)
endforeach(FLAG)
# Extract include paths from compile command line
string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
set(MPI_INCLUDE_PATH_WORK)
foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
string(REGEX REPLACE "//" "/" IPATH ${IPATH})
list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
endforeach(IPATH)
if (NOT MPI_INCLUDE_PATH_WORK)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any include paths on it,
# but we may have -showme:incdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:incdirs
OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_INCLUDE_PATH_WORK)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_INCLUDE_PATH_WORK)
if (NOT MPI_INCLUDE_PATH_WORK)
# If all else fails, just search for mpi.h in the normal include
# paths.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include
)
set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
endif (NOT MPI_INCLUDE_PATH_WORK)
# Extract linker paths from the link command line
string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_PATH)
foreach(LPATH ${MPI_ALL_LINK_PATHS})
string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
string(REGEX REPLACE "//" "/" LPATH ${LPATH})
list(APPEND MPI_LINK_PATH ${LPATH})
endforeach(LPATH)
if (NOT MPI_LINK_PATH)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any linking paths on it,
# but we may have -showme:libdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:libdirs
OUTPUT_VARIABLE MPI_LINK_PATH
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_LINK_PATH)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_LINK_PATH)
# Extract linker flags from the link command line
string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_LINK_FLAGS})
if (MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
else(MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK ${FLAG})
endif(MPI_LINK_FLAGS_WORK)
endforeach(FLAG)
if ( MPI_LINK_FLAGS_WORK )
string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
endif ()
# Extract the set of libraries to link against from the link command
# line
string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
# Determine full path names for all of the libraries that one needs
# to link against in an MPI program
set(MPI_LIBRARIES)
foreach(LIB ${MPI_LIBNAMES})
string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
if (MPI_LIB)
list(APPEND MPI_LIBRARIES ${MPI_LIB})
elseif (NOT MPI_FIND_QUIETLY)
message(WARNING "Unable to find MPI library ${LIB}")
endif ()
endforeach(LIB)
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
# Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
# MPI_EXTRA_LIBRARY.
list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
else (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
endif (MPI_NUMLIBS GREATER 1)
# Set up all of the appropriate cache entries
set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
else (MPI_COMPILE_CMDLINE)
# No MPI compiler to interogate so attempt to find everything with find functions.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include Inc
)
# Decide between 32-bit and 64-bit libraries for Microsoft's MPI
if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
set(MS_MPI_ARCH_DIR amd64)
else()
set(MS_MPI_ARCH_DIR i386)
endif()
find_library(MPI_LIBRARY
NAMES mpi mpich msmpi
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
)
find_library(MPI_EXTRA_LIBRARY
NAMES mpi++
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib
DOC "Extra MPI libraries to link against.")
set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Set up extra variables to conform to
if (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
else (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY})
endif (MPI_EXTRA_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND TRUE)
else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND FALSE)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
# handle the QUIETLY and REQUIRED arguments
#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
MPI_EXTRA_LIBRARY)
# unset to cleanup namespace
unset(_MPI_PACKAGE_DIR)
unset(_MPI_PREFIX_PATH)
unset(_MPI_BASE_DIR)

41
cmake/Find_TIMER.cmake
View File

@ -4,7 +4,7 @@
# CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR )
# This function assumes that USE_TIMER is set to indicate if the timer should be used
# If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer
# If USE_TIMER is not set we will create a summy timer that does nothing.
# If USE_TIMER is not set we will create a dummy timer that does nothing.
# The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default.
# The input argument NULL_TIMER_DIR specifies the location to install the dummy timer.
# If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used.
@ -13,7 +13,7 @@
# TIMER_CXXFLAGS - C++ flags for the timer library
# TIMER_LDFLAGS - Linker flags to link the timer library
# TIMER_LDLIBS - Linker libraries to link the timer library
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR QUIET )
# Determine if we want to use the timer utility
CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} )
SET( TIMER_INCLUDE )
@ -33,20 +33,23 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FIND_LIBRARY( TIMER_LIBS NAMES timerutility PATHS ${TIMER_DIRECTORY}/lib NO_DEFAULT_PATH )
SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include )
SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" )
SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib )
SET( TIMER_LDLIBS -ltimerutility )
SET( TIMER_LDFLAGS )
SET( TIMER_LDLIBS "${TIMER_LIBS}" )
ELSE()
MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported. Use -D TIMER_DIRECTORY=" )
ENDIF()
SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
ADD_DEFINITIONS( -DUSE_TIMER )
MESSAGE( "Using timer utility" )
MESSAGE( " TIMER_LIBRARIES = ${TIMER_LIBS}" )
IF ( NOT QUIET )
MESSAGE( STATUS "Using timer utility" )
MESSAGE( STATUS " TIMER_LIBRARIES = ${TIMER_LIBS}" )
ENDIF()
ELSE()
IF ( "${NULL_TIMER_DIR}" STREQUAL "" )
SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" )
ENDIF()
# Write ProfilerApp.h
FILE(WRITE "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...) do {} while(0)\n" )
@ -61,9 +64,25 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" )
# Write MemoryApp.h
FILE(WRITE "${NULL_TIMER_DIR}/MemoryApp.h" "#include <cstring>\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "class MemoryApp final {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "public:\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " struct MemoryStats {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " size_t bytes_new, bytes_delete, N_new, N_delete, tot_bytes_used, system_memory, stack_used, stack_size;\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " MemoryStats() { memset(this,0,sizeof(MemoryStats)); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " };\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline void print( std::ostream& ) {}\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getTotalMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getSystemMemory() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline MemoryStats getMemoryStats() { return MemoryStats(); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "};\n" )
SET( TIMER_INCLUDE "${NULL_TIMER_DIR}" )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
MESSAGE( "Disabling timer utility" )
IF ( NOT QUIET )
MESSAGE( STATUS "Disabling timer utility" )
ENDIF()
ENDIF()
SET( TIMER_INCLUDE "${TIMER_INCLUDE}" PARENT_SCOPE )
SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE )
@ -88,12 +107,12 @@ MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ${FLAG} STREQUAL "" )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "FALSE" ) OR ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
SET( ${FLAG} 0 )
ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "TRUE" ) OR ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
SET( ${FLAG} 1 )
ELSE()
MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
MESSAGE( FATAL_ERROR "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
ENDIF ()
ENDMACRO()

3
cmake/ctest_script.cmake
View File

@ -32,7 +32,6 @@ SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} )
SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} )
SET( LDLIBS $ENV{LDLIBS} )
SET( LDFLAGS $ENV{LDFLAGS} )
SET( MPI_COMPILER $ENV{MPI_COMPILER} )
SET( MPI_DIRECTORY $ENV{MPI_DIRECTORY} )
SET( MPI_INCLUDE $ENV{MPI_INCLUDE} )
SET( MPI_LINK_FLAGS $ENV{MPI_LINK_FLAGS} )
@ -198,7 +197,7 @@ SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLA
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
IF ( USE_MPI )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}")
IF ( NOT USE_VALGRIND )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true")
ENDIF()

124
cmake/libraries.cmake
View File

@ -41,93 +41,61 @@ ENDMACRO()
# Macro to find and configure the MPI libraries
MACRO( CONFIGURE_MPI )
# Determine if we want to use MPI
CHECK_ENABLE_FLAG(USE_MPI 1 )
CHECK_ENABLE_FLAG( USE_MPI 1 )
IF ( USE_MPI )
# Check if we specified the MPI directory
IF ( MPI_DIRECTORY )
# Check the provided MPI directory for include files
VERIFY_PATH( "${MPI_DIRECTORY}" )
IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" )
ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" )
ELSE()
MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" )
ENDIF ()
INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
# Set MPI libraries
IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} )
ENDIF()
# Set the mpi executable
MESSAGE( "Configuring MPI" )
IF ( MPIEXEC )
# User specified the MPI command directly, use as is
ELSEIF ( MPIEXEC_CMD )
# User specified the name of the MPI executable
SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
IF ( NOT EXISTS ${MPIEXEC} )
MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
ENDIF ()
ELSE ()
# Search for the MPI executable in the current directory
FIND_PROGRAM( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH )
IF ( NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
ENDIF()
ENDIF ()
# Set MPI flags
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
ENDIF()
ELSEIF ( MPI_COMPILER )
# The mpi compiler should take care of everything
IF ( MPI_INCLUDE )
INCLUDE_DIRECTORIES( ${MPI_INCLUDE} )
SET( MPIEXEC_EXECUTABLE ${MPIEXEC} )
ENDIF()
IF ( NOT MPI_SKIP_SEARCH )
FIND_PACKAGE( MPI )
ELSE()
# Perform the default search for MPI
INCLUDE ( FindMPI )
IF ( NOT MPI_FOUND )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
MESSAGE( FATAL_ERROR "Did not find MPI" )
ENDIF ()
INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" )
SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" )
# Write mpi test
SET( MPI_TEST_SRC "${CMAKE_CURRENT_BINARY_DIR}/test_mpi.cpp" )
FILE(WRITE ${MPI_TEST_SRC} "#include <mpi.h>\n" )
FILE(APPEND ${MPI_TEST_SRC} "int main(int argc, char** argv) {\n" )
FILE(APPEND ${MPI_TEST_SRC} " MPI_Init(NULL, NULL);\n")
FILE(APPEND ${MPI_TEST_SRC} " MPI_Finalize();\n" )
FILE(APPEND ${MPI_TEST_SRC} "}\n" )
# Test the compile
IF ( CMAKE_CXX_COMPILER )
SET( TMP_FLAGS -DINCLUDE_DIRECTORIES=${MPI_CXX_INCLUDE_PATH} )
TRY_COMPILE( MPI_TEST_CXX ${CMAKE_CURRENT_BINARY_DIR} ${MPI_TEST_SRC}
CMAKE_FLAGS ${TMP_FLAGS}
LINK_OPTIONS ${MPI_CXX_LINK_FLAGS}
LINK_LIBRARIES ${MPI_CXX_LIBRARIES}
OUTPUT_VARIABLE OUT_TXT)
IF ( NOT ${MPI_TEST} )
MESSAGE( FATAL_ERROR "Skipping MPI search and default compile fails:\n${OUT_TXT}" )
ENDIF()
# Check if we need to use MPI for serial tests
CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
# Set defaults if they have not been set
IF ( NOT MPIEXEC )
SET( MPIEXEC mpirun )
SET( MPI_C_FOUND TRUE )
SET( MPI_CXX_FOUND TRUE )
SET( MPI_Fortran_FOUND TRUE )
ENDIF()
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
ENDIF()
# Set the definitions
ADD_DEFINITIONS( "-DUSE_MPI" )
MESSAGE( "Using MPI" )
STRING( STRIP "${MPI_CXX_COMPILE_FLAGS}" MPI_CXX_COMPILE_FLAGS )
STRING( STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS )
STRING( STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES )
MESSAGE( " MPI_CXX_FOUND = ${MPI_CXX_FOUND}" )
MESSAGE( " MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}" )
MESSAGE( " MPI_CXX_COMPILE_FLAGS = ${MPI_CXX_COMPILE_FLAGS}" )
MESSAGE( " MPI_CXX_INCLUDE_PATH = ${MPI_CXX_INCLUDE_PATH}" )
MESSAGE( " MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}" )
MESSAGE( " MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}" )
MESSAGE( " MPIEXEC = ${MPIEXEC}" )
MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
ELSE()
SET( USE_MPI_FOR_SERIAL_TESTS 0 )
SET( MPIEXEC "" )
SET( MPIEXEC_NUMPROC_FLAG "" )
SET( MPI_INCLUDE "" )
SET( MPI_LINK_FLAGS "" )
SET( MPI_LIBRARIES "" )
MESSAGE( "Not using MPI, all parallel tests will be disabled" )
MESSAGE( " MPIEXEC_PREFLAGS = ${MPIEXEC_PREFLAGS}" )
MESSAGE( " MPIEXEC_POSTFLAGS = ${MPIEXEC_POSTFLAGS}" )
ADD_DEFINITIONS( -DUSE_MPI )
INCLUDE_DIRECTORIES( ${MPI_CXX_INCLUDE_PATH} )
SET( MPI_LIBRARIES ${MPI_CXX_LIBRARIES} )
SET( MPI_LINK_FLAGS ${MPI_CXX_LINK_FLAGS} )
IF ( NOT MPI_CXX_FOUND )
MESSAGE( FATAL_ERROR "MPI not found" )
ENDIF()
IF ( USE_MPI AND NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Unable to find MPIEXEC, please set it before continuing" )
ENDIF()
ENDIF()
ENDMACRO()

4
cmake/macros.cmake
View File

@ -681,8 +681,8 @@ MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( MPI_LIBRARIES )
TARGET_LINK_LIBRARIES( ${EXE} ${ARGN} ${tmp} )
FOREACH ( tmp ${MPI_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )

7
common/Communication.h
View File

@ -67,6 +67,10 @@ public:
//! Destructor
~fillHalo( );
fillHalo() = delete;
fillHalo(const fillHalo&) = delete;
fillHalo& operator=(const fillHalo&) = delete;
/*!
* @brief Communicate the halos
* @param[in] array The array on which we fill the halos
@ -93,9 +97,6 @@ private:
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor
fillHalo& operator=(const fillHalo&); // Private assignment operator
void pack( const Array<TYPE>& array, int i, int j, int k, TYPE *buffer );
void unpack( Array<TYPE>& array, int i, int j, int k, const TYPE *buffer );
};

26
common/Domain.cpp
View File

@ -558,17 +558,13 @@ void Domain::Decomp( const std::string& Filename )
int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2;
if (z_transition_size < 0) z_transition_size=0;
char LocalRankFilename[40];
char *loc_id;
loc_id = new char [(nx+2)*(ny+2)*(nz+2)];
// Set up the sub-domains
if (RANK==0){
printf("Distributing subdomains across %i processors \n",nprocs);
printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz);
printf("Subdomain size: %i x %i x %i \n",nx,ny,nz);
printf("Size of transition region: %ld \n", z_transition_size);
auto loc_id = new char [(nx+2)*(ny+2)*(nz+2)];
for (int kp=0; kp<nprocz; kp++){
for (int jp=0; jp<nprocy; jp++){
for (int ip=0; ip<nprocx; ip++){
@ -609,6 +605,7 @@ void Domain::Decomp( const std::string& Filename )
Comm.send(loc_id,N,rnk,15);
}
// Write the data for this rank data
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rnk+rank_offset);
FILE *ID = fopen(LocalRankFilename,"wb");
fwrite(loc_id,1,(nx+2)*(ny+2)*(nz+2),ID);
@ -616,9 +613,8 @@ void Domain::Decomp( const std::string& Filename )
}
}
}
}
else{
delete [] loc_id;
} else {
// Recieve the subdomain from rank = 0
//printf("Ready to recieve data %i at process %i \n", N,rank);
Comm.recv(id.data(),N,0,15);
@ -645,6 +641,7 @@ void Domain::Decomp( const std::string& Filename )
porosity = sum*iVol_global;
if (rank()==0) printf("Media porosity = %f \n",porosity);
//.........................................................
delete [] SegData;
}
void Domain::AggregateLabels( const std::string& filename ){
@ -669,8 +666,7 @@ void Domain::AggregateLabels( const std::string& filename ){
int local_size = (nx-2)*(ny-2)*(nz-2);
long int full_size = long(full_nx)*long(full_ny)*long(full_nz);
signed char *LocalID;
LocalID = new signed char [local_size];
auto LocalID = new signed char [local_size];
//printf("aggregate labels: local size=%i, global size = %i",local_size, full_size);
// assign the ID for the local sub-region
@ -687,8 +683,7 @@ void Domain::AggregateLabels( const std::string& filename ){
// populate the FullID
if (rank() == 0){
signed char *FullID;
FullID = new signed char [full_size];
auto FullID = new signed char [full_size];
// first handle local ID for rank 0
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
@ -727,6 +722,7 @@ void Domain::AggregateLabels( const std::string& filename ){
FILE *OUTFILE = fopen(filename.c_str(),"wb");
fwrite(FullID,1,full_size,OUTFILE);
fclose(OUTFILE);
delete [] FullID;
}
else{
// send LocalID to rank=0
@ -734,8 +730,8 @@ void Domain::AggregateLabels( const std::string& filename ){
int dstrank = 0;
Comm.send(LocalID,local_size,dstrank,tag);
}
delete [] LocalID;
Comm.barrier();
}
/********************************************************
@ -1232,11 +1228,10 @@ void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatyp
// user needs to modify the input file accordingly before LBPM simulator read
// the input file.
//........................................................................................
int rank_offset = 0;
int RANK = rank();
int nprocs, nprocx, nprocy, nprocz, nx, ny, nz;
int64_t global_Nx,global_Ny,global_Nz;
int64_t i,j,k,n;
int64_t i,j,k;
//TODO These offset we may still need them
int64_t xStart,yStart,zStart;
xStart=yStart=zStart=0;
@ -1393,7 +1388,6 @@ void Domain::AggregateLabels( const std::string& filename, DoubleArray &UserData
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
for (int i=1; i<nx-1; i++){
int n = k*nx*ny+j*nx+i;
double local_id_val = UserData(i,j,k);
LocalID[(k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1] = local_id_val;
}

105
common/MPI.cpp
View File

@ -401,7 +401,6 @@ MPI_CLASS::MPI_CLASS()
communicator = MPI_CLASS_COMM_NULL;
d_maxTag = mpi_max_tag;
#endif
d_ranks = nullptr;
d_count = nullptr;
d_manage = false;
comm_rank = 0;
@ -435,8 +434,6 @@ void MPI_CLASS::reset()
++N_MPI_Comm_destroyed;
#endif
}
if ( d_ranks != nullptr )
delete[] d_ranks;
delete d_count;
}
if ( d_currentTag == nullptr ) {
@ -448,7 +445,6 @@ void MPI_CLASS::reset()
}
d_manage = false;
d_count = nullptr;
d_ranks = nullptr;
comm_rank = 0;
comm_size = 1;
d_maxTag = 0;
@ -467,7 +463,6 @@ MPI_CLASS::MPI_CLASS( const MPI_CLASS &comm )
d_manage( comm.d_manage ),
comm_rank( comm.comm_rank ),
comm_size( comm.comm_size ),
d_ranks( comm.d_ranks ),
d_maxTag( comm.d_maxTag ),
d_currentTag( comm.d_currentTag )
{
@ -490,7 +485,6 @@ MPI_CLASS::MPI_CLASS( MPI_CLASS &&rhs ) : MPI_CLASS()
std::swap( profile_level, rhs.profile_level );
std::swap( comm_rank, rhs.comm_rank );
std::swap( comm_size, rhs.comm_size );
std::swap( d_ranks, rhs.d_ranks );
std::swap( d_maxTag, rhs.d_maxTag );
std::swap( d_currentTag, rhs.d_currentTag );
std::swap( d_count, rhs.d_count );
@ -511,7 +505,6 @@ MPI_CLASS &MPI_CLASS::operator=( const MPI_CLASS &comm )
this->communicator = comm.communicator;
this->comm_rank = comm.comm_rank;
this->comm_size = comm.comm_size;
this->d_ranks = comm.d_ranks;
this->d_isNull = comm.d_isNull;
this->d_manage = comm.d_manage;
this->d_maxTag = comm.d_maxTag;
@ -537,7 +530,6 @@ MPI_CLASS &MPI_CLASS::operator=( MPI_CLASS &&rhs )
std::swap( profile_level, rhs.profile_level );
std::swap( comm_rank, rhs.comm_rank );
std::swap( comm_size, rhs.comm_size );
std::swap( d_ranks, rhs.d_ranks );
std::swap( d_maxTag, rhs.d_maxTag );
std::swap( d_currentTag, rhs.d_currentTag );
std::swap( d_count, rhs.d_count );
@ -560,7 +552,6 @@ std::atomic_int d_global_count_self = { 1 };
MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
{
d_count = nullptr;
d_ranks = nullptr;
d_manage = false;
tmp_alignment = -1;
// Check if we are using our version of comm_world
@ -623,11 +614,7 @@ MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
}
if ( d_manage )
++N_MPI_Comm_created;
// Create d_ranks
if ( comm_size > 1 ) {
d_ranks = new int[comm_size];
d_ranks[0] = -1;
}
#else
// We are not using MPI, intialize based on the communicator
NULL_USE( manage );
@ -663,34 +650,32 @@ MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
************************************************************************/
std::vector<int> MPI_CLASS::globalRanks() const
{
// Get my global rank if it has not been set
static int myGlobalRank = -1;
if ( myGlobalRank == -1 ) {
#ifdef USE_MPI
if ( MPI_active() )
MPI_Comm_rank( MPI_CLASS_COMM_WORLD, &myGlobalRank );
#else
myGlobalRank = 0;
#endif
}
// Check if we are dealing with a serial or null communicator
if ( comm_size == 1 )
return std::vector<int>( 1, myGlobalRank );
if ( d_ranks == nullptr || communicator == MPI_COMM_NULL )
if ( d_isNull )
return std::vector<int>();
// Fill d_ranks if necessary
if ( d_ranks[0] == -1 ) {
if ( communicator == MPI_CLASS_COMM_WORLD ) {
for ( int i = 0; i < comm_size; i++ )
d_ranks[i] = i;
} else {
MPI_ASSERT( myGlobalRank != -1 );
this->allGather( myGlobalRank, d_ranks );
#ifdef USE_MPI
// Get my global rank and size if it has not been set
static int globalRank = -1;
static int globalSize = -1;
if ( globalRank == -1 && MPI_active() ) {
MPI_Comm_rank( MPI_CLASS_COMM_WORLD, &globalRank );
MPI_Comm_size( MPI_CLASS_COMM_WORLD, &globalSize );
}
// Check if we are dealing with a serial or global communicator
if ( comm_size == 1 )
return std::vector<int>( 1, globalRank );
if ( comm_size == globalSize ) {
std::vector<int> ranks( globalSize );
for ( int i = 0; i < globalSize; i++ )
ranks[i] = i;
return ranks;
}
// Return d_ranks
return std::vector<int>( d_ranks, d_ranks + comm_size );
// Get the global rank from each rank in the communicator
auto ranks = allGather( globalRank );
std::sort( ranks.begin(), ranks.end() );
return ranks;
#else
return std::vector<int>( 1, 1 );
#endif
}
@ -2806,49 +2791,44 @@ MPI_Request MPI_CLASS::IrecvBytes(
}
/************************************************************************
* sendrecv *
************************************************************************/
#if defined( USE_MPI )
template<>
void MPI_CLASS::sendrecv<char>( const char* sendbuf, int sendcount, int dest, int sendtag,
char* recvbuf, int recvcount, int source, int recvtag ) const
void MPI_CLASS::sendrecv<char>( const char *sendbuf, int sendcount, int dest, int sendtag,
char *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<char>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag,
recvbuf, recvcount, MPI_CHAR, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag, recvbuf, recvcount, MPI_CHAR, source,
recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<char>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<int>( const int* sendbuf, int sendcount, int dest, int sendtag,
int* recvbuf, int recvcount, int source, int recvtag ) const
void MPI_CLASS::sendrecv<int>( const int *sendbuf, int sendcount, int dest, int sendtag,
int *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<int>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag,
recvbuf, recvcount, MPI_INT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag, recvbuf, recvcount, MPI_INT, source,
recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<int>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<float>( const float* sendbuf, int sendcount, int dest, int sendtag,
float* recvbuf, int recvcount, int source, int recvtag ) const
void MPI_CLASS::sendrecv<float>( const float *sendbuf, int sendcount, int dest, int sendtag,
float *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<float>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag,
recvbuf, recvcount, MPI_FLOAT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag, recvbuf, recvcount, MPI_FLOAT,
source, recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<float>", profile_level );
}
template<>
void MPI_CLASS::sendrecv<double>( const double* sendbuf, int sendcount, int dest, int sendtag,
double* recvbuf, int recvcount, int source, int recvtag ) const
void MPI_CLASS::sendrecv<double>( const double *sendbuf, int sendcount, int dest, int sendtag,
double *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<double>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag,
recvbuf, recvcount, MPI_DOUBLE, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag, recvbuf, recvcount, MPI_DOUBLE,
source, recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<double>", profile_level );
}
#endif
@ -3815,17 +3795,16 @@ MPI MPI::loadBalance( double local, std::vector<double> work )
MPI_ASSERT( (int) work.size() == getSize() );
auto perf = allGather( local );
std::vector<int> I( work.size() );
for ( size_t i=0; i<work.size(); i++)
for ( size_t i = 0; i < work.size(); i++ )
I[i] = i;
auto J = I;
quicksort( perf, I );
quicksort( work, J );
std::vector<int> key( work.size() );
for ( size_t i=0; i<work.size(); i++)
for ( size_t i = 0; i < work.size(); i++ )
key[J[i]] = I[i];
return split( 0, key[getRank()] );
}
} // namespace Utilities

20
common/MPI.h
View File

@ -8,10 +8,14 @@ Copyright (c) 2012 UT-Battelle, LLC
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
Collection of administrative costs for redistribution of the source code or binary form is allowed. However, collection of a royalty or other fee in excess of good faith amount for cost recovery for such redistribution is prohibited.
Redistribution and use in source and binary forms, with or without modification, are permitted
provided that the following conditions are met: Redistributions of source code must retain the above
copyright notice, this list of conditions and the following disclaimer. Redistributions in binary
form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution. Collection of
administrative costs for redistribution of the source code or binary form is allowed. However,
collection of a royalty or other fee in excess of good faith amount for cost recovery for such
redistribution is prohibited.
*/
@ -274,6 +278,8 @@ public: // Member functions
* \brief Return the global ranks for the comm
* \details This returns a vector which contains the global ranks for each
* member of the communicator. The global ranks are defined according to WORLD comm.
* Note: this function is a blocking collective on the current communicator
* (unless the current communicator is global, self, or null)
*/
std::vector<int> globalRanks() const;
@ -796,7 +802,8 @@ public: // Member functions
* @brief This function sends and recieves data using a blocking call
*/
template<class type>
void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf, int recvcount, int source, int recvtag ) const;
void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf,
int recvcount, int source, int recvtag ) const;
/*!
@ -1126,9 +1133,6 @@ private: // data members
// The rank and size of the communicator
int comm_rank, comm_size;
// The ranks of the comm in the global comm
mutable int *volatile d_ranks;
// Some attributes
int d_maxTag;
int *volatile d_currentTag;

154
common/ScaLBL.cpp
View File

@ -1,5 +1,8 @@
#include "common/ScaLBL.h"
#include <chrono>
ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
//......................................................................................
Lock=false; // unlock the communicator
@ -306,9 +309,128 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
}
ScaLBL_Communicator::~ScaLBL_Communicator(){
// destrutor does nothing (bad idea)
// -- note that there needs to be a way to free memory allocated on the device!!!
ScaLBL_Communicator::~ScaLBL_Communicator()
{
ScaLBL_FreeDeviceMemory( sendbuf_x );
ScaLBL_FreeDeviceMemory( sendbuf_X );
ScaLBL_FreeDeviceMemory( sendbuf_y );
ScaLBL_FreeDeviceMemory( sendbuf_Y );
ScaLBL_FreeDeviceMemory( sendbuf_z );
ScaLBL_FreeDeviceMemory( sendbuf_Z );
ScaLBL_FreeDeviceMemory( sendbuf_xy );
ScaLBL_FreeDeviceMemory( sendbuf_xY );
ScaLBL_FreeDeviceMemory( sendbuf_Xy );
ScaLBL_FreeDeviceMemory( sendbuf_XY );
ScaLBL_FreeDeviceMemory( sendbuf_xz );
ScaLBL_FreeDeviceMemory( sendbuf_xZ );
ScaLBL_FreeDeviceMemory( sendbuf_Xz );
ScaLBL_FreeDeviceMemory( sendbuf_XZ );
ScaLBL_FreeDeviceMemory( sendbuf_yz );
ScaLBL_FreeDeviceMemory( sendbuf_yZ );
ScaLBL_FreeDeviceMemory( sendbuf_Yz );
ScaLBL_FreeDeviceMemory( sendbuf_YZ );
ScaLBL_FreeDeviceMemory( recvbuf_x );
ScaLBL_FreeDeviceMemory( recvbuf_X );
ScaLBL_FreeDeviceMemory( recvbuf_y );
ScaLBL_FreeDeviceMemory( recvbuf_Y );
ScaLBL_FreeDeviceMemory( recvbuf_z );
ScaLBL_FreeDeviceMemory( recvbuf_Z );
ScaLBL_FreeDeviceMemory( recvbuf_xy );
ScaLBL_FreeDeviceMemory( recvbuf_xY );
ScaLBL_FreeDeviceMemory( recvbuf_Xy );
ScaLBL_FreeDeviceMemory( recvbuf_XY );
ScaLBL_FreeDeviceMemory( recvbuf_xz );
ScaLBL_FreeDeviceMemory( recvbuf_xZ );
ScaLBL_FreeDeviceMemory( recvbuf_Xz );
ScaLBL_FreeDeviceMemory( recvbuf_XZ );
ScaLBL_FreeDeviceMemory( recvbuf_yz );
ScaLBL_FreeDeviceMemory( recvbuf_yZ );
ScaLBL_FreeDeviceMemory( recvbuf_Yz );
ScaLBL_FreeDeviceMemory( recvbuf_YZ );
ScaLBL_FreeDeviceMemory( dvcSendList_x );
ScaLBL_FreeDeviceMemory( dvcSendList_X );
ScaLBL_FreeDeviceMemory( dvcSendList_y );
ScaLBL_FreeDeviceMemory( dvcSendList_Y );
ScaLBL_FreeDeviceMemory( dvcSendList_z );
ScaLBL_FreeDeviceMemory( dvcSendList_Z );
ScaLBL_FreeDeviceMemory( dvcSendList_xy );
ScaLBL_FreeDeviceMemory( dvcSendList_xY );
ScaLBL_FreeDeviceMemory( dvcSendList_Xy );
ScaLBL_FreeDeviceMemory( dvcSendList_XY );
ScaLBL_FreeDeviceMemory( dvcSendList_xz );
ScaLBL_FreeDeviceMemory( dvcSendList_xZ );
ScaLBL_FreeDeviceMemory( dvcSendList_Xz );
ScaLBL_FreeDeviceMemory( dvcSendList_XZ );
ScaLBL_FreeDeviceMemory( dvcSendList_yz );
ScaLBL_FreeDeviceMemory( dvcSendList_yZ );
ScaLBL_FreeDeviceMemory( dvcSendList_Yz );
ScaLBL_FreeDeviceMemory( dvcSendList_YZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_x );
ScaLBL_FreeDeviceMemory( dvcRecvList_X );
ScaLBL_FreeDeviceMemory( dvcRecvList_y );
ScaLBL_FreeDeviceMemory( dvcRecvList_Y );
ScaLBL_FreeDeviceMemory( dvcRecvList_z );
ScaLBL_FreeDeviceMemory( dvcRecvList_Z );
ScaLBL_FreeDeviceMemory( dvcRecvList_xy );
ScaLBL_FreeDeviceMemory( dvcRecvList_xY );
ScaLBL_FreeDeviceMemory( dvcRecvList_Xy );
ScaLBL_FreeDeviceMemory( dvcRecvList_XY );
ScaLBL_FreeDeviceMemory( dvcRecvList_xz );
ScaLBL_FreeDeviceMemory( dvcRecvList_xZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_Xz );
ScaLBL_FreeDeviceMemory( dvcRecvList_XZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_yz );
ScaLBL_FreeDeviceMemory( dvcRecvList_yZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_Yz );
ScaLBL_FreeDeviceMemory( dvcRecvList_YZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_x );
ScaLBL_FreeDeviceMemory( dvcRecvDist_X );
ScaLBL_FreeDeviceMemory( dvcRecvDist_y );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Y );
ScaLBL_FreeDeviceMemory( dvcRecvDist_z );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Z );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xy );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xY );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Xy );
ScaLBL_FreeDeviceMemory( dvcRecvDist_XY );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Xz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_XZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_yz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_yZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Yz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_YZ );
}
double ScaLBL_Communicator::GetPerformance(int *NeighborList, double *fq, int Np){
/* EACH MPI PROCESS GETS ITS OWN MEASUREMENT*/
/* use MRT kernels to check performance without communication / synchronization */
int TIMESTEPS=500;
double RLX_SETA=1.0;
double RLX_SETB = 8.f*(2.f-RLX_SETA)/(8.f-RLX_SETA);
double FX = 0.0;
double FY = 0.0;
double FZ = 0.0;
ScaLBL_D3Q19_Init(fq, Np);
//.......create and start timer............
Barrier();
auto t1 = std::chrono::system_clock::now();
for (int t=0; t<TIMESTEPS; t++){
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAeven_MRT(fq, FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAeven_MRT(fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
}
auto t2 = std::chrono::system_clock::now();
Barrier();
// Compute the walltime per timestep
double diff = std::chrono::duration<double>( t2 - t1 ).count();
double cputime = 0.5*diff/TIMESTEPS;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
return MLUPS;
}
int ScaLBL_Communicator::LastExterior(){
return next;
@ -364,7 +486,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
int idx,i,j,k,n;
// Check that Map has size matching sub-domain
if (Map.size(0) != Nx)
if ( (int) Map.size(0) != Nx)
ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Map array dimensions do not match! \n");
// Initialize Map
@ -1501,35 +1623,31 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){
// Pack the distributions
//...Packing for x face(2,8,10,12,14)................................
ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
//...................................................................................
// Send all the distributions
//...................................................................................
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
}

89
common/ScaLBL.h
View File

@ -70,49 +70,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int
extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
// ION TRANSPORT MODEL
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
// LBM Poisson solver
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
//maybe deprecated
//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
// int strideY, int strideZ,int start, int finish, int Np);
// LBM Stokes Model (adapted from MRT model)
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
@ -129,6 +86,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure,
double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
// ION TRANSPORT MODEL
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
@ -197,6 +155,12 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz);
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz);
@ -220,6 +184,44 @@ extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, doub
extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double *ColorGrad, int start, int finish, int Np);
// FREE ENERGY LEE MODEL
extern "C" void ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *mu_phi, double *ColorGrad, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad,
double rhonA, double rhoB, double tauM, double W, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np);
// BOUNDARY CONDITION ROUTINES
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
@ -317,6 +319,7 @@ public:
int FirstInterior();
int LastInterior();
double GetPerformance(int *NeighborList, double *fq, int Np);
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np, int width);
void Barrier(){
ScaLBL_DeviceBarrier();

120
common/WideHalo.cpp
View File

@ -58,17 +58,16 @@ ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain
rank_xyZ = rank_info.rank[0][0][2];
rank_XyZ = rank_info.rank[2][0][2];
rank_xYZ = rank_info.rank[0][2][2];
MPI_COMM_SCALBL.barrier();
/* Fill in communications patterns for the lists */
/* Send lists */
sendCount_x = getHaloBlock(width,2*width,width,Nyh-width,width,Nzh-width,dvcSendList_x);
sendCount_x =getHaloBlock(width,2*width,width,Nyh-width,width,Nzh-width,dvcSendList_x);
sendCount_X =getHaloBlock(Nxh-2*width,Nxh-width,width,Nyh-width,width,Nzh-width,dvcSendList_X);
sendCount_y =getHaloBlock(width,Nxh-width,width,2*width,width,Nzh-width,dvcSendList_y);
sendCount_Y =getHaloBlock(width,Nxh-width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_Y);
sendCount_z =getHaloBlock(width,Nxh-width,width,Nyh-width,width,2*width,dvcSendList_z);
sendCount_X =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_Z);
sendCount_Z =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_Z);
// xy
sendCount_xy =getHaloBlock(width,2*width,width,2*width,width,Nzh-width,dvcSendList_xy);
sendCount_xY =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_xY);
@ -88,7 +87,7 @@ ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain
sendCount_xyz =getHaloBlock(width,2*width,width,2*width,width,2*width,dvcSendList_xyz);
sendCount_xyZ =getHaloBlock(width,2*width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_xyZ);
sendCount_xYz =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_xYz);
sendCount_xYz =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xYZ);
sendCount_xYZ =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xYZ);
sendCount_Xyz =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,width,2*width,dvcSendList_Xyz);
sendCount_XyZ =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_XyZ);
sendCount_XYz =getHaloBlock(Nxh-2*width,Nxh-width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_XYz);
@ -235,60 +234,61 @@ void ScaLBLWideHalo_Communicator::Send(double *data){
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = MPI_COMM_SCALBL.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = MPI_COMM_SCALBL.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = MPI_COMM_SCALBL.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = MPI_COMM_SCALBL.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = MPI_COMM_SCALBL.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = MPI_COMM_SCALBL.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = MPI_COMM_SCALBL.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = MPI_COMM_SCALBL.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = MPI_COMM_SCALBL.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = MPI_COMM_SCALBL.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = MPI_COMM_SCALBL.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = MPI_COMM_SCALBL.Irecv(&recvCount_z,1,rank_z,recvtag+5);
req1[6] = MPI_COMM_SCALBL.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = MPI_COMM_SCALBL.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = MPI_COMM_SCALBL.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = MPI_COMM_SCALBL.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = MPI_COMM_SCALBL.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = MPI_COMM_SCALBL.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = MPI_COMM_SCALBL.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = MPI_COMM_SCALBL.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
req1[10] = MPI_COMM_SCALBL.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = MPI_COMM_SCALBL.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = MPI_COMM_SCALBL.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = MPI_COMM_SCALBL.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = MPI_COMM_SCALBL.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = MPI_COMM_SCALBL.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = MPI_COMM_SCALBL.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = MPI_COMM_SCALBL.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
req1[14] = MPI_COMM_SCALBL.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = MPI_COMM_SCALBL.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = MPI_COMM_SCALBL.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = MPI_COMM_SCALBL.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = MPI_COMM_SCALBL.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = MPI_COMM_SCALBL.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = MPI_COMM_SCALBL.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = MPI_COMM_SCALBL.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x,sendCount_x,rank_x,sendtag+0);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag+0);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X,sendCount_X,rank_X,sendtag+1);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag+1);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y,sendCount_y,rank_y,sendtag+2);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag+2);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag+3);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag+3);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z,sendCount_z,rank_z,sendtag+4);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag+4);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag+5);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag+5);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag+6);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag+6);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag+7);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag+7);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag+8);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag+8);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag+9);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag+9);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag+10);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag+10);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag+11);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag+11);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag+12);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag+12);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag+13);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag+13);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag+14);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag+14);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag+15);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag+15);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag+16);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag+16);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag+17);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag+17);
/* Corners */
req1[18] = MPI_COMM_SCALBL.Isend(&sendCount_xyz,1,rank_xyz,sendtag+18);
req2[18] = MPI_COMM_SCALBL.Irecv(&recvCount_XYZ,1,rank_XYZ,recvtag+18);
req1[19] = MPI_COMM_SCALBL.Isend(&sendCount_XYz,1,rank_XYz,sendtag+19);
req2[19] = MPI_COMM_SCALBL.Irecv(&recvCount_xyZ,1,rank_xyZ,recvtag+19);
req1[20] = MPI_COMM_SCALBL.Isend(&sendCount_Xyz,1,rank_Xyz,sendtag+20);
req2[20] = MPI_COMM_SCALBL.Irecv(&recvCount_xYZ,1,rank_xYZ,recvtag+20);
req1[21] = MPI_COMM_SCALBL.Isend(&sendCount_xYz,1,rank_xYz,sendtag+21);
req2[21] = MPI_COMM_SCALBL.Irecv(&recvCount_XyZ,1,rank_XyZ,recvtag+21);
req1[22] = MPI_COMM_SCALBL.Isend(&sendCount_xyZ,1,rank_xyZ,sendtag+22);
req2[22] = MPI_COMM_SCALBL.Irecv(&recvCount_XYz,1,rank_XYz,recvtag+22);
req1[23] = MPI_COMM_SCALBL.Isend(&sendCount_XYZ,1,rank_XYZ,sendtag+23);
req2[23] = MPI_COMM_SCALBL.Irecv(&recvCount_xyz,1,rank_xyz,recvtag+23);
req1[24] = MPI_COMM_SCALBL.Isend(&sendCount_XyZ,1,rank_XyZ,sendtag+24);
req2[24] = MPI_COMM_SCALBL.Irecv(&recvCount_xYz,1,rank_xYz,recvtag+24);
req1[25] = MPI_COMM_SCALBL.Isend(&sendCount_xYZ,1,rank_xYZ,sendtag+25);
req2[25] = MPI_COMM_SCALBL.Irecv(&recvCount_Xyz,1,rank_Xyz,recvtag+25);
req1[18] = MPI_COMM_SCALBL.Isend(sendbuf_xyz,sendCount_xyz,rank_xyz,sendtag+18);
req2[18] = MPI_COMM_SCALBL.Irecv(recvbuf_XYZ,recvCount_XYZ,rank_XYZ,recvtag+18);
req1[19] = MPI_COMM_SCALBL.Isend(sendbuf_XYz,sendCount_XYz,rank_XYz,sendtag+19);
req2[19] = MPI_COMM_SCALBL.Irecv(recvbuf_xyZ,recvCount_xyZ,rank_xyZ,recvtag+19);
req1[20] = MPI_COMM_SCALBL.Isend(sendbuf_Xyz,sendCount_Xyz,rank_Xyz,sendtag+20);
req2[20] = MPI_COMM_SCALBL.Irecv(recvbuf_xYZ,recvCount_xYZ,rank_xYZ,recvtag+20);
req1[21] = MPI_COMM_SCALBL.Isend(sendbuf_xYz,sendCount_xYz,rank_xYz,sendtag+21);
req2[21] = MPI_COMM_SCALBL.Irecv(recvbuf_XyZ,recvCount_XyZ,rank_XyZ,recvtag+21);
req1[22] = MPI_COMM_SCALBL.Isend(sendbuf_xyZ,sendCount_xyZ,rank_xyZ,sendtag+22);
req2[22] = MPI_COMM_SCALBL.Irecv(recvbuf_XYz,recvCount_XYz,rank_XYz,recvtag+22);
req1[23] = MPI_COMM_SCALBL.Isend(sendbuf_XYZ,sendCount_XYZ,rank_XYZ,sendtag+23);
req2[23] = MPI_COMM_SCALBL.Irecv(recvbuf_xyz,recvCount_xyz,rank_xyz,recvtag+23);
req1[24] = MPI_COMM_SCALBL.Isend(sendbuf_XyZ,sendCount_XyZ,rank_XyZ,sendtag+24);
req2[24] = MPI_COMM_SCALBL.Irecv(recvbuf_xYz,recvCount_xYz,rank_xYz,recvtag+24);
req1[25] = MPI_COMM_SCALBL.Isend(sendbuf_xYZ,sendCount_xYZ,rank_xYZ,sendtag+25);
req2[25] = MPI_COMM_SCALBL.Irecv(recvbuf_Xyz,recvCount_Xyz,rank_Xyz,recvtag+25);
//...................................................................................
}
@ -298,11 +298,13 @@ ScaLBLWideHalo_Communicator::~ScaLBLWideHalo_Communicator()
void ScaLBLWideHalo_Communicator::Recv(double *data){
//...................................................................................
MPI_Waitall(26,req1,stat1);
MPI_Waitall(26,req2,stat2);
Utilities::MPI::waitAll(26,req1);
Utilities::MPI::waitAll(26,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
//...................................................................................
//printf("Ready to unpack %i to x\n",recvCount_x);
//printf(" print first 10 values...\n");
//for (int idx=0; idx<10; idx++) printf(" recvBuf[%i]=%f \n",idx,recvbuf_x[idx]);
ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh);

7
common/WideHalo.h
View File

@ -4,6 +4,7 @@ This class implements support for halo widths larger than 1
#ifndef WideHalo_H
#define WideHalo_H
#include "common/ScaLBL.h"
#include "common/MPI.h"
class ScaLBLWideHalo_Communicator{
public:
@ -52,9 +53,7 @@ private:
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
MPI_Request req1[26],req2[26];
MPI_Status stat1[26],stat2[26];
//......................................................................................
// MPI ranks for all 18 neighbors
//......................................................................................
@ -95,11 +94,11 @@ private:
int *dvcRecvList_xyZ,*dvcRecvList_XyZ,*dvcRecvList_xYZ,*dvcRecvList_XYZ;
//......................................................................................
inline int getHaloBlock(int imin, int imax, int jmin, int jmax, int kmin, int kmax, int *dvcList){
inline int getHaloBlock(int imin, int imax, int jmin, int jmax, int kmin, int kmax, int *& dvcList){
int count = 0;
int *List;
List = new int [(imax-imin)*(jmax-jmin)*(kmax-kmin)];
for (int k=kmin; k<kmax; k++){
for (k=kmin; k<kmax; k++){
for (j=jmin; j<jmax; j++){
for (i=imin; i<imax; i++){
List[count++] = k*Nxh*Nyh + j*Nxh + i;

194
cpu/Color.cpp
View File

@ -2489,10 +2489,200 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
int nr1,nr2,nr3,nr4,nr5,nr6;
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* neighbors */
nr1 = neighborList[n+0*Np];
nr2 = neighborList[n+1*Np];
nr3 = neighborList[n+2*Np];
nr4 = neighborList[n+3*Np];
nr5 = neighborList[n+4*Np];
nr6 = neighborList[n+5*Np];
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
// compute phase indicator field
phi=(nA-nB)/(nA+nB);
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
// q = 1
//nread = neighborList[n+Np];
Aq[nr2] = a1;
Bq[nr2] = b1;
// q=2
//nread = neighborList[n];
Aq[nr1] = a2;
Bq[nr1] = b2;
//...............................................
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
// q = 3
//nread = neighborList[n+3*Np];
Aq[nr4] = a1;
Bq[nr4] = b1;
// q = 4
//nread = neighborList[n+2*Np];
Aq[nr3] = a2;
Bq[nr3] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
// q = 5
//nread = neighborList[n+5*Np];
Aq[nr6] = a1;
Bq[nr6] = b1;
// q = 6
//nread = neighborList[n+4*Np];
Aq[nr5] = a2;
Bq[nr5] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
Aq[1*Np+n] = a1;
Bq[1*Np+n] = b1;
Aq[2*Np+n] = a2;
Bq[2*Np+n] = b2;
//...............................................
// q = 2
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
Aq[3*Np+n] = a1;
Bq[3*Np+n] = b1;
Aq[4*Np+n] = a2;
Bq[4*Np+n] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
Aq[5*Np+n] = a1;
Bq[5*Np+n] = b1;
Aq[6*Np+n] = a2;
Bq[6*Np+n] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq,
double *Den, double *Phi, int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){
@ -2579,7 +2769,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){

4918
cpu/FreeLee.cpp
View File

File diff suppressed because it is too large Load Diff

2
ctest_script.cmake
View File

@ -164,7 +164,7 @@ SET( CTEST_OPTIONS )
SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
IF ( USE_CUDA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" )
ELSE()

2122
cuda/FreeLee.cu Normal file
View File

File diff suppressed because it is too large Load Diff

76
cuda/MixedGradient.cu Normal file
View File

@ -0,0 +1,76 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
#include <cuda.h>
#include <stdio.h>
#include <cuda_profiler_api.h>
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N,idx;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
N = Nx*Ny*Nz;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = start + S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (idx<finish){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}
}
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
cudaProfilerStart();
dvc_ScaLBL_D3Q19_MixedGradient<<<NBLOCKS,NTHREADS >>>(Map, Phi, Gradient, start, finish, Np, Nx, Ny, Nz);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_MixedGradient: %s \n",cudaGetErrorString(err));
}
cudaProfilerStop();
}

2
example/Piston/input.db
View File

@ -35,4 +35,6 @@ Analysis {
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}
Visualization {
}

15
example/Plates/input.db
View File

@ -7,10 +7,10 @@ Color {
beta = 0.95;
F = 0, 0, 0
Restart = false
timestepMax = 3000
timestepMax = 500
flux = 0.0
ComponentLabels = -2, -1
ComponentAffinity = -1.0, -0.5;
ComponentAffinity = 1.0, 1.0;
}
Domain {
@ -26,13 +26,14 @@ Domain {
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 1000 // Frequency to write restart data
visualization_interval = 1000 // Frequency to write visualization data
analysis_interval = 100 // Frequency to perform analysis
visualization_interval = 500
subphase_analysis_interval = 100
restart_interval = 100000
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}
Visualization {
}

5
hip/CMakeLists.txt
View File

@ -1,6 +1,7 @@
SET( HIP_SEPERABLE_COMPILATION ON )
SET_SOURCE_FILES_PROPERTIES( BGK.cu Color.cu CudaExtras.cu D3Q19.cu D3Q7.cu dfh.cu Extras.cu MRT.hip PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 )
HIP_ADD_LIBRARY( lbpm-hip BGK.cu Color.cu CudaExtras.cu D3Q19.cu D3Q7.cu dfh.cu Extras.cu MRT.cu SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} )
FILE( GLOB HIP_SOURCES "*.cu" )
SET_SOURCE_FILES_PROPERTIES( ${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 )
HIP_ADD_LIBRARY( lbpm-hip ${HIP_SOURCES} SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} )
#TARGET_LINK_LIBRARIES( lbpm-hip /opt/rocm-3.3.0/lib/libhip_hcc.so )
#TARGET_LINK_LIBRARIES( lbpm-wia lbpm-hip )
#ADD_DEPENDENCIES( lbpm-hip copy-include )

72
hip/D3Q19.cu
View File

@ -89,9 +89,25 @@ __global__ void sum_kernel_block(double *sum, double *input, int n)
__inline__ __device__
double warpReduceSum(double val) {
#if 0
for (int offset = warpSize/2; offset > 0; offset /= 2)
val += __shfl_down_sync(0xFFFFFFFF, val, offset, 32);
return val;
#else
short int id = threadIdx.x % warpSize;
__shared__ double tmp[64];
tmp[id] = val;
__syncthreads();
if ( warpSize == 64) {
tmp[id] += tmp[id+32]; __syncthreads();
}
tmp[id] += tmp[id+16]; __syncthreads();
tmp[id] += tmp[id+8]; __syncthreads();
tmp[id] += tmp[id+4]; __syncthreads();
tmp[id] += tmp[id+2]; __syncthreads();
tmp[id] += tmp[id+1]; __syncthreads();
return tmp[0];
#endif
}
__inline__ __device__
@ -1730,6 +1746,44 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist,
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f5 = 0.111111111111111111111111 - dist[6*Np+n];
double f11 = 0.05555555555555555555556 - dist[12*Np+n];
double f14 = 0.05555555555555555555556 - dist[13*Np+n];
double f15 = 0.05555555555555555555556 - dist[16*Np+n];
double f18 = 0.05555555555555555555556 - dist[17*Np+n];
dist[6*Np+n] = f5;
dist[12*Np+n] = f11;
dist[13*Np+n] = f14;
dist[16*Np+n] = f15;
dist[17*Np+n] = f18;
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f6 = 0.111111111111111111111111 - dist[5*Np+n];
double f12 = 0.05555555555555555555556 - dist[11*Np+n];
double f13 = 0.05555555555555555555556 - dist[14*Np+n] ;
double f16 = 0.05555555555555555555556 - dist[15*Np+n];
double f17 = 0.05555555555555555555556 - dist[18*Np+n];
dist[5*Np+n] = f6;
dist[11*Np+n] = f12;
dist[14*Np+n] = f13;
dist[15*Np+n] = f16;
dist[18*Np+n] = f17;
}
}
__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
{
int idx, n;
@ -2605,6 +2659,24 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
}
extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("HIP error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("HIP error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" double deviceReduce(double *in, double* out, int N) {
int threads = 512;
int blocks = min((N + threads - 1) / threads, 1024);

536
hip/D3Q7BC.cu Normal file
View File

@ -0,0 +1,536 @@
#include <math.h>
#include <stdio.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
}
}
__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = value_q + value_b;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vin;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vout;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
// unknown distributions
nr6 = d_neighborList[n+5*Np];
dist[nr6] = f6;
}
}
//*************************************************************************
extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Dirichlet_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Neumann_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<<GRID,512>>>(list, dist, Vin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<<GRID,512>>>(list, dist, Vout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Vin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Vout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_z<<<GRID,512>>>(list, Map, Psi, Vin, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_Z<<<GRID,512>>>(list, Map, Psi, Vout, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<<GRID,512>>>(list, dist, Cin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<<GRID,512>>>(list, dist, Cout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Cin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Cout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}

2122
hip/FreeLee.cu Normal file
View File

File diff suppressed because it is too large Load Diff

2745
hip/Greyscale.cu Normal file
View File

File diff suppressed because it is too large Load Diff

3038
hip/GreyscaleColor.cu Normal file
View File

File diff suppressed because it is too large Load Diff

392
hip/Ion.cu Normal file
View File

@ -0,0 +1,392 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
int n,nread;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
Ci += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
Ci += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
Ci += fq;
// q=4
nread = neighborList[n+3*Np];
fq = dist[nread];
Ci += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
Ci += fq;
// q=6
nread = neighborList[n+5*Np];
fq = dist[nread];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
int n;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
fq = dist[2*Np+n];
Ci += fq;
// q=2
fq = dist[1*Np+n];
Ci += fq;
// q=3
fq = dist[4*Np+n];
Ci += fq;
// q=4
fq = dist[3*Np+n];
Ci += fq;
// q=5
fq = dist[6*Np+n];
Ci += fq;
// q=6
fq = dist[5*Np+n];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
// q=2
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q=4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q=6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
int n;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
Den[n] = DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
int n;
double DenInit;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
DenInit = Den[n];
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
int n;
double Ci;//ion concentration of species i
double CD;//charge density
double CD_tmp;
double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
Ci = Den[n+ion_component*Np];
CD = ChargeDensity[n];
CD_tmp = F*IonValence*Ci;
ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<<NBLOCKS,NTHREADS >>>(dist,Den,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init<<<NBLOCKS,NTHREADS >>>(dist,Den,DenInit,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_Init: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init_FromFile<<<NBLOCKS,NTHREADS >>>(dist,Den,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<<NBLOCKS,NTHREADS >>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

77
hip/MixedGradient.cu Normal file
View File

@ -0,0 +1,77 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
#include <stdio.h>
//#include <cuda_profiler_api.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static const int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N,idx;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
N = Nx*Ny*Nz;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = start + S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (idx<finish){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}
}
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
hipProfilerStart();
dvc_ScaLBL_D3Q19_MixedGradient<<<NBLOCKS,NTHREADS >>>(Map, Phi, Gradient, start, finish, Np, Nx, Ny, Nz);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_MixedGradient: %s \n",hipGetErrorString(err));
}
hipProfilerStop();
}

330
hip/Poisson.cu Normal file
View File

@ -0,0 +1,330 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int nread;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
psi += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
psi += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
psi += fq;
// q = 4
nread = neighborList[n+3*Np];
fq = dist[nread];
psi += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
psi += fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
fq = dist[2*Np+n];
psi += fq;
// q=2
fq = dist[1*Np+n];
psi += fq;
// q=3
fq = dist[4*Np+n];
psi += fq;
// q=4
fq = dist[3*Np+n];
psi += fq;
// q=5
fq = dist[6*Np+n];
psi += fq;
// q=6
fq = dist[5*Np+n];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q = 4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q = 6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
int ijk;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
ijk = Map[n];
dist[0*Np+n] = 0.25*Psi[ijk];
dist[1*Np+n] = 0.125*Psi[ijk];
dist[2*Np+n] = 0.125*Psi[ijk];
dist[3*Np+n] = 0.125*Psi[ijk];
dist[4*Np+n] = 0.125*Psi[ijk];
dist[5*Np+n] = 0.125*Psi[ijk];
dist[6*Np+n] = 0.125*Psi[ijk];
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson<<<NBLOCKS,NTHREADS >>>(Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Poisson_Init<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Poisson_Init: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

996
hip/Stokes.cu Normal file
View File

@ -0,0 +1,996 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
int nread;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
fq = dist[nread]; // reading the f1 data into register fq
//fp = dist[10*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
fq = dist[nread]; // reading the f2 data into register fq
//fq = dist[Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
nread = neighborList[n+2*Np]; // neighbor 4
fq = dist[nread];
//fq = dist[11*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
nread = neighborList[n+3*Np]; // neighbor 3
fq = dist[nread];
//fq = dist[2*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
//fq = dist[12*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
//fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
nread = neighborList[n+6*Np];
fq = dist[nread];
//fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
nread = neighborList[n+7*Np];
fq = dist[nread];
//fq = dist[4*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
nread = neighborList[n+8*Np];
fq = dist[nread];
//fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
nread = neighborList[n+9*Np];
fq = dist[nread];
//fq = dist[5*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
nread = neighborList[n+10*Np];
fq = dist[nread];
//fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
nread = neighborList[n+11*Np];
fq = dist[nread];
//fq = dist[6*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
nread = neighborList[n+12*Np];
fq = dist[nread];
//fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
nread = neighborList[n+13*Np];
fq = dist[nread];
//fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
nread = neighborList[n+14*Np];
fq = dist[nread];
//fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
nread = neighborList[n+15*Np];
fq = dist[nread];
//fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
//fq = dist[18*Np+n];
nread = neighborList[n+16*Np];
fq = dist[nread];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
nread = neighborList[n+17*Np];
fq = dist[nread];
//fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
nread = neighborList[n+Np];
dist[nread] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
nread = neighborList[n];
dist[nread] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
nread = neighborList[n+3*Np];
dist[nread] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
nread = neighborList[n+2*Np];
dist[nread] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
nread = neighborList[n+5*Np];
dist[nread] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
nread = neighborList[n+4*Np];
dist[nread] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
nread = neighborList[n+7*Np];
dist[nread] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
nread = neighborList[n+6*Np];
dist[nread] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
nread = neighborList[n+9*Np];
dist[nread] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
nread = neighborList[n+8*Np];
dist[nread] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
nread = neighborList[n+11*Np];
dist[nread] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
nread = neighborList[n+10*Np];
dist[nread]= fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
nread = neighborList[n+13*Np];
dist[nread] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
nread = neighborList[n+12*Np];
dist[nread] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
nread = neighborList[n+15*Np];
dist[nread] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
nread = neighborList[n+14*Np];
dist[nread] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
nread = neighborList[n+17*Np];
dist[nread] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
nread = neighborList[n+16*Np];
dist[nread] = fq;
}
}
}
__global__ void dvc_ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
fq = dist[2*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
fq = dist[1*Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
fq = dist[4*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
fq = dist[6*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
fq = dist[5*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
fq = dist[10*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
fq = dist[12*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
fq = dist[11*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
fq = dist[18*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//........................................................................
// READ THE DISTRIBUTIONS
// (read from opposite array due to previous swap operation)
//........................................................................
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
dist[1*Np+n] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
dist[2*Np+n] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
dist[3*Np+n] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
dist[4*Np+n] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
dist[5*Np+n] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
dist[6*Np+n] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
dist[7*Np+n] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
dist[8*Np+n] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
dist[9*Np+n] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
dist[10*Np+n] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
dist[11*Np+n] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
dist[12*Np+n] = fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
dist[13*Np+n] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
dist[14*Np+n] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
dist[15*Np+n] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
dist[16*Np+n] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
dist[17*Np+n] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
dist[18*Np+n] = fq;
//........................................................................
}
}
}
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAodd_StokesMRT<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_AAodd_StokesMRT: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAeven_StokesMRT<<<NBLOCKS,NTHREADS >>>(dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_AAeven_StokesMRT: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

261
models/ColorModel.cpp
View File

@ -9,15 +9,33 @@ color lattice boltzmann model
#include <stdlib.h>
#include <time.h>
ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
rank(RANK), nprocs(NP), Restart(0), timestep(0), timestepMax(0),
tauA(0), tauB(0), rhoA(0), rhoB(0), alpha(0), beta(0),
Fx(0), Fy(0), Fz(0), flux(0), din(0), dout(0),
inletA(0), inletB(0), outletA(0), outletB(0),
Nx(0), Ny(0), Nz(0), N(0), Np(0), nprocx(0), nprocy(0), nprocz(0),
BoundaryCondition(0), Lx(0), Ly(0), Lz(0), id(nullptr),
NeighborList(nullptr), dvcMap(nullptr), fq(nullptr), Aq(nullptr), Bq(nullptr),
Den(nullptr), Phi(nullptr), ColorGrad(nullptr), Velocity(nullptr), Pressure(nullptr),
comm(COMM)
{
REVERSE_FLOW_DIRECTION = false;
}
ScaLBL_ColorModel::~ScaLBL_ColorModel(){
ScaLBL_ColorModel::~ScaLBL_ColorModel()
{
delete [] id;
ScaLBL_FreeDeviceMemory( NeighborList );
ScaLBL_FreeDeviceMemory( dvcMap );
ScaLBL_FreeDeviceMemory( fq );
ScaLBL_FreeDeviceMemory( Aq );
ScaLBL_FreeDeviceMemory( Bq );
ScaLBL_FreeDeviceMemory( Den );
ScaLBL_FreeDeviceMemory( Phi );
ScaLBL_FreeDeviceMemory( Pressure );
ScaLBL_FreeDeviceMemory( Velocity );
ScaLBL_FreeDeviceMemory( ColorGrad );
}
/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np)
@ -238,9 +256,26 @@ void ScaLBL_ColorModel::ReadInput(){
}
}
// MeanFilter(Averages->SDs);
Minkowski Solid(Dm);
if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
CalcDist(Averages->SDs,id_solid,*Mask);
Solid.ComputeScalar(Averages->SDs,0.0);
/* save averages */
Averages->solid.V = Solid.Vi;
Averages->solid.A = Solid.Ai;
Averages->solid.H = Solid.Ji;
Averages->solid.X = Solid.Xi;
Averages->gsolid.V = Solid.Vi_global;
Averages->gsolid.A = Solid.Ai_global;
Averages->gsolid.H = Solid.Ji_global;
Averages->gsolid.X = Solid.Xi_global;
/* write to file */
if (rank == 0) {
FILE *SOLID = fopen("solid.csv","w");
fprintf(SOLID,"Vs As Hs Xs\n");
fprintf(SOLID,"%.8g %.8g %.8g %.8g\n",Solid.Vi_global,Solid.Ai_global,Solid.Ji_global,Solid.Xi_global);
fclose(SOLID);
}
if (rank == 0) cout << "Domain set." << endl;
Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
@ -254,12 +289,17 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
auto LabelList = color_db->getVector<int>( "ComponentLabels" );
auto AffinityList = color_db->getVector<double>( "ComponentAffinity" );
auto WettingConvention = color_db->getWithDefault<std::string>( "WettingConvention", "none" );
NLABELS=LabelList.size();
if (NLABELS != AffinityList.size()){
ERROR("Error: ComponentLabels and ComponentAffinity must be the same length! \n");
}
if (WettingConvention == "SCAL"){
for (size_t idx=0; idx<NLABELS; idx++) AffinityList[idx] *= -1.0;
}
double label_count[NLABELS];
double label_count_global[NLABELS];
// Assign the labels
@ -391,11 +431,13 @@ void ScaLBL_ColorModel::Create(){
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
delete [] neighborList;
// initialize phi based on PhaseLabel (include solid component labels)
double *PhaseLabel;
PhaseLabel = new double[N];
AssignComponentLabels(PhaseLabel);
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
delete [] PhaseLabel;
}
/********************************************************
@ -489,6 +531,121 @@ void ScaLBL_ColorModel::Initialize(){
ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double));
}
double ScaLBL_ColorModel::Run(int returntime){
int nprocs=nprocx*nprocy*nprocz;
//************ MAIN ITERATION LOOP ***************************************/
comm.barrier();
PROFILE_START("Loop");
//std::shared_ptr<Database> analysis_db;
bool Regular = false;
auto current_db = db->cloneDatabase();
auto t1 = std::chrono::system_clock::now();
int START_TIMESTEP = timestep;
int EXIT_TIMESTEP = min(timestepMax,returntime);
while (timestep < EXIT_TIMESTEP ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
// *************ODD TIMESTEP*************
timestep++;
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
if (BoundaryCondition > 0 && BoundaryCondition < 5){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
// Halo exchange for phase field
ScaLBL_Comm_Regular->SendHalo(Phi);
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm_Regular->RecvHalo(Phi);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set BCs
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 5){
ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
}
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->Barrier();
// *************EVEN TIMESTEP*************
timestep++;
// Compute the Phase indicator field
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
// Halo exchange for phase field
if (BoundaryCondition > 0 && BoundaryCondition < 5){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
ScaLBL_Comm_Regular->SendHalo(Phi);
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm_Regular->RecvHalo(Phi);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 5){
ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
}
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->Barrier();
//************************************************************************
}
PROFILE_STOP("Update");
PROFILE_STOP("Loop");
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
// Compute the walltime per timestep
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / (timestep - START_TIMESTEP);
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
return(MLUPS);
MLUPS *= nprocs;
}
void ScaLBL_ColorModel::Run(){
int nprocs=nprocx*nprocy*nprocz;
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
@ -538,7 +695,6 @@ void ScaLBL_ColorModel::Run(){
if (color_db->keyExists( "krA_morph_factor" )){
KRA_MORPH_FACTOR = color_db->getScalar<double>( "krA_morph_factor" );
}
/* defaults for simulation protocols */
auto protocol = color_db->getWithDefault<std::string>( "protocol", "none" );
if (protocol == "image sequence"){
@ -583,7 +739,7 @@ void ScaLBL_ColorModel::Run(){
if (analysis_db->keyExists( "seed_water" )){
seed_water = analysis_db->getScalar<double>( "seed_water" );
if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water);
USE_SEED = true;
ASSERT(protocol == "seed water");
}
if (analysis_db->keyExists( "morph_delta" )){
morph_delta = analysis_db->getScalar<double>( "morph_delta" );
@ -614,7 +770,6 @@ void ScaLBL_ColorModel::Run(){
MAX_MORPH_TIMESTEPS = analysis_db->getScalar<int>( "max_morph_timesteps" );
}
if (rank==0){
printf("********************************************************\n");
if (protocol == "image sequence"){
@ -651,20 +806,15 @@ void ScaLBL_ColorModel::Run(){
fflush(stdout);
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
comm.barrier();
PROFILE_START("Loop");
//std::shared_ptr<Database> analysis_db;
bool Regular = false;
auto current_db = db->cloneDatabase();
runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
//analysis.createThreads( analysis_method, 4 );
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
@ -997,10 +1147,10 @@ void ScaLBL_ColorModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_Comm->Barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
@ -1080,7 +1230,6 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double));
// Extract only the connected part of NWP
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm);
comm.barrier();
@ -1202,6 +1351,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
}
return(volume_change);
}
double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){
srand(time(NULL));
double mass_loss =0.f;
@ -1283,7 +1433,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
double vF = 0.f;
double vS = 0.f;
double delta_volume;
double WallFactor = 0.0;
double WallFactor = 1.0;
bool USE_CONNECTED_NWP = false;
DoubleArray phase(Nx,Ny,Nz);
@ -1306,6 +1456,11 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
}
}
double volume_initial = Dm->Comm.sumReduce( count);
double PoreVolume = Dm->Volume*Dm->Porosity();
/*ensure target isn't an absurdly small fraction of pore volume */
if (volume_initial < target_delta_volume*PoreVolume){
volume_initial = target_delta_volume*PoreVolume;
}
/*
sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank);
FILE *INPUT = fopen(LocalRankFilename,"wb");
@ -1317,7 +1472,6 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
double volume_connected = 0.0;
double second_biggest = 0.0;
if (USE_CONNECTED_NWP){
BlobIDstruct new_index;
ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm);
comm.barrier();
@ -1566,3 +1720,68 @@ void ScaLBL_ColorModel::WriteDebug(){
fclose(CGZ_FILE);
*/
}
FlowAdaptor::FlowAdaptor(ScaLBL_ColorModel &M){
Nx = M.Dm->Nx;
Ny = M.Dm->Ny;
Nz = M.Dm->Nz;
timestep=-1;
timestep_previous=-1;
phi.resize(Nx,Ny,Nz); phi.fill(0); // phase indicator field
phi_t.resize(Nx,Ny,Nz); phi_t.fill(0); // time derivative for the phase indicator field
}
FlowAdaptor::~FlowAdaptor(){
}
double FlowAdaptor::MoveInterface(ScaLBL_ColorModel &M){
double INTERFACE_CUTOFF = M.color_db->getWithDefault<double>( "move_interface_cutoff", 0.975 );
double MOVE_INTERFACE_FACTOR = M.color_db->getWithDefault<double>( "move_interface_factor", 10.0 );
ScaLBL_CopyToHost( phi.data(), M.Phi, Nx*Ny*Nz* sizeof( double ) );
/* compute the local derivative of phase indicator field */
double beta = M.beta;
double factor = 0.5/beta;
double total_interface_displacement = 0.0;
double total_interface_sites = 0.0;
for (int n=0; n<Nx*Ny*Nz; n++){
/* compute the distance to the interface */
double value1 = M.Averages->Phi(n);
double dist1 = factor*log((1.0+value1)/(1.0-value1));
double value2 = phi(n);
double dist2 = factor*log((1.0+value2)/(1.0-value2));
phi_t(n) = value2;
if (value1 < INTERFACE_CUTOFF && value1 > -1*INTERFACE_CUTOFF && value2 < INTERFACE_CUTOFF && value2 > -1*INTERFACE_CUTOFF ){
/* time derivative of distance */
double dxdt = 0.125*(dist2-dist1);
/* extrapolate to move the distance further */
double dist3 = dist2 + MOVE_INTERFACE_FACTOR*dxdt;
/* compute the new phase interface */
phi_t(n) = (2.f*(exp(-2.f*beta*(dist3)))/(1.f+exp(-2.f*beta*(dist3))) - 1.f);
total_interface_displacement += fabs(MOVE_INTERFACE_FACTOR*dxdt);
total_interface_sites += 1.0;
}
}
ScaLBL_CopyToDevice( M.Phi, phi_t.data(), Nx*Ny*Nz* sizeof( double ) );
/* ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){
if (Dm->kproc()==0){
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2);
}
if (Dm->kproc() == nprocz-1){
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3);
}
}
*/
}

20
models/ColorModel.h
View File

@ -16,6 +16,10 @@ Implementation of color lattice boltzmann model
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
#ifndef ScaLBL_ColorModel_INC
#define ScaLBL_ColorModel_INC
class ScaLBL_ColorModel{
public:
ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM);
@ -29,7 +33,9 @@ public:
void Create();
void Initialize();
void Run();
double Run(int returntime);
void WriteDebug();
void getPhaseField(DoubleArray &f);
bool Restart,pBC;
bool REVERSE_FLOW_DIRECTION;
@ -86,3 +92,17 @@ private:
double MorphOpenConnected(double target_volume_change);
};
class FlowAdaptor{
public:
FlowAdaptor(ScaLBL_ColorModel &M);
~FlowAdaptor();
double MoveInterface(ScaLBL_ColorModel &M);
DoubleArray phi;
DoubleArray phi_t;
private:
int Nx, Ny, Nz;
int timestep;
int timestep_previous;
};
#endif

10
models/DFHModel.cpp
View File

@ -490,14 +490,10 @@ void ScaLBL_DFHModel::Run(){
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps: %i \n", timestepMax);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
auto t1 = std::chrono::system_clock::now();
bool Regular = true;
PROFILE_START("Loop");
runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
@ -589,10 +585,10 @@ void ScaLBL_DFHModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
if (rank==0) printf("********************************************************\n");

949
models/FreeLeeModel.cpp
View File

File diff suppressed because it is too large Load Diff

33
models/FreeLeeModel.h
View File

@ -16,6 +16,9 @@ Implementation of Lee et al JCP 2016 lattice boltzmann model
#include "common/ScaLBL.h"
#include "common/WideHalo.h"
#ifndef ScaLBL_FreeLeeModel_INC
#define ScaLBL_FreeLeeModel_INC
class ScaLBL_FreeLeeModel{
public:
ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM);
@ -26,16 +29,27 @@ public:
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void WriteDebug();
void Create_TwoFluid();
void Initialize_TwoFluid();
double Run_TwoFluid(int returntime);
void WriteDebug_TwoFluid();
void Create_SingleFluid();
void Initialize_SingleFluid();
void Run_SingleFluid();
void WriteDebug_SingleFluid();
// test utilities
void Create_DummyPhase_MGTest();
void MGTest();
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
double tauA,tauB,rhoA,rhoB;
double W,gamma;
double tau, rho0;//only for single-fluid Lee model
double tauM;//relaxation time for phase field (or mass)
double W,gamma,kappa,beta;
double Fx,Fy,Fz,flux;
double din,dout,inletA,inletB,outletA,outletB;
@ -61,12 +75,16 @@ public:
signed char *id;
int *NeighborList;
int *dvcMap;
double *fq, *hq;
double *gqbar, *hq;
double *mu_phi, *Den, *Phi;
double *ColorGrad;
double *Velocity;
double *Pressure;
void getPhase(DoubleArray &PhaseValues);
void getPotential(DoubleArray &PressureValues, DoubleArray &MuValues);
void getVelocity(DoubleArray &Vx, DoubleArray &Vy, DoubleArray &Vz);
DoubleArray SignDist;
private:
@ -81,6 +99,7 @@ private:
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);
void AssignComponentLabels_ChemPotential_ColorGrad();
};
#endif

7
models/GreyscaleColorModel.cpp
View File

@ -910,10 +910,8 @@ void ScaLBL_GreyscaleColorModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -923,6 +921,7 @@ void ScaLBL_GreyscaleColorModel::Run(){
auto current_db = db->cloneDatabase();
//runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
//analysis.createThreads( analysis_method, 4 );
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
@ -1319,10 +1318,10 @@ void ScaLBL_GreyscaleColorModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_Comm->Barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

7
models/GreyscaleModel.cpp
View File

@ -485,10 +485,8 @@ void ScaLBL_GreyscaleModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
Minkowski Morphology(Mask);
@ -500,6 +498,7 @@ void ScaLBL_GreyscaleModel::Run(){
double rlx_eff = 1.0/tau_eff;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
// *************ODD TIMESTEP*************//
@ -744,10 +743,10 @@ void ScaLBL_GreyscaleModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

6
models/IonModel.cpp
View File

@ -784,7 +784,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
//.......create and start timer............
//double starttime,stoptime,cputime;
//ScaLBL_Comm->Barrier(); comm.barrier();
//starttime = MPI_Wtime();
//auto t1 = std::chrono::system_clock::now();
for (int ic=0; ic<number_ion_species; ic++){
timestep=0;
@ -886,10 +886,10 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np);
}
//************************************************************************/
//stoptime = MPI_Wtime();
//if (rank==0) printf("-------------------------------------------------------------------\n");
//// Compute the walltime per timestep
//cputime = (stoptime - starttime)/timestep;
//auto t2 = std::chrono::system_clock::now();
//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
//// Performance obtained from each node
//double MLUPS = double(Np)/cputime/1000000;

12
models/MRTModel.cpp
View File

@ -26,6 +26,8 @@ void ScaLBL_MRTModel::ReadParams(string filename){
tolerance = 1.0e-8;
Fx = Fy = 0.0;
Fz = 1.0e-5;
dout = 1.0;
din = 1.0;
// Color Model parameters
if (mrt_db->keyExists( "timestepMax" )){
@ -194,7 +196,8 @@ void ScaLBL_MRTModel::Create(){
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
comm.barrier();
double MLUPS = ScaLBL_Comm->GetPerformance(NeighborList,fq,Np);
printf(" MLPUS=%f from rank %i\n",MLUPS,rank);
}
void ScaLBL_MRTModel::Initialize(){
@ -227,14 +230,13 @@ void ScaLBL_MRTModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = MPI_Wtime();
if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax);
if (rank==0) printf("********************************************************\n");
timestep=0;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
timestep++;
@ -351,10 +353,10 @@ void ScaLBL_MRTModel::Run(){
}
}
//************************************************************************/
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

12
models/MultiPhysController.cpp
View File

@ -2,7 +2,7 @@
ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),
analysis_interval(0),visualization_interval(0),tolerance(0),comm(COMM)
analysis_interval(0),visualization_interval(0),tolerance(0),time_conv_max(0),comm(COMM)
{
}
@ -25,6 +25,7 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){
analysis_interval = 500;
visualization_interval = 10000;
tolerance = 1.0e-6;
time_conv_max = 0.0;
// load input parameters
if (study_db->keyExists( "timestepMax" )){
@ -135,3 +136,12 @@ vector<int> ScaLBL_Multiphys_Controller::getIonNumIter_PNP_coupling(double Stoke
}
return num_iter_ion;
}
void ScaLBL_Multiphys_Controller::getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv){
//Return maximum of the time converting factor from Stokes and ion solvers
vector<double> TimeConv;
TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end());
TimeConv.insert(TimeConv.begin(),StokesTimeConv);
time_conv_max = *max_element(TimeConv.begin(),TimeConv.end());
}

2
models/MultiPhysController.h
View File

@ -27,6 +27,7 @@ public:
int getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
vector<int> getIonNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
//void getIonNumIter_PNP_coupling(double StokesTimeConv,vector<double> &IonTimeConv,vector<int> &IonTimeMax);
void getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
bool Restart;
int timestepMax;
@ -35,6 +36,7 @@ public:
int analysis_interval;
int visualization_interval;
double tolerance;
double time_conv_max;
//double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity
int rank,nprocs;

195
models/PoissonSolver.cpp
View File

@ -8,8 +8,11 @@
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),
nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM)
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
{
}
@ -33,10 +36,12 @@ void ScaLBL_Poisson::ReadParams(string filename){
epsilonR = 78.4;//default dielectric constant of water
epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity
analysis_interval = 1000;
Vin = 1.0; //Boundary-z (inlet) electric potential
Vout = 1.0; //Boundary-Z (outlet) electric potential
chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3]
WriteLog = false;
TestPeriodic = false;
TestPeriodicTime = 1.0;//unit: [sec]
TestPeriodicTimeConv = 0.01; //unit [sec/lt]
TestPeriodicSaveInterval = 0.1; //unit [sec]
// LB-Poisson Model parameters
if (electric_db->keyExists( "timestepMax" )){
@ -57,6 +62,18 @@ void ScaLBL_Poisson::ReadParams(string filename){
if (electric_db->keyExists( "WriteLog" )){
WriteLog = electric_db->getScalar<bool>( "WriteLog" );
}
if (electric_db->keyExists( "TestPeriodic" )){
TestPeriodic = electric_db->getScalar<bool>( "TestPeriodic" );
}
if (electric_db->keyExists( "TestPeriodicTime" )){
TestPeriodicTime = electric_db->getScalar<double>( "TestPeriodicTime" );
}
if (electric_db->keyExists( "TestPeriodicTimeConv" )){
TestPeriodicTimeConv = electric_db->getScalar<double>( "TestPeriodicTimeConv" );
}
if (electric_db->keyExists( "TestPeriodicSaveInterval" )){
TestPeriodicSaveInterval = electric_db->getScalar<double>( "TestPeriodicSaveInterval" );
}
// Read solid boundary condition specific to Poisson equation
BoundaryConditionSolid = 1;
@ -65,10 +82,15 @@ void ScaLBL_Poisson::ReadParams(string filename){
}
// Read boundary condition for electric potential
// BC = 0: normal periodic BC
// BC = 1: fixed inlet and outlet potential
BoundaryCondition = 0;
if (electric_db->keyExists( "BC" )){
BoundaryCondition = electric_db->getScalar<int>( "BC" );
// BC = 1: fixed electric potential
// BC = 2: sine/cosine periodic electric potential (need extra input parameters)
BoundaryConditionInlet = 0;
BoundaryConditionOutlet = 0;
if (electric_db->keyExists( "BC_Inlet" )){
BoundaryConditionInlet = electric_db->getScalar<int>( "BC_Inlet" );
}
if (electric_db->keyExists( "BC_Outlet" )){
BoundaryConditionOutlet = electric_db->getScalar<int>( "BC_Outlet" );
}
// Read domain parameters
@ -117,8 +139,17 @@ void ScaLBL_Poisson::SetDomain(){
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
comm.barrier();
Dm->BoundaryCondition = BoundaryCondition;
Mask->BoundaryCondition = BoundaryCondition;
if (BoundaryConditionInlet==0 && BoundaryConditionOutlet==0){
Dm->BoundaryCondition = 0;
Mask->BoundaryCondition = 0;
}
else if (BoundaryConditionInlet>0 && BoundaryConditionOutlet>0){
Dm->BoundaryCondition = 1;
Mask->BoundaryCondition = 1;
}
else {//i.e. non-periodic and periodic BCs are mixed
ERROR("Error: check the type of inlet and outlet boundary condition! Mixed periodic and non-periodic BCs are found!\n");
}
Dm->CommInit();
comm.barrier();
@ -343,15 +374,91 @@ void ScaLBL_Poisson::Create(){
void ScaLBL_Poisson::Potential_Init(double *psi_init){
if (BoundaryCondition==1){
//set up default boundary input parameters
Vin0 = Vout0 = 1.0; //unit: [V]
freqIn = freqOut = 50.0; //unit: [Hz]
t0_In = t0_Out = 0.0; //unit: [sec]
Vin_Type = Vout_Type = 1; //1->sin; 2->cos
Vin = 1.0; //Boundary-z (inlet) electric potential
Vout = 1.0; //Boundary-Z (outlet) electric potential
if (BoundaryConditionInlet>0){
switch (BoundaryConditionInlet){
case 1:
if (electric_db->keyExists( "Vin" )){
Vin = electric_db->getScalar<double>( "Vin" );
}
if (rank==0) printf("LB-Poisson Solver: inlet boundary; fixed electric potential Vin = %.3g [V]\n",Vin);
break;
case 2:
if (electric_db->keyExists( "Vin0" )){//voltage amplitude; unit: Volt
Vin0 = electric_db->getScalar<double>( "Vin0" );
}
if (electric_db->keyExists( "freqIn" )){//unit: Hz
freqIn = electric_db->getScalar<double>( "freqIn" );
}
if (electric_db->keyExists( "t0_In" )){//timestep shift, unit: lt
t0_In = electric_db->getScalar<double>( "t0_In" );
}
if (electric_db->keyExists( "Vin_Type" )){
//type=1 -> sine
//tyep=2 -> cosine
Vin_Type = electric_db->getScalar<int>( "Vin_Type" );
if (Vin_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vin_Type is currently not supported! \n");
}
if (rank==0){
if (Vin_Type==1){
printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vin0,freqIn,t0_In);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
}
else if (Vin_Type==2){
printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V] \n",Vin0,freqIn,t0_In);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
}
}
break;
}
}
if (BoundaryConditionOutlet>0){
switch (BoundaryConditionOutlet){
case 1:
if (electric_db->keyExists( "Vout" )){
Vout = electric_db->getScalar<double>( "Vout" );
}
if (rank==0) printf("LB-Poisson Solver: outlet boundary; fixed electric potential Vout = %.3g [V] \n",Vout);
break;
case 2:
if (electric_db->keyExists( "Vout0" )){//voltage amplitude; unit: Volt
Vout0 = electric_db->getScalar<double>( "Vout0" );
}
if (electric_db->keyExists( "freqOut" )){//unit: Hz
freqOut = electric_db->getScalar<double>( "freqOut" );
}
if (electric_db->keyExists( "t0_Out" )){//timestep shift, unit: lt
t0_Out = electric_db->getScalar<double>( "t0_Out" );
}
if (electric_db->keyExists( "Vout_Type" )){
//type=1 -> sine
//tyep=2 -> cosine
Vout_Type = electric_db->getScalar<int>( "Vout_Type" );
if (Vout_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vout_Type is currently not supported! \n");
}
if (rank==0){
if (Vout_Type==1){
printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
}
else if (Vout_Type==2){
printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
}
}
break;
}
}
//By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis
if (BoundaryConditionInlet==2) Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,0);
if (BoundaryConditionOutlet==2) Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,0);
double slope = (Vout-Vin)/(Nz-2);
double psi_linearized;
for (int k=0;k<Nz;k++){
@ -375,10 +482,15 @@ void ScaLBL_Poisson::Potential_Init(double *psi_init){
}
}
double ScaLBL_Poisson::getBoundaryVoltagefromPeriodicBC(double V0, double freq, double t0, int V_type, int time_step){
return V0*(V_type==1)*sin(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv))+V0*(V_type==2)*cos(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv));
}
void ScaLBL_Poisson::Initialize(){
void ScaLBL_Poisson::Initialize(double time_conv_from_Study){
/*
* This function initializes model
* "time_conv_from_Study" is the phys to LB time conversion factor, unit=[sec/lt]
* which is used for periodic voltage input for inlet and outlet boundaries
*/
if (rank==0) printf ("LB-Poisson Solver: initializing D3Q7 distributions\n");
//NOTE the initialization involves two steps:
@ -386,6 +498,7 @@ void ScaLBL_Poisson::Initialize(){
//2. Initialize electric potential for pore nodes
double *psi_host;
psi_host = new double [Nx*Ny*Nz];
time_conv = time_conv_from_Study;
AssignSolidBoundary(psi_host);//step1
Potential_Init(psi_host);//step2
ScaLBL_CopyToDevice(Psi, psi_host, Nx*Ny*Nz*sizeof(double));
@ -405,12 +518,12 @@ void ScaLBL_Poisson::Initialize(){
//}
}
void ScaLBL_Poisson::Run(double *ChargeDensity){
void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
//.......create and start timer............
//double starttime,stoptime,cputime;
//ScaLBL_Comm->Barrier(); comm.barrier();
//starttime = MPI_Wtime();
//comm.barrier();
//auto t1 = std::chrono::system_clock::now();
timestep=0;
double error = 1.0;
@ -420,13 +533,13 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){
// *************ODD TIMESTEP*************//
timestep++;
SolveElectricPotentialAAodd();//update electric potential
SolveElectricPotentialAAodd(timestep_from_Study);//update electric potential
SolvePoissonAAodd(ChargeDensity);//perform collision
ScaLBL_Comm->Barrier(); comm.barrier();
// *************EVEN TIMESTEP*************//
timestep++;
SolveElectricPotentialAAeven();//update electric potential
SolveElectricPotentialAAeven(timestep_from_Study);//update electric potential
SolvePoissonAAeven(ChargeDensity);//perform collision
ScaLBL_Comm->Barrier(); comm.barrier();
//************************************************************************/
@ -466,11 +579,11 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){
}
//************************************************************************/
//stoptime = MPI_Wtime();
////if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n");
////if (rank==0) printf("---------------------------------------------------------------------------\n");
//// Compute the walltime per timestep
//cputime = (stoptime - starttime)/timestep;
//auto t2 = std::chrono::system_clock::now();
//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
//// Performance obtained from each node
//double MLUPS = double(Np)/cputime/1000000;
@ -506,29 +619,65 @@ void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
}
}
void ScaLBL_Poisson::SolveElectricPotentialAAodd(){
void ScaLBL_Poisson::SolveElectricPotentialAAodd(int timestep_from_Study){
ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 1){
if (BoundaryConditionInlet > 0){
switch (BoundaryConditionInlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
case 2:
Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
}
}
if (BoundaryConditionOutlet > 0){
switch (BoundaryConditionOutlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
case 2:
Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
}
}
//-------------------------//
ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);
}
void ScaLBL_Poisson::SolveElectricPotentialAAeven(){
void ScaLBL_Poisson::SolveElectricPotentialAAeven(int timestep_from_Study){
ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL
ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 1){
if (BoundaryConditionInlet > 0){
switch (BoundaryConditionInlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
case 2:
Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
}
}
if (BoundaryConditionOutlet > 0){
switch (BoundaryConditionOutlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
case 2:
Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
}
}
//-------------------------//
ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);

21
models/PoissonSolver.h
View File

@ -9,6 +9,7 @@
#include <exception>
#include <stdexcept>
#include <fstream>
#include <cmath>
#include "common/ScaLBL.h"
#include "common/Communication.h"
@ -16,6 +17,7 @@
#include "analysis/Minkowski.h"
#include "ProfilerApp.h"
#define _USE_MATH_DEFINES
#ifndef ScaLBL_POISSON_INC
#define ScaLBL_POISSON_INC
@ -30,8 +32,8 @@ public:
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run(double *ChargeDensity);
void Initialize(double time_conv_from_Study);
void Run(double *ChargeDensity,int timestep_from_Study);
void getElectricPotential(DoubleArray &ReturnValues);
void getElectricPotential_debug(int timestep);
void getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z);
@ -41,7 +43,8 @@ public:
//bool Restart,pBC;
int timestep,timestepMax;
int analysis_interval;
int BoundaryCondition;
int BoundaryConditionInlet;
int BoundaryConditionOutlet;
int BoundaryConditionSolid;
double tau;
double tolerance;
@ -50,11 +53,18 @@ public:
double Vin, Vout;
double chargeDen_dummy;//for debugging
bool WriteLog;
double Vin0,freqIn,t0_In,Vin_Type;
double Vout0,freqOut,t0_Out,Vout_Type;
bool TestPeriodic;
double TestPeriodicTime;//unit: [sec]
double TestPeriodicTimeConv; //unit [sec/lt]
double TestPeriodicSaveInterval; //unit [sec]
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
double h;//image resolution
double time_conv;//phys to LB time converting factor; unit=[sec/lt]
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
@ -91,12 +101,13 @@ private:
void AssignSolidBoundary(double *poisson_solid);
void Potential_Init(double *psi_init);
void ElectricField_LB_to_Phys(DoubleArray &Efield_reg);
void SolveElectricPotentialAAodd();
void SolveElectricPotentialAAeven();
void SolveElectricPotentialAAodd(int timestep_from_Study);
void SolveElectricPotentialAAeven(int timestep_from_Study);
//void SolveElectricField();
void SolvePoissonAAodd(double *ChargeDensity);
void SolvePoissonAAeven(double *ChargeDensity);
void getConvergenceLog(int timestep,double error);
double getBoundaryVoltagefromPeriodicBC(double V0,double freq,double t0,int V_type,int time_step);
};
#endif

8
models/StokesModel.cpp
View File

@ -573,16 +573,14 @@ void ScaLBL_StokesModel::Run(){
}
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier(); comm.barrier();
starttime = MPI_Wtime();
if (rank==0) printf("****************************************************************\n");
if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: timestepMax = %i\n", timestepMax);
if (rank==0) printf("****************************************************************\n");
timestep=0;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
timestep++;
@ -700,10 +698,10 @@ void ScaLBL_StokesModel::Run(){
}
}
//************************************************************************/
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

1
sample_scripts/config_build_eos
View File

@ -33,7 +33,6 @@ cmake \
-D CMAKE_CXX_STANDARD=14 \
-D USE_TIMER=false \
-D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_CUDA=0 \

1
sample_scripts/config_build_rhea
View File

@ -27,7 +27,6 @@ cmake \
-D CMAKE_CXX_STD=11 \
-D USE_TIMER=false \
-D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_CUDA=0 \

1
sample_scripts/config_build_titan
View File

@ -25,7 +25,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_SILO=1 \

1
sample_scripts/config_build_titan_silo
View File

@ -20,7 +20,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_NETCDF=0 \

1
sample_scripts/config_poplar_hip
View File

@ -36,6 +36,5 @@ cmake \
# MPI_THREAD_MULTIPLE=1 MV2_USE_RDMA_CM=0 MV2_USE_RDMA_CM= MV2_NUM_HCAS=1 MV2_USE_CUDA=1 MV2_ENABLE_AFFINITY=0 srun -n 2 -N 1 --cpu-bind=v -c 1 ./test_MPI
# -D MPI_COMPILER:BOOL=TRUE \
# -D MPIEXEC=mpirun \
# -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \

1
sample_scripts/config_summit_hip
View File

@ -21,7 +21,6 @@ cmake \
-D HIP_NVCC_OPTIONS="-arch sm_70" \
-D LINK_LIBRARIES="/sw/summit/cuda/9.2.148/lib64/libcudart.so" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D USE_NETCDF=0 \
-D USE_SILO=1 \
-D SILO_DIRECTORY=${TPL_DIR}/silo \

1
sample_scripts/config_titan
View File

@ -9,7 +9,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/usr/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
${HOME}/LBPM-WIA

1
sample_scripts/configure_arc_cluster
View File

@ -10,7 +10,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_arden
View File

@ -5,7 +5,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D CMAKE_C_FLAGS="-O3 -fPIC" \
-D CMAKE_CXX_FLAGS="-O3 -fPIC " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_basic_cluster
View File

@ -8,7 +8,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_blueridge
View File

@ -8,7 +8,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_cascades_cpu
View File

@ -11,7 +11,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="-fPIC" \
-D CMAKE_CXX_FLAGS="-fPIC" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_desktop
View File

@ -7,7 +7,6 @@ cmake \
-D CMAKE_C_FLAGS="-g " \
-D CMAKE_CXX_FLAGS="-g " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

3
sample_scripts/configure_huckleberry
View File

@ -12,7 +12,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="-fPIC" \
-D CMAKE_CXX_FLAGS="-fPIC" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \
@ -26,5 +25,5 @@ cmake \
-D USE_TIMER=0 \
~/LBPM-WIA
make VERBOSE=1 -j8 && make install
make VERBOSE=1 -j4 && make install

1
sample_scripts/configure_titan_jem
View File

@ -15,7 +15,6 @@ cmake \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

4
sample_scripts/configure_ubuntu
View File

@ -5,9 +5,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D CMAKE_C_FLAGS="-O3 -fPIC" \
-D CMAKE_CXX_FLAGS="-O3 -fPIC " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \
-D CUDA_FLAGS="-arch sm_35" \
-D CUDA_HOST_COMPILER="/usr/bin/gcc" \
@ -16,7 +14,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D USE_SILO=1 \
-D SILO_LIB="/opt/arden/silo/4.10.2/lib/libsiloh5.a" \
-D SILO_DIRECTORY="/opt/arden/silo/4.10.2" \
-D USE_NETCDF=1 \
-D USE_NETCDF=0 \
-D NETCDF_DIRECTORY="/opt/arden/netcdf/4.6.1" \
-D USE_CUDA=0 \
-D USE_TIMER=0 \

1
sample_scripts/daedalus_config
View File

@ -12,7 +12,6 @@ i -D CMAKE_CXX_COMPILER:PATH=/home/christopher/openmpi/install_dir/bin/mpicxx
-D USE_DOXYGEN=false \
# -D CMAKE_C_FLAGS="-std=gnu++11 -w" \
# -D CMAKE_CXX_FLAGS="-std=gnu++11 -w" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=/home/christopher/openmpi/install_dir/bin/mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/promethius_config
View File

@ -7,7 +7,6 @@ cmake \
-D CMAKE_C_FLAGS="-g " \
-D CMAKE_CXX_FLAGS="-g -Wno-deprecated-declarations" \
-D CXX_STD=11 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

10
tests/CMakeLists.txt
View File

@ -4,8 +4,10 @@
ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator )
ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator )
ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator )
ADD_LBPM_EXECUTABLE( lbpm_freelee_simulator )
ADD_LBPM_EXECUTABLE( lbpm_freelee_SingleFluidBGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )
@ -33,14 +35,15 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest )
#ADD_LBPM_EXECUTABLE( BlobAnalysis )
#ADD_LBPM_EXECUTABLE( BlobIdentify )
#ADD_LBPM_EXECUTABLE( BlobIdentifyParallel )
#ADD_LBPM_EXECUTABLE( convertIO )
#ADD_LBPM_EXECUTABLE( DataAggregator )
ADD_LBPM_EXECUTABLE( convertIO )
ADD_LBPM_EXECUTABLE( DataAggregator )
#ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )(
ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar )
ADD_LBPM_EXECUTABLE( TestPoissonSolver )
ADD_LBPM_EXECUTABLE( TestIonModel )
ADD_LBPM_EXECUTABLE( TestNernstPlanck )
ADD_LBPM_EXECUTABLE( TestPNP_Stokes )
ADD_LBPM_EXECUTABLE( TestMixedGrad )
@ -59,6 +62,7 @@ ADD_LBPM_TEST( TestMap )
ADD_LBPM_TEST( TestWideHalo )
ADD_LBPM_TEST( TestColorGradDFH )
ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db)
ADD_LBPM_TEST( testGlobalMassFreeLee ../example/Bubble/input.db)
#ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db)
ADD_LBPM_TEST( TestPressVel ../example/Bubble/input.db)
ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db)

199
tests/TestMixedGrad.cpp Normal file
View File

@ -0,0 +1,199 @@
#include <exception>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include "common/Utilities.h"
#include "models/FreeLeeModel.h"
inline void Initialize_Mask(ScaLBL_FreeLeeModel &LeeModel){
// initialize a bubble
int i,j,k,n;
int rank = LeeModel.Mask->rank();
int Nx = LeeModel.Mask->Nx;
int Ny = LeeModel.Mask->Ny;
int Nz = LeeModel.Mask->Nz;
if (rank == 0) printf(" initialize mask...\n");
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
LeeModel.Mask->id[n]=1;
LeeModel.id[n] = LeeModel.Mask->id[n];
}
}
}
}
inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel, double ax, double ay, double az){
// initialize a bubble
int i,j,k,n;
int rank = LeeModel.Mask->rank();
int Nx = LeeModel.Mask->Nx;
int Ny = LeeModel.Mask->Ny;
int Nz = LeeModel.Mask->Nz;
if (rank == 0) printf("Setting up dummy phase field with gradient {x,y,z} = {%f , %f , %f}...\n",ax,ay,az);
double * Dummy;
int Nh = (Nx+2)*(Ny+2)*(Nz+2);
Dummy = new double [(Nx+2)*(Ny+2)*(Nz+2)];
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
LeeModel.Mask->id[n]=1;
LeeModel.id[n] = LeeModel.Mask->id[n];
int nh = (k+1)*(Nx+2)*(Ny+2) + (j+1)*(Nx+2) + i+1;
Dummy[nh] = ax*double(i) + ay*double(j) + az*double(k);
}
}
}
ScaLBL_CopyToDevice(LeeModel.Phi, Dummy, sizeof(double)*Nh);
LeeModel.MGTest();
}
inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){
int i,j,k,iq,stride,nread;
int Nxh = LeeModel.Nxh;
int Nyh = LeeModel.Nyh;
int Np = LeeModel.Np;
int *TmpMap;
TmpMap = new int[Np];
ScaLBL_CopyToHost(TmpMap, LeeModel.dvcMap, Np*sizeof(int));
int *neighborList;
neighborList = new int[18*Np];
ScaLBL_CopyToHost(neighborList, LeeModel.NeighborList, 18*Np*sizeof(int));
printf("Check stride for interior neighbors \n");
int count = 0;
for (int n=LeeModel.ScaLBL_Comm->FirstInterior(); n<LeeModel.ScaLBL_Comm->LastInterior(); n++){
// q=0
int idx = TmpMap[n];
k = idx/Nxh/Nyh;
j = (idx-k*Nxh*Nyh)/Nxh;
i = (idx-k*Nxh*Nyh -j*Nxh);
// q=1
nread = neighborList[n];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != 1){
printf(" %i, %i, %i q = 1 stride=%i \n ",i,j,k,stride);
count++;
}
// q=2
nread = neighborList[n+Np];
iq = TmpMap[nread%Np];
stride = iq - idx;
if (stride != 1){
printf(" %i, %i, %i q = 2 stride=%i \n ",i,j,k,stride);
count++;
}
// q=3
nread = neighborList[n+2*Np];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != Nxh){
printf(" %i, %i, %i q = 3 stride=%i \n ",i,j,k,stride);
count++;
}
// q = 4
nread = neighborList[n+3*Np];
iq = TmpMap[nread%Np];
stride = iq-idx;
if (stride != Nxh){
printf(" %i, %i, %i q = 4 stride=%i \n ",i,j,k,stride);
count++;
}
// q=5
nread = neighborList[n+4*Np];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != Nxh*Nyh){
count++;
printf(" %i, %i, %i q = 5 stride=%i \n ",i,j,k,stride);
}
// q = 6
nread = neighborList[n+5*Np];
iq = TmpMap[nread%Np];
stride = iq - idx;
if (stride != Nxh*Nyh){
count++;
printf(" %i, %i, %i q = 6 stride=%i \n ",i,j,k,stride);
}
}
return count;
}
int main( int argc, char **argv )
{
// Initialize
Utilities::startup( argc, argv );
int errors = 0;
// Load the input database
auto db = std::make_shared<Database>( argv[1] );
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
if ( rank == 0 ) {
printf( "********************************************************\n" );
printf( "Running Mixed Gradient Test \n" );
printf( "********************************************************\n" );
}
// Initialize compute device
int device = ScaLBL_SetDevice( rank );
NULL_USE( device );
ScaLBL_DeviceBarrier();
comm.barrier();
PROFILE_ENABLE( 1 );
// PROFILE_ENABLE_TRACE();
// PROFILE_ENABLE_MEMORY();
PROFILE_SYNCHRONIZE();
PROFILE_START( "Main" );
Utilities::setErrorHandlers();
auto filename = argv[1];
ScaLBL_FreeLeeModel LeeModel( rank, nprocs, comm );
LeeModel.ReadParams( filename );
LeeModel.SetDomain();
Initialize_Mask(LeeModel);
//LeeModel.Create_DummyPhase_MGTest();
LeeModel.Create_TwoFluid();
errors=MultiHaloNeighborCheck(LeeModel);
Initialize_DummyPhaseField(LeeModel,1.0, 2.0, 3.0);
LeeModel.WriteDebug_TwoFluid();
PROFILE_STOP( "Main" );
PROFILE_SAVE( file, level );
// ****************************************************
} // Limit scope so variables that contain communicators will free before MPI_Finialize
Utilities::shutdown();
return errors;
}

4
tests/TestNernstPlanck.cpp
View File

@ -66,7 +66,7 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
PoissonSolver.Initialize(0);
int timestep=0;
double error = 1.0;
@ -74,7 +74,7 @@ int main(int argc, char **argv)
while (timestep < Study.timestepMax && error > Study.tolerance){
timestep++;
PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental
PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
IonModel.Run(IonModel.FluidVelocityDummy,PoissonSolver.ElectricField); //solve for ion transport and electric potential
timestep++;//AA operations

9
tests/TestNetcdf.cpp
View File

@ -1,7 +1,7 @@
// Test reading/writing netcdf files
#include "IO/netcdf.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/UnitTest.h"
@ -13,7 +13,8 @@ void load( const std::string& );
void test_NETCDF( UnitTest& ut )
{
const int rank = comm_rank( MPI_COMM_WORLD );
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocx = 2;
int nprocy = 2;
int nprocz = 2;
@ -26,11 +27,11 @@ void test_NETCDF( UnitTest& ut )
size_t z = info.kz*data.size(2);
const char* filename = "test.nc";
std::vector<int> dim = { (int) data.size(0)*nprocx, (int) data.size(1)*nprocy, (int) data.size(2)*nprocz };
int fid = netcdf::open( filename, netcdf::CREATE, MPI_COMM_WORLD );
int fid = netcdf::open( filename, netcdf::CREATE, comm );
auto dims = netcdf::defDim( fid, {"X", "Y", "Z"}, dim );
netcdf::write( fid, "tmp", dims, data, info );
netcdf::close( fid );
MPI_Barrier( MPI_COMM_WORLD );
comm.barrier();
// Read the contents of the file we created
fid = netcdf::open( filename, netcdf::READ );
Array<float> tmp = netcdf::getVar<float>( fid, "tmp" );

4
tests/TestPNP_Stokes.cpp
View File

@ -82,7 +82,7 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
PoissonSolver.Initialize(0);
int timestep=0;
@ -94,7 +94,7 @@ int main(int argc, char **argv)
while (timestep < Study.timestepMax && error > Study.tolerance){
timestep++;
PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental
PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity
IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential

27
tests/TestPoissonSolver.cpp
View File

@ -51,14 +51,37 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
if (PoissonSolver.TestPeriodic==true){
PoissonSolver.Initialize(PoissonSolver.TestPeriodicTimeConv);
}
else {
PoissonSolver.Initialize(0);
}
//Initialize dummy charge density for test
PoissonSolver.DummyChargeDensity();
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy);
if (PoissonSolver.TestPeriodic==true){
if (rank==0) printf("Testing periodic voltage input is enabled. Total test time is %.3g[s], saving data every %.3g[s]; user-specified time resolution is %.3g[s/lt]\n",
PoissonSolver.TestPeriodicTime,PoissonSolver.TestPeriodicSaveInterval,PoissonSolver.TestPeriodicTimeConv);
int timestep = 0;
int timeMax = int(PoissonSolver.TestPeriodicTime/PoissonSolver.TestPeriodicTimeConv);
int timeSave = int(PoissonSolver.TestPeriodicSaveInterval/PoissonSolver.TestPeriodicTimeConv);
while (timestep<timeMax){
timestep++;
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,timestep);
if (timestep%timeSave==0){
if (rank==0) printf(" Time = %.3g[s]; saving electric potential and field\n",timestep*PoissonSolver.TestPeriodicTimeConv);
PoissonSolver.getElectricPotential_debug(timestep);
PoissonSolver.getElectricField_debug(timestep);
}
}
}
else {
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,1);
PoissonSolver.getElectricPotential_debug(1);
PoissonSolver.getElectricField_debug(1);
}
if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
if (rank==0) printf("*************************************************************\n");

8
tests/TestWideHalo.cpp
View File

@ -159,6 +159,13 @@ int main(int argc, char **argv)
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
// initialize phi based on PhaseLabel (include solid component labels)
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
PhaseLabel[n] = 0.2;
}
}
}
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
//...........................................................................
@ -172,7 +179,6 @@ int main(int argc, char **argv)
int SIZE=3*Np*sizeof(double);
ScaLBL_CopyToHost(&COLORGRAD[0],&ColorGrad[0],SIZE);
double CX,CY,CZ;
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){

Some files were not shown because too many files have changed in this diff Show More