OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'FOM' of github.com:JamesEMcClure/LBPM-WIA into FOM

Browse Source
This commit is contained in:
James E McClure 2021-03-25 17:31:46 -04:00
commit 1068e19b37
111 changed files with 21174 additions and 6765 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
.clang-format Normal file
View File

@ -0,0 +1,108 @@
# To run clang tools:
# cd to root directory
# To update format only:
# find . -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
# git status -s . | sed s/^...// | grep -E "(\.cpp|\.h|\.cc|\.hpp|\.I)" | xargs -I{} clang-format -i {}
# To run modernize
# export CLANG_PATH=/packages/llvm/build/llvm-60
# export PATH=${CLANG_PATH}/bin:${CLANG_PATH}/share/clang:$PATH
# find src -name "*.cpp" -or -name "*.cc" | xargs -I{} clang-tidy -checks=modernize* -p=/projects/AtomicModel/build/debug -fix {}
# find src -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
---
Language: Cpp
# BasedOnStyle: LLVM
AccessModifierOffset: -4
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: true
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: true
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: true
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
BreakBeforeBinaryOperators: None
#BreakBeforeBraces: Stroustrup
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
ColumnLimit: 100
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: false
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IndentCaseLabels: false
IndentWidth: 4
IndentWrappedFunctionNames: false
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBlockIndentWidth: 4
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: true
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: true
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: true
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 4
UseTab: Never
...

2
CMakeLists.txt
View File

@ -150,7 +150,7 @@ IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" FALSE )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )

4
IO/IOHelpers.h
View File

@ -57,8 +57,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
}
};
}; // namespace IO
#endif

259
IO/Mesh.cpp
View File

@ -1,4 +1,5 @@
#include "Mesh.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#include <limits>
@ -21,35 +22,40 @@ inline Point nullPoint()
/****************************************************
* Mesh *
****************************************************/
Mesh::Mesh( )
{
}
Mesh::~Mesh( )
{
}
Mesh::Mesh() {}
Mesh::~Mesh() {}
/****************************************************
* MeshDataStruct *
****************************************************/
bool MeshDataStruct::check() const
#define checkResult( pass, msg ) \
do { \
if ( !( pass ) ) { \
if ( abort ) \
ERROR( msg ); \
return false; \
} \
} while ( 0 )
bool MeshDataStruct::check( bool abort ) const
{
enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
bool pass = mesh != nullptr;
for ( const auto &var : vars ) {
pass = pass && static_cast<int>(var->type)>=1 && static_cast<int>(var->type)<=3;
pass = pass && !var->data.empty();
checkResult( var->type == VariableType::NodeVariable ||
var->type == VariableType::EdgeVariable ||
var->type == VariableType::SurfaceVariable ||
var->type == VariableType::VolumeVariable,
"Invalid data type" );
checkResult( !var->data.empty(), "Variable data is empty" );
}
if ( !pass )
return false;
const std::string &meshClass = mesh->className();
if ( meshClass == "PointList" ) {
const auto mesh2 = dynamic_cast<IO::PointList*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
auto mesh2 = dynamic_cast<IO::PointList *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR( "Invalid type for PointList" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
@ -61,62 +67,63 @@ bool MeshDataStruct::check() const
}
}
} else if ( meshClass == "TriMesh" || meshClass == "TriList" ) {
const auto mesh2 = getTriMesh( mesh );
if ( mesh2 == nullptr )
return false;
auto mesh2 = getTriMesh( mesh );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->vertices->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim;
checkResult( var->data.size( 0 ) == mesh2->A.size(), "sizeof VolumeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof VolumeVariable" );
} else {
ERROR( "Invalid variable type" );
}
}
} else if ( meshClass == "DomainMesh" ) {
const auto mesh2 = dynamic_cast<IO::DomainMesh*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
auto mesh2 = dynamic_cast<IO::DomainMesh *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
ArraySize varSize;
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1)
&& (int) var->data.size(2)==(mesh2->nz+1) && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx + 1, mesh2->ny + 1, mesh2->nz + 1, var->dim );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny
&& (int) var->data.size(2)==mesh2->nz && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx, mesh2->ny, mesh2->nz, var->dim );
} else {
ERROR( "Invalid variable type" );
}
if ( var->data.size( 0 ) == varSize[0] * varSize[1] * varSize[2] &&
var->data.size( 1 ) == varSize[3] )
var->data.resize( varSize );
for ( int d = 0; d < 4; d++ )
checkResult( var->data.size( d ) == varSize[d], "DomainMesh Variable" );
}
} else {
ERROR( "Unknown mesh class: " + mesh->className() );
}
return pass;
return true;
}
/****************************************************
* PointList *
****************************************************/
PointList::PointList( )
{
}
PointList::PointList() {}
PointList::PointList( size_t N )
{
Point tmp = nullPoint();
points.resize( N, tmp );
}
PointList::~PointList( )
{
}
PointList::~PointList() {}
size_t PointList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
@ -165,9 +172,7 @@ void PointList::unpack( const std::pair<size_t,void*>& data_in )
/****************************************************
* TriList *
****************************************************/
TriList::TriList( )
{
}
TriList::TriList() {}
TriList::TriList( size_t N_tri )
{
Point tmp = nullPoint();
@ -190,9 +195,7 @@ TriList::TriList( const TriMesh& mesh )
for ( size_t i = 0; i < C.size(); i++ )
C[i] = P[mesh.C[i]];
}
TriList::~TriList( )
{
}
TriList::~TriList() {}
size_t TriList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
@ -257,9 +260,7 @@ void TriList::unpack( const std::pair<size_t,void*>& data_in )
/****************************************************
* TriMesh *
****************************************************/
TriMesh::TriMesh( )
{
}
TriMesh::TriMesh() {}
TriMesh::TriMesh( size_t N_tri, size_t N_point )
{
vertices.reset( new PointList( N_point ) );
@ -312,8 +313,10 @@ std::pair<size_t,void*> TriMesh::pack( int level ) const
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
const std::vector<Point> &points = vertices->points;
data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int);
double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)];
data_out.first =
( 3 + 3 * points.size() ) * sizeof( double ) + 3 * A.size() * sizeof( int );
double *data_ptr =
new double[4 + 3 * points.size() + ( 3 * A.size() * sizeof( int ) ) / sizeof( double )];
data_out.second = data_ptr;
uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_ptr );
data_int64[0] = level;
@ -368,22 +371,34 @@ void TriMesh::unpack( const std::pair<size_t,void*>& data_in )
/****************************************************
* Domain mesh *
****************************************************/
DomainMesh::DomainMesh():
nprocx(0), nprocy(0), nprocz(0), rank(0),
nx(0), ny(0), nz(0),
Lx(0), Ly(0), Lz(0)
DomainMesh::DomainMesh()
: nprocx( 0 ),
nprocy( 0 ),
nprocz( 0 ),
rank( 0 ),
nx( 0 ),
ny( 0 ),
nz( 0 ),
Lx( 0 ),
Ly( 0 ),
Lz( 0 )
{
}
DomainMesh::DomainMesh( RankInfoStruct data,
int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ):
nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]),
nx(nx2), ny(ny2), nz(nz2),
Lx(Lx2), Ly(Ly2), Lz(Lz2)
{
}
DomainMesh::~DomainMesh()
DomainMesh::DomainMesh(
RankInfoStruct data, int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 )
: nprocx( data.nx ),
nprocy( data.ny ),
nprocz( data.nz ),
rank( data.rank[1][1][1] ),
nx( nx2 ),
ny( ny2 ),
nz( nz2 ),
Lx( Lx2 ),
Ly( Ly2 ),
Lz( Lz2 )
{
}
DomainMesh::~DomainMesh() {}
size_t DomainMesh::numberPointsVar( VariableType type ) const
{
size_t N = 0;
@ -477,5 +492,129 @@ std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh )
}
} // IO namespace
/****************************************************
* Convert enum values *
****************************************************/
std::string getString( VariableType type )
{
if ( type == VariableType::NodeVariable )
return "node";
else if ( type == VariableType::EdgeVariable )
return "edge";
else if ( type == VariableType::SurfaceVariable )
return "face";
else if ( type == VariableType::VolumeVariable )
return "cell";
else if ( type == VariableType::NullVariable )
return "null";
else
ERROR( "Invalid type" );
return "";
}
VariableType getVariableType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "node" )
return VariableType::NodeVariable;
else if ( type == "edge" || type == "1" )
return VariableType::EdgeVariable;
else if ( type == "face" )
return VariableType::SurfaceVariable;
else if ( type == "cell" || type == "3" )
return VariableType::VolumeVariable;
else if ( type == "null" )
return VariableType::NullVariable;
else
ERROR( "Invalid type: " + type );
return VariableType::NullVariable;
}
std::string getString( DataType type )
{
if ( type == DataType::Double )
return "double";
else if ( type == DataType::Float )
return "float";
else if ( type == DataType::Int )
return "int";
else if ( type == DataType::Null )
return "null";
else
ERROR( "Invalid type" );
return "";
}
DataType getDataType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "double" )
return DataType::Double;
else if ( type == "float" )
return DataType::Float;
else if ( type == "int" )
return DataType::Int;
else if ( type == "null" )
return DataType::Null;
else
ERROR( "Invalid type: " + type );
return DataType::Null;
}
std::string getString( MeshType type )
{
if ( type == MeshType::PointMesh )
return "PointMesh";
else if ( type == MeshType::SurfaceMesh )
return "SurfaceMesh";
else if ( type == MeshType::VolumeMesh )
return "VolumeMesh";
else if ( type == MeshType::Unknown )
return "unknown";
else
ERROR( "Invalid type" );
return "";
}
MeshType getMeshType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "PointMesh" || type == "1" )
return MeshType::PointMesh;
else if ( type == "SurfaceMesh" || type == "2" )
return MeshType::SurfaceMesh;
else if ( type == "VolumeMesh" || type == "3" )
return MeshType::VolumeMesh;
else if ( type == "unknown" || type == "-1" )
return MeshType::Unknown;
else
ERROR( "Invalid type: " + type );
return MeshType::Unknown;
}
std::string getString( FileFormat type )
{
if ( type == FileFormat::OLD )
return "old";
else if ( type == FileFormat::NEW )
return "new";
else if ( type == FileFormat::NEW_SINGLE )
return "new(single)";
else if ( type == FileFormat::SILO )
return "silo";
else
ERROR( "Invalid type" );
return "";
}
FileFormat getFileFormat( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "old" || type == "1" )
return FileFormat::OLD;
else if ( type == "new" || type == "2" )
return FileFormat::NEW;
else if ( type == "new(single)" || type == "3" )
return FileFormat::NEW_SINGLE;
else if ( type == "silo" || type == "4" )
return FileFormat::SILO;
else
ERROR( "Invalid type: " + type );
return FileFormat::SILO;
}
} // namespace IO

59
IO/Mesh.h
View File

@ -6,17 +6,36 @@
#include <string.h>
#include <vector>
#include "analysis/PointList.h"
#include "common/Array.h"
#include "common/Communication.h"
#include "analysis/PointList.h"
namespace IO {
//! Possible variable types
enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 };
//! Enums to define types
enum class VariableType {
NodeVariable,
EdgeVariable,
SurfaceVariable,
VolumeVariable,
NullVariable
};
enum class DataType { Double, Float, Int, Null };
enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown };
enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO };
//! Convert enums to/from strings (more future-proof than static_cast<int>)
std::string getString( VariableType );
std::string getString( DataType );
std::string getString( MeshType );
std::string getString( FileFormat );
VariableType getVariableType( const std::string & );
DataType getDataType( const std::string & );
MeshType getMeshType( const std::string & );
FileFormat getFileFormat( const std::string & );
/*! \class Mesh
@ -35,6 +54,7 @@ public:
virtual std::pair<size_t, void *> pack( int level ) const = 0;
//! Unpack the data
virtual void unpack( const std::pair<size_t, void *> &data ) = 0;
protected:
//! Empty constructor
Mesh();
@ -65,6 +85,7 @@ public:
virtual void unpack( const std::pair<size_t, void *> &data );
//! Access the points
const std::vector<Point> &getPoints() const { return points; }
public:
std::vector<Point> points; //!< List of points vertex
};
@ -93,6 +114,7 @@ public:
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::vector<Point> A; //!< First vertex
std::vector<Point> B; //!< Second vertex
@ -101,7 +123,8 @@ public:
/*! \class TriMesh
\brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes
\brief A class used to hold a list of trianges specified by their vertex number and list of
coordiantes
*/
class TriMesh : public Mesh
{
@ -124,6 +147,7 @@ public:
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::shared_ptr<PointList> vertices; //!< List of verticies
std::vector<int> A; //!< First vertex
@ -152,6 +176,7 @@ public:
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t, void *> &data );
public:
int nprocx, nprocy, nprocz, rank;
int nx, ny, nz;
@ -159,12 +184,10 @@ public:
};
/*! \class Variable
\brief A base class for variables
*/
struct Variable
{
struct Variable {
public:
// Internal variables
unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
@ -175,13 +198,19 @@ public:
//! Empty constructor
Variable() : dim( 0 ), type( VariableType::NullVariable ), precision( DataType::Double ) {}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_) {}
Variable( int dim_, IO::VariableType type_, const std::string &name_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ )
{
}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array<double>& data_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {}
Variable(
int dim_, IO::VariableType type_, const std::string &name_, const Array<double> &data_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ), data( data_ )
{
}
//! Destructor
virtual ~Variable() {}
protected:
//! Empty constructor
Variable( const Variable & );
@ -189,7 +218,6 @@ protected:
};
/*! \class MeshDataStruct
\brief A class used to hold database info for saving a mesh
*/
@ -201,7 +229,7 @@ struct MeshDataStruct {
//! Empty constructor
MeshDataStruct() : precision( DataType::Double ) {}
//! Check the data
bool check() const;
bool check( bool abort = true ) const;
};
@ -214,7 +242,6 @@ std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh );
std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh );
} // IO namespace
} // namespace IO
#endif

207
IO/MeshDatabase.cpp
View File

@ -1,51 +1,43 @@
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/Mesh.h"
#include "IO/PackData.h"
#include "IO/IOHelpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <vector>
#include <cstdio>
#include <map>
#include <set>
#include <cstdio>
#include <vector>
#include <ProfilerApp.h>
// Default pack/unpack
// clang-format off
#define INSTANTIATE_PACK( TYPE ) \
template<> \
size_t packsize<TYPE>( const TYPE &rhs ) \
{ \
return sizeof( TYPE ); \
} \
template<> \
void pack<TYPE>( const TYPE &rhs, char *buffer ) \
{ \
memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); \
} \
template<> \
void unpack<TYPE>( TYPE &data, const char *buffer ) \
{ \
memcpy( &data, buffer, sizeof( IO::MeshType ) ); \
}
INSTANTIATE_PACK( IO::VariableType )
INSTANTIATE_PACK( IO::DataType )
INSTANTIATE_PACK( IO::MeshType )
INSTANTIATE_PACK( IO::FileFormat )
// clang-format on
// MeshType
template<>
size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
{
return sizeof(IO::MeshType);
}
template<>
void pack<IO::MeshType>( const IO::MeshType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::MeshType));
}
template<>
void unpack<IO::MeshType>( IO::MeshType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::MeshType));
}
// Variable::VariableType
template<>
size_t packsize<IO::VariableType>( const IO::VariableType& rhs )
{
return sizeof(IO::VariableType);
}
template<>
void pack<IO::VariableType>( const IO::VariableType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::VariableType));
}
template<>
void unpack<IO::VariableType>( IO::VariableType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::VariableType));
}
// DatabaseEntry
template<>
size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs )
@ -56,17 +48,23 @@ template<>
void pack<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs, char *buffer )
{
size_t i = 0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.file,&buffer[i]); i+=packsize(rhs.file);
pack(rhs.offset,&buffer[i]); i+=packsize(rhs.offset);
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.file, &buffer[i] );
i += packsize( rhs.file );
pack( rhs.offset, &buffer[i] );
i += packsize( rhs.offset );
}
template<>
void unpack<IO::DatabaseEntry>( IO::DatabaseEntry &data, const char *buffer )
{
size_t i = 0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.file,&buffer[i]); i+=packsize(data.file);
unpack(data.offset,&buffer[i]); i+=packsize(data.offset);
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.file, &buffer[i] );
i += packsize( data.file );
unpack( data.offset, &buffer[i] );
i += packsize( data.offset );
}
// VariableDatabase
template<>
@ -78,53 +76,69 @@ template<>
void pack<IO::VariableDatabase>( const IO::VariableDatabase &rhs, char *buffer )
{
size_t i = 0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.dim,&buffer[i]); i+=packsize(rhs.dim);
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.dim, &buffer[i] );
i += packsize( rhs.dim );
}
template<>
void unpack<IO::VariableDatabase>( IO::VariableDatabase &data, const char *buffer )
{
size_t i = 0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.dim,&buffer[i]); i+=packsize(data.dim);
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.dim, &buffer[i] );
i += packsize( data.dim );
}
// MeshDatabase
template<>
size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase &data )
{
return packsize(data.name)
+ packsize(data.type)
+ packsize(data.meshClass)
+ packsize(data.format)
+ packsize(data.domains)
+ packsize(data.variables)
+ packsize(data.variable_data);
return packsize( data.name ) + packsize( data.type ) + packsize( data.meshClass ) +
packsize( data.format ) + packsize( data.domains ) + packsize( data.variables ) +
packsize( data.variable_data );
}
template<>
void pack<IO::MeshDatabase>( const IO::MeshDatabase &rhs, char *buffer )
{
size_t i = 0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.meshClass,&buffer[i]); i+=packsize(rhs.meshClass);
pack(rhs.format,&buffer[i]); i+=packsize(rhs.format);
pack(rhs.domains,&buffer[i]); i+=packsize(rhs.domains);
pack(rhs.variables,&buffer[i]); i+=packsize(rhs.variables);
pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data);
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.meshClass, &buffer[i] );
i += packsize( rhs.meshClass );
pack( rhs.format, &buffer[i] );
i += packsize( rhs.format );
pack( rhs.domains, &buffer[i] );
i += packsize( rhs.domains );
pack( rhs.variables, &buffer[i] );
i += packsize( rhs.variables );
pack( rhs.variable_data, &buffer[i] );
i += packsize( rhs.variable_data );
}
template<>
void unpack<IO::MeshDatabase>( IO::MeshDatabase &data, const char *buffer )
{
size_t i = 0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.meshClass,&buffer[i]); i+=packsize(data.meshClass);
unpack(data.format,&buffer[i]); i+=packsize(data.format);
unpack(data.domains,&buffer[i]); i+=packsize(data.domains);
unpack(data.variables,&buffer[i]); i+=packsize(data.variables);
unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data);
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.meshClass, &buffer[i] );
i += packsize( data.meshClass );
unpack( data.format, &buffer[i] );
i += packsize( data.format );
unpack( data.domains, &buffer[i] );
i += packsize( data.domains );
unpack( data.variables, &buffer[i] );
i += packsize( data.variables );
unpack( data.variable_data, &buffer[i] );
i += packsize( data.variable_data );
}
@ -146,10 +160,7 @@ bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const
{
return operator>( rhs ) || operator==( rhs );
}
bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const
{
return !operator>(rhs);
}
bool VariableDatabase::operator<=( const VariableDatabase &rhs ) const { return !operator>( rhs ); }
bool VariableDatabase::operator>( const VariableDatabase &rhs ) const
{
if ( name > rhs.name )
@ -175,12 +186,8 @@ bool VariableDatabase::operator<(const VariableDatabase& rhs ) const
/****************************************************
* MeshDatabase *
****************************************************/
MeshDatabase::MeshDatabase()
{
}
MeshDatabase::~MeshDatabase()
{
}
MeshDatabase::MeshDatabase() {}
MeshDatabase::~MeshDatabase() {}
MeshDatabase::MeshDatabase( const MeshDatabase &rhs )
{
name = rhs.name;
@ -223,21 +230,21 @@ std::string DatabaseEntry::write( ) const
}
DatabaseEntry::DatabaseEntry( const char *line )
{
std::vector<std::string> list = splitList(line,';');
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const char *line )
{
std::vector<std::string> list = splitList(line,';');
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const std::string &line )
{
std::vector<std::string> list = splitList(line.c_str(),';');
auto list = splitList( line.c_str(), ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
@ -246,7 +253,8 @@ void DatabaseEntry::read( const std::string& line )
// Gather the mesh databases from all processors
inline int tod( int N ) { return ( N + 7 ) / sizeof( double ); }
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm )
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm )
{
if ( comm.getSize() == 1 )
return meshes;
@ -301,7 +309,8 @@ std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, co
}
for ( auto it = data.begin(); it != data.end(); ++it ) {
// Get the unique variables
std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
std::set<VariableDatabase> data2(
it->second.variables.begin(), it->second.variables.end() );
it->second.variables = std::vector<VariableDatabase>( data2.begin(), data2.end() );
}
// Free temporary memory
@ -311,7 +320,7 @@ std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, co
// Return the results
std::vector<MeshDatabase> data2( data.size() );
size_t i = 0;
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
for ( auto it = data.begin(); it != data.end(); ++it, ++i )
data2[i] = it->second;
PROFILE_STOP( "gatherAll-unpack", 2 );
PROFILE_STOP( "gatherAll" );
@ -326,22 +335,23 @@ void write( const std::vector<MeshDatabase>& meshes, const std::string& filename
FILE *fid = fopen( filename.c_str(), "wb" );
for ( size_t i = 0; i < meshes.size(); i++ ) {
fprintf( fid, "%s\n", meshes[i].name.c_str() );
fprintf(fid," type: %i\n",static_cast<int>(meshes[i].type));
fprintf( fid, " type: %s\n", getString( meshes[i].type ).data() );
fprintf( fid, " meshClass: %s\n", meshes[i].meshClass.c_str() );
fprintf(fid," format: %i\n",static_cast<int>(meshes[i].format));
fprintf( fid, " format: %s\n", getString( meshes[i].format ).data() );
for ( size_t j = 0; j < meshes[i].domains.size(); j++ )
fprintf( fid, " domain: %s\n", meshes[i].domains[j].write().c_str() );
fprintf( fid, " variables: " );
for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) {
const VariableDatabase &var = meshes[i].variables[j];
fprintf(fid,"%s|%i|%i; ",var.name.c_str(),static_cast<int>(var.type),var.dim);
fprintf( fid, "%s|%s|%i; ", var.name.data(), getString( var.type ).data(), var.dim );
}
fprintf( fid, "\n" );
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) {
for ( auto it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end();
++it ) {
const char *domain = it->first.first.c_str();
const char *variable = it->first.second.c_str();
fprintf(fid," variable(%s,%s): %s\n",domain,variable,it->second.write().c_str());
fprintf(
fid, " variable(%s,%s): %s\n", domain, variable, it->second.write().c_str() );
}
}
fclose( fid );
@ -368,9 +378,9 @@ std::vector<MeshDatabase> read( const std::string& filename )
name.resize( name.size() - 1 );
meshes.back().name = name;
} else if ( strncmp( line, " format:", 10 ) == 0 ) {
meshes.back().format = static_cast<unsigned char>(atoi(&line[10]));
meshes.back().format = getFileFormat( &line[10] );
} else if ( strncmp( line, " type:", 8 ) == 0 ) {
meshes.back().type = static_cast<MeshType>(atoi(&line[8]));
meshes.back().type = getMeshType( &line[8] );
} else if ( strncmp( line, " meshClass:", 13 ) == 0 ) {
meshes.back().meshClass = deblank( std::string( &line[13] ) );
} else if ( strncmp( line, " domain:", 10 ) == 0 ) {
@ -384,7 +394,7 @@ std::vector<MeshDatabase> read( const std::string& filename )
std::vector<std::string> tmp = splitList( variables[i].c_str(), '|' );
ASSERT( tmp.size() == 3 );
mesh.variables[i].name = tmp[0];
mesh.variables[i].type = static_cast<VariableType>(atoi(tmp[1].c_str()));
mesh.variables[i].type = getVariableType( tmp[1] );
mesh.variables[i].dim = atoi( tmp[2].c_str() );
}
} else if ( strncmp( line, " variable(", 12 ) == 0 ) {
@ -410,14 +420,14 @@ std::vector<MeshDatabase> read( const std::string& filename )
// Return the mesh type
IO::MeshType meshType( const IO::Mesh &mesh )
{
IO::MeshType type = IO::Unknown;
IO::MeshType type = IO::MeshType::Unknown;
const std::string meshClass = mesh.className();
if ( meshClass == "PointList" ) {
type = IO::PointMesh;
type = IO::MeshType::PointMesh;
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
type = IO::SurfaceMesh;
type = IO::MeshType::SurfaceMesh;
} else if ( meshClass == "DomainMesh" ) {
type = IO::VolumeMesh;
type = IO::MeshType::VolumeMesh;
} else {
ERROR( "Unknown mesh" );
}
@ -425,5 +435,4 @@ IO::MeshType meshType( const IO::Mesh& mesh )
}
} // IO namespace
} // namespace IO

17
IO/MeshDatabase.h
View File

@ -5,21 +5,14 @@
#include "common/MPI.h"
#include <iostream>
#include <map>
#include <memory>
#include <string.h>
#include <vector>
#include <map>
namespace IO {
class Mesh;
//! Enum to identify mesh type
//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
//! Helper struct for containing offsets for the mesh info
struct DatabaseEntry {
@ -56,11 +49,12 @@ struct MeshDatabase {
std::string name; //!< Name of the mesh
MeshType type; //!< Mesh type
std::string meshClass; //!< Mesh class
unsigned char format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
FileFormat format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
std::vector<DatabaseEntry> domains; //!< List of the domains
std::vector<VariableDatabase> variables; //!< List of the variables
std::map<variable_id, DatabaseEntry> variable_data; //!< Data for the variables
VariableDatabase getVariableDatabase( const std::string &varname ) const;
public:
MeshDatabase();
~MeshDatabase();
@ -70,7 +64,8 @@ public:
//! Gather the mesh databases from all processors
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, const Utilities::MPI& comm );
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm );
//! Write the mesh databases to a file
@ -85,6 +80,6 @@ std::vector<MeshDatabase> read( const std::string& filename );
IO::MeshType meshType( const IO::Mesh &mesh );
} // IO namespace
} // namespace IO
#endif

35
IO/PIO.cpp
View File

@ -1,10 +1,10 @@
#include "IO/PIO.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <cstring>
#include <fstream>
#include <string>
#include <cstring>
namespace IO {
@ -20,7 +20,6 @@ std::ostream perr(&perr_buffer);
std::ostream plog( &plog_buffer );
/****************************************************************************
* Functions to control logging *
****************************************************************************/
@ -95,18 +94,12 @@ void Utilities::stopLogging( )
/****************************************************************************
* ParallelStreamBuffer class *
****************************************************************************/
ParallelStreamBuffer::ParallelStreamBuffer( ):
d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL)
ParallelStreamBuffer::ParallelStreamBuffer()
: d_rank( 0 ), d_size( 0 ), d_buffer_size( 0 ), d_buffer( NULL )
{
}
ParallelStreamBuffer:: ~ParallelStreamBuffer()
{
delete [] d_buffer;
}
void ParallelStreamBuffer::setOutputStream( std::ostream *stream )
{
d_stream.push_back( stream );
}
ParallelStreamBuffer::~ParallelStreamBuffer() { delete[] d_buffer; }
void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) { d_stream.push_back( stream ); }
int ParallelStreamBuffer::sync()
{
for ( size_t i = 0; i < d_stream.size(); i++ ) {
@ -140,7 +133,9 @@ std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize
reserve( d_size + n );
memcpy( &d_buffer[d_size], text, n );
d_size += n;
if ( text[n-1]==0 || text[n-1]==10 ) { sync(); }
if ( text[n - 1] == 0 || text[n - 1] == 10 ) {
sync();
}
return n;
}
int ParallelStreamBuffer::overflow( int ch )
@ -148,13 +143,12 @@ int ParallelStreamBuffer::overflow(int ch)
reserve( d_size + 1 );
d_buffer[d_size] = ch;
d_size++;
if ( ch==0 || ch==10 ) { sync(); }
if ( ch == 0 || ch == 10 ) {
sync();
}
return std::char_traits<char>::to_int_type( ch );
}
int ParallelStreamBuffer::underflow()
{
return -1;
}
int ParallelStreamBuffer::underflow() { return -1; }
void ParallelStreamBuffer::reset()
{
sync();
@ -165,5 +159,4 @@ void ParallelStreamBuffer::reset()
}
} // IO namespace
} // namespace IO

1
IO/PIO.h
View File

@ -45,7 +45,6 @@ inline int printp( const char *format, ... );
class ParallelStreamBuffer : public std::streambuf
{
public:
/*!
* Create a parallel buffer class. The object will require further
* initialization to set up the I/O streams and prefix string.

4
IO/PIO.hpp
View File

@ -3,9 +3,9 @@
#include "IO/PIO.h"
#include <cstdio>
#include <iostream>
#include <stdarg.h>
#include <cstdio>
namespace IO {
@ -24,6 +24,6 @@ inline int printp( const char *format, ... )
}
} // IO namespace
} // namespace IO
#endif

1
IO/PackData.cpp
View File

@ -102,4 +102,3 @@ void unpack<std::string>( std::string& data, const char *buffer )
{
data = std::string( buffer );
}

5
IO/PackData.h
View File

@ -2,9 +2,9 @@
#ifndef included_PackData
#define included_PackData
#include <vector>
#include <set>
#include <map>
#include <set>
#include <vector>
//! Template function to return the buffer size required to pack a class
@ -75,4 +75,3 @@ void unpack( std::set<TYPE>& data, const char *buffer );
#include "IO/PackData.hpp"
#endif

24
IO/PackData.hpp
View File

@ -4,11 +4,10 @@
#include "IO/PackData.h"
#include <map>
#include <set>
#include <string.h>
#include <vector>
#include <set>
#include <map>
/********************************************************
@ -92,8 +91,10 @@ void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer )
size_t pos = sizeof( size_t );
typename std::map<TYPE1, TYPE2>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack(it->first,&buffer[pos]); pos+=packsize(it->first);
pack(it->second,&buffer[pos]); pos+=packsize(it->second);
pack( it->first, &buffer[pos] );
pos += packsize( it->first );
pack( it->second, &buffer[pos] );
pos += packsize( it->second );
}
}
template<class TYPE1, class TYPE2>
@ -105,8 +106,10 @@ void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer )
data.clear();
for ( size_t i = 0; i < N; i++ ) {
std::pair<TYPE1, TYPE2> tmp;
unpack(tmp.first,&buffer[pos]); pos+=packsize(tmp.first);
unpack(tmp.second,&buffer[pos]); pos+=packsize(tmp.second);
unpack( tmp.first, &buffer[pos] );
pos += packsize( tmp.first );
unpack( tmp.second, &buffer[pos] );
pos += packsize( tmp.second );
data.insert( tmp );
}
}
@ -133,7 +136,8 @@ void pack( const std::set<TYPE>& rhs, char *buffer )
size_t pos = sizeof( size_t );
typename std::set<TYPE>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack(*it); pos+=packsize(*it);
pack( *it );
pos += packsize( *it );
}
}
template<class TYPE>
@ -145,11 +149,11 @@ void unpack( std::set<TYPE>& data, const char *buffer )
data.clear();
for ( size_t i = 0; i < N; i++ ) {
TYPE tmp;
unpack(tmp,&buffer[pos]); pos+=packsize(tmp);
unpack( tmp, &buffer[pos] );
pos += packsize( tmp );
data.insert( tmp );
}
}
#endif

138
IO/Reader.cpp
View File

@ -1,7 +1,7 @@
#include "IO/Reader.h"
#include "IO/IOHelpers.h"
#include "IO/Mesh.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#ifdef USE_SILO
@ -10,18 +10,30 @@
#include <ProfilerApp.h>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <map>
#include <string.h>
#include <vector>
#include <map>
#include <cstdio>
// Inline function to read line without a return argument
static inline void fgetl( char *str, int num, FILE *stream )
{
char *ptr = fgets( str, num, stream );
if ( 0 ) {char *temp = (char *)&ptr; temp++;}
if ( 0 ) {
char *temp = (char *) &ptr;
temp++;
}
}
// Check if the file exists
bool fileExists( const std::string &filename )
{
std::ifstream ifile( filename.c_str() );
return ifile.good();
}
@ -31,16 +43,42 @@ std::string IO::getPath( const std::string& filename )
std::string file( filename );
size_t k1 = file.rfind( 47 );
size_t k2 = file.rfind( 92 );
if ( k1==std::string::npos ) { k1=0; }
if ( k2==std::string::npos ) { k2=0; }
if ( k1 == std::string::npos ) {
k1 = 0;
}
if ( k2 == std::string::npos ) {
k2 = 0;
}
return file.substr( 0, std::max( k1, k2 ) );
}
// List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string& filename )
// List the timesteps in the given directory (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string &path, const std::string &format )
{
// Get the name of the summary filename
std::string filename = path + "/";
if ( format == "old" || format == "new" ) {
filename += "summary.LBM";
} else if ( format == "silo" ) {
filename += "LBM.visit";
} else if ( format == "auto" ) {
bool test_old = fileExists( path + "/summary.LBM" );
bool test_silo = fileExists( path + "/LBM.visit" );
if ( test_old && test_silo ) {
ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" );
} else if ( test_old ) {
filename += "summary.LBM";
} else if ( test_silo ) {
filename += "LBM.visit";
} else {
ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" );
}
} else {
ERROR( "Unknown format: " + format );
}
PROFILE_START( "readTimesteps" );
// Read the data
FILE *fid = fopen( filename.c_str(), "rb" );
if ( fid == NULL )
ERROR( "Error opening file" );
@ -59,11 +97,52 @@ std::vector<std::string> IO::readTimesteps( const std::string& filename )
fclose( fid );
PROFILE_STOP( "readTimesteps" );
return timesteps;
return timesteps;
}
// Get the maximum number of domains
int IO::maxDomains( const std::string &path, const std::string &format, const Utilities::MPI &comm )
{
int rank = comm.getRank();
int n_domains = 0;
if ( rank == 0 ) {
// Get the timesteps
auto timesteps = IO::readTimesteps( path, format );
ASSERT( !timesteps.empty() );
// Get the database for the first domain
auto db = IO::getMeshList( path, timesteps[0] );
for ( size_t i = 0; i < db.size(); i++ )
n_domains = std::max<int>( n_domains, db[i].domains.size() );
}
return comm.bcast( n_domains, 0 );
}
// Read the data for the given timestep
std::vector<IO::MeshDataStruct> IO::readData(
const std::string &path, const std::string &timestep, int rank )
{
// Get the mesh databases
auto db = IO::getMeshList( path, timestep );
// Create the data
std::vector<IO::MeshDataStruct> data( db.size() );
for ( size_t i = 0; i < data.size(); i++ ) {
data[i].precision = IO::DataType::Double;
data[i].meshName = db[i].name;
data[i].mesh = getMesh( path, timestep, db[i], rank );
data[i].vars.resize( db[i].variables.size() );
for ( size_t j = 0; j < db[i].variables.size(); j++ )
data[i].vars[j] = getVariable( path, timestep, db[i], rank, db[i].variables[j].name );
INSIST( data[i].check(), "Failed check of " + data[i].meshName );
}
return data;
}
// Read the list of variables for the given timestep
std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const std::string& timestep )
std::vector<IO::MeshDatabase> IO::getMeshList(
const std::string &path, const std::string &timestep )
{
std::string filename = path + "/" + timestep + "/LBM.summary";
return IO::read( filename );
@ -76,7 +155,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
{
PROFILE_START( "getMesh" );
std::shared_ptr<IO::Mesh> mesh;
if ( meshDatabase.format==1 ) {
if ( meshDatabase.format == FileFormat::OLD ) {
// Old format (binary doubles)
std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file;
FILE *fid = fopen( filename.c_str(), "rb" );
@ -90,7 +169,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
fclose( fid );
if ( count % 3 != 0 )
ERROR( "Error reading file" );
if ( meshDatabase.type==IO::PointMesh ) {
if ( meshDatabase.type == IO::MeshType::PointMesh ) {
size_t N = count / 3;
std::shared_ptr<PointList> pointlist( new PointList( N ) );
std::vector<Point> &P = pointlist->points;
@ -100,7 +179,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
P[i].z = data[3 * i + 2];
}
mesh = pointlist;
} else if ( meshDatabase.type==IO::SurfaceMesh ) {
} else if ( meshDatabase.type == IO::MeshType::SurfaceMesh ) {
if ( count % 9 != 0 )
ERROR( "Error reading file (2)" );
size_t N_tri = count / 9;
@ -120,14 +199,15 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
C[i].z = data[9 * i + 8];
}
mesh = trilist;
} else if ( meshDatabase.type==IO::VolumeMesh ) {
} else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) {
// this was never supported in the old format
mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
} else {
ERROR( "Unknown mesh type" );
}
delete[] data;
} else if ( meshDatabase.format==2 ) {
} else if ( meshDatabase.format == FileFormat::NEW ||
meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen( filename.c_str(), "rb" );
@ -154,7 +234,7 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
}
mesh->unpack( std::pair<size_t, void *>( bytes, data ) );
delete[] data;
} else if ( meshDatabase.format==4 ) {
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const DatabaseEntry &database = meshDatabase.domains[domain];
@ -200,7 +280,8 @@ std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::strin
silo::readUniformMesh( fid, database.name, range, N );
auto rankinfo = silo::read<int>( fid, database.name + "_rankinfo" );
RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0],
range[3] - range[2], range[5] - range[4] ) );
} else {
ERROR( "Unknown mesh class" );
}
@ -221,12 +302,11 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const st
const MeshDatabase &meshDatabase, int domain, const std::string &variable )
{
std::pair<std::string, std::string> key( meshDatabase.domains[domain].name, variable );
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
it = meshDatabase.variable_data.find(key);
auto it = meshDatabase.variable_data.find( key );
if ( it == meshDatabase.variable_data.end() )
return std::shared_ptr<IO::Variable>();
std::shared_ptr<IO::Variable> var;
if ( meshDatabase.format == 2 ) {
if ( meshDatabase.format == FileFormat::NEW || meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = it->second;
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen( filename.c_str(), "rb" );
@ -238,15 +318,15 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const st
std::vector<std::string> values = splitList( &line[i2 + 1], ',' );
ASSERT( values.size() == 5 );
int dim = atoi( values[0].c_str() );
int type = atoi(values[1].c_str());
auto type = values[1];
size_t N = atol( values[2].c_str() );
size_t bytes = atol( values[3].c_str() );
std::string precision = values[4];
var = std::shared_ptr<IO::Variable>( new IO::Variable() );
var->dim = dim;
var->type = static_cast<IO::VariableType>(type);
var->type = getVariableType( type );
var->name = variable;
var->data.resize(N*dim);
var->data.resize( N, dim );
if ( precision == "double" ) {
size_t count = fread( var->data.data(), sizeof( double ), N * dim, fid );
ASSERT( count * sizeof( double ) == bytes );
@ -254,7 +334,7 @@ std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const st
ERROR( "Format not implimented" );
}
fclose( fid );
} else if ( meshDatabase.format == 4 ) {
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const auto &database = meshDatabase.domains[domain];
@ -291,9 +371,12 @@ void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
if ( mesh.className() == "DomainMesh" ) {
const IO::DomainMesh &mesh2 = dynamic_cast<const IO::DomainMesh &>( mesh );
if ( var.type == VariableType::NodeVariable ) {
size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1));
ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() );
var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } );
size_t N2 =
var.data.length() / ( ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) );
ASSERT(
( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) * N2 == var.data.length() );
var.data.reshape(
{ (size_t) mesh2.nx + 1, (size_t) mesh2.ny + 1, (size_t) mesh2.nz + 1, N2 } );
} else if ( var.type == VariableType::EdgeVariable ) {
ERROR( "Not finished" );
} else if ( var.type == VariableType::SurfaceVariable ) {
@ -335,6 +418,3 @@ void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
ERROR( "Unknown mesh type" );
}
}

45
IO/Reader.h
View File

@ -17,8 +17,47 @@ namespace IO {
std::string getPath( const std::string &filename );
//! List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> readTimesteps( const std::string& filename );
/*!
* @brief Get the maximum number of domains written
* @details This function reads the summary files to determine the maximum
* number of domains in the output.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @param[in] comm Optional comm to use to reduce IO load by
* reading on rank 0 and then communicating the result
*/
int maxDomains( const std::string &path, const std::string &format = "auto",
const Utilities::MPI &comm = MPI_COMM_SELF );
/*!
* @brief Read the timestep list
* @details This function reads the timestep list from the summary file.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @return append Append any existing data (default is false)
*/
std::vector<std::string> readTimesteps(
const std::string &path, const std::string &format = "auto" );
/*!
* @brief Read the data for the timestep
* @details This function reads the mesh and variable data provided for the given timestep.
* @param[in] path The path to use for reading
* @param[in] timestep The timestep iteration
* @param[in] domain The desired domain to read
*/
std::vector<IO::MeshDataStruct> readData(
const std::string &path, const std::string &timestep, int domain );
//! Read the list of mesh databases for the given timestep
@ -53,6 +92,6 @@ std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::s
void reformatVariable( const IO::Mesh &mesh, IO::Variable &var );
} // IO namespace
} // namespace IO
#endif

216
IO/Writer.cpp
View File

@ -1,21 +1,62 @@
#include "IO/Writer.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/MeshDatabase.h"
#include "IO/silo.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <sys/stat.h>
#include <algorithm>
#include <vector>
#include <set>
#include <memory>
#include <set>
#include <sys/stat.h>
#include <vector>
enum class Format { OLD, NEW, SILO, UNKNOWN };
/****************************************************
* Recursively create the subdirectory *
****************************************************/
static void recursiveMkdir( const std::string &path, mode_t mode )
{
// Iterate through the root directories until we create the desired path
for ( size_t pos = 0; pos < path.size(); ) {
// slide backwards in string until next slash found
pos++;
for ( ; pos < path.size(); pos++ ) {
if ( path[pos] == '/' || path[pos] == 92 )
break;
}
// Create the temporary path
auto path2 = path.substr( 0, pos );
// Check if the temporary path exists
struct stat status;
int result = stat( path2.data(), &status );
if ( result == 0 ) {
// if there is a part of the path that already exists make sure it is really a directory
if ( !S_ISDIR( status.st_mode ) ) {
ERROR(
"Error in recursiveMkdir...\n"
" Cannot create directories in path = " +
path +
"\n because some intermediate item in path exists and is NOT a directory" );
}
continue;
}
// Create the directory and test the result
result = mkdir( path2.data(), mode );
if ( result != 0 ) {
// Maybe another rank created the directory, check
int result = stat( path2.data(), &status );
if ( result != 0 && !S_ISDIR( status.st_mode ) )
ERROR( "Error in Utilities::recursiveMkdir...\n"
" Cannot create directory = " +
path2 );
}
}
}
/****************************************************
* Initialize the writer *
@ -38,7 +79,7 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
ERROR( "Unknown format" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( !append && rank == 0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
recursiveMkdir( path, S_IRWXU | S_IRGRP );
std::string filename;
if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW )
filename = global_IO_path + "/summary.LBM";
@ -53,9 +94,9 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
// Write the mesh data in the original format
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
for ( size_t i = 0; i < meshData.size(); i++ ) {
char domainname[100], filename[100], fullpath[200];
@ -69,25 +110,28 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO
mesh_entry.name = meshData[i].meshName;
mesh_entry.type = meshType( *mesh );
mesh_entry.meshClass = meshData[i].mesh->className();
mesh_entry.format = 1;
mesh_entry.format = IO::FileFormat::OLD;
IO::DatabaseEntry domain;
domain.name = domainname;
domain.file = filename;
domain.offset = 0;
mesh_entry.domains.push_back( domain );
if ( !meshData[i].vars.empty() ) {
printf("Warning: variables are not supported with this format\n");
printf( "Warning: variables are not supported with this format (original)\n" );
// for (size_t j=0; j<meshData[i].vars.size(); j++)
// mesh_entry.variables.push_back( meshData[i].vars[j]->name );
}
const std::string meshClass = mesh->className();
if ( meshClass == "PointList" ) {
// List of points
std::shared_ptr<IO::PointList> pointlist = std::dynamic_pointer_cast<IO::PointList>(mesh);
std::shared_ptr<IO::PointList> pointlist =
std::dynamic_pointer_cast<IO::PointList>( mesh );
const std::vector<Point> &P = pointlist->points;
for ( size_t i = 0; i < P.size(); i++ ) {
double x[3];
x[0] = P[i].x; x[1] = P[i].y; x[2] = P[i].z;
x[0] = P[i].x;
x[1] = P[i].y;
x[2] = P[i].z;
fwrite( x, sizeof( double ), 3, fid );
}
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
@ -98,9 +142,15 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO
const std::vector<Point> &C = trilist->C;
for ( size_t i = 0; i < A.size(); i++ ) {
double tri[9];
tri[0] = A[i].x; tri[1] = A[i].y; tri[2] = A[i].z;
tri[3] = B[i].x; tri[4] = B[i].y; tri[5] = B[i].z;
tri[6] = C[i].x; tri[7] = C[i].y; tri[8] = C[i].z;
tri[0] = A[i].x;
tri[1] = A[i].y;
tri[2] = A[i].z;
tri[3] = B[i].x;
tri[4] = B[i].y;
tri[5] = B[i].z;
tri[6] = C[i].x;
tri[7] = C[i].y;
tri[8] = C[i].z;
fwrite( tri, sizeof( double ), 9, fid );
}
} else if ( meshClass == "DomainMesh" ) {
@ -110,7 +160,9 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO
}
fclose( fid );
std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() );
mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() );
mesh_entry.variables.erase(
std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ),
mesh_entry.variables.end() );
meshes_written.push_back( mesh_entry );
}
return meshes_written;
@ -118,9 +170,9 @@ static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO
// Create the database entry for the mesh data
static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase getDatabase(
const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
char domainname[100];
sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank );
// Create the MeshDatabase
@ -158,12 +210,11 @@ static IO::MeshDatabase getDatabase( const std::string& filename, const IO::Mesh
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
const IO::MeshDataStruct& mesh, int format )
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
const int level = 0;
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
// Create the MeshDatabase
IO::MeshDatabase database = getDatabase( filename, mesh, format );
IO::MeshDatabase database = getDatabase( filename, mesh, format, rank );
// Write the mesh
IO::DatabaseEntry &domain = database.domains[0];
domain.offset = ftell( fid );
@ -174,20 +225,17 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
delete[]( char * ) data.second;
// Write the variables
for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
ASSERT( mesh.vars[i]->type != IO::VariableType::NullVariable );
std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
IO::DatabaseEntry& variable = database.variable_data[key];
auto &variable = database.variable_data[key];
variable.offset = ftell( fid );
int dim = mesh.vars[i]->dim;
int type = static_cast<int>(mesh.vars[i]->type);
auto type = getString( mesh.vars[i]->type );
size_t N = mesh.vars[i]->data.length();
if ( type == static_cast<int>(IO::VariableType::NullVariable) ) {
ERROR("Variable type not set");
}
size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type );
ASSERT( N == dim * N_mesh );
fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n",
database.name.c_str(), rank, variable.name.c_str(),
dim, type, N_mesh, N*sizeof(double) );
fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank,
variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) );
fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid );
fprintf( fid, "\n" );
}
@ -198,7 +246,8 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
#ifdef USE_SILO
// Write a PointList mesh (and variables) to a file
template<class TYPE>
static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname )
static void writeSiloPointMesh(
DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
{
const auto &points = mesh.getPoints();
std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
@ -208,9 +257,10 @@ static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const st
z[i] = points[i].z;
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
}
static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloPointList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::PointList &mesh = dynamic_cast<IO::PointList &>( *meshData.mesh );
const std::string meshname = database.domains[0].name;
@ -229,19 +279,19 @@ static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData,
z[i] = points[i].z;
}
const double *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh( fid, meshname, 3, points.size(), coords );
IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writePointMeshVariable( fid, meshname, var.name, var.data );
IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else {
ERROR( "Unsupported format" );
}
@ -260,7 +310,7 @@ static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::s
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
}
static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
const IO::TriMesh &mesh, IO::MeshDatabase database )
@ -275,57 +325,60 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
}
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
if ( var.precision == IO::DataType::Double ) {
silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else {
ERROR( "Unsupported format" );
}
}
}
static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
writeSiloTriMesh2( fid, meshData, mesh, database );
}
static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
auto mesh = getTriMesh( meshData.mesh );
writeSiloTriMesh2( fid, meshData, *mesh, database );
}
// Write a DomainMesh mesh (and variables) to a file
static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloDomainMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
std::array<double,6> range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx,
info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny,
info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz };
std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
( info.kz + 1 ) * mesh.Lz / info.nz };
std::array<int, 3> N = { mesh.nx, mesh.ny, mesh.nz };
auto meshname = database.domains[0].name;
silo::writeUniformMesh<3>( fid, meshname, range, N );
silo::write<int>( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
IO::silo::write<int>(
fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const auto &var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
if ( var.precision == IO::DataType::Double ) {
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else {
ERROR( "Unsupported format" );
}
@ -333,10 +386,10 @@ static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData
}
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
const IO::MeshDataStruct& mesh, int format )
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
// Create the MeshDatabase
auto database = getDatabase( filename, mesh, format );
auto database = getDatabase( filename, mesh, format, rank );
if ( database.meshClass == "PointList" ) {
writeSiloPointList( fid, mesh, database );
} else if ( database.meshClass == "TriMesh" ) {
@ -372,9 +425,10 @@ std::pair<int,int> getSiloMeshType( const std::string& meshClass )
}
return std::make_pair( meshType, varType );
}
void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, const std::string& filename )
void writeSiloSummary(
const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
{
auto fid = silo::open( filename, silo::CREATE );
auto fid = IO::silo::open( filename, IO::silo::CREATE );
for ( const auto &data : meshes_written ) {
auto type = getSiloMeshType( data.meshClass );
std::vector<int> meshTypes( data.domains.size(), type.first );
@ -382,24 +436,24 @@ void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, cons
std::vector<std::string> meshNames;
for ( const auto &tmp : data.domains )
meshNames.push_back( tmp.file + ":" + tmp.name );
silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
for ( const auto &variable : data.variables ) {
std::vector<std::string> varnames;
for ( const auto &tmp : data.domains )
varnames.push_back( tmp.file + ":" + variable.name );
silo::writeMultiVar( fid, variable.name, varnames, varTypes );
IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
}
}
silo::close( fid );
IO::silo::close( fid );
}
#endif
// Write the mesh data in the new format
static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf( filename, "%05i", rank );
@ -407,7 +461,7 @@ static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
FILE *fid = fopen( fullpath, "wb" );
for ( size_t i = 0; i < meshData.size(); i++ ) {
std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
meshes_written.push_back( write_domain(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) );
}
fclose( fid );
return meshes_written;
@ -416,22 +470,26 @@ static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
// Write the mesh data to silo
static std::vector<IO::MeshDatabase> writeMeshesSilo(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
#ifdef USE_SILO
int rank = Utilities::MPI(MPI_COMM_WORLD).getRank();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf( filename, "%05i.silo", rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
auto fid = silo::open( fullpath, silo::CREATE );
auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
for ( size_t i = 0; i < meshData.size(); i++ ) {
auto mesh = meshData[i].mesh;
meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
}
silo::close( fid );
IO::silo::close( fid );
return meshes_written;
#else
NULL_USE( meshData );
NULL_USE( path );
NULL_USE( format );
NULL_USE( rank );
ERROR( "Application built without silo support" );
return std::vector<IO::MeshDatabase>();
#endif
@ -441,34 +499,30 @@ static std::vector<IO::MeshDatabase> writeMeshesSilo(
/****************************************************
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm )
{
if ( global_IO_path.empty() )
IO::initialize();
PROFILE_START( "writeData" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
// Check the meshData before writing
for ( const auto& data : meshData ) {
if ( !data.check() )
ERROR("Error in meshData");
}
for ( const auto &data : meshData )
ASSERT( data.check() );
// Create the output directory
std::string path = global_IO_path + "/" + subdir;
if ( rank == 0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
}
comm.barrier();
recursiveMkdir( path, S_IRWXU | S_IRGRP );
// Write the mesh files
std::vector<IO::MeshDatabase> meshes_written;
if ( global_IO_format == Format::OLD ) {
// Write the original triangle format
meshes_written = writeMeshesOrigFormat( meshData, path );
meshes_written = writeMeshesOrigFormat( meshData, path, rank );
} else if ( global_IO_format == Format::NEW ) {
// Write the new format (double precision)
meshes_written = writeMeshesNewFormat( meshData, path, 2 );
meshes_written = writeMeshesNewFormat( meshData, path, IO::FileFormat::NEW, rank );
} else if ( global_IO_format == Format::SILO ) {
// Write silo
meshes_written = writeMeshesSilo( meshData, path, 4 );
meshes_written = writeMeshesSilo( meshData, path, IO::FileFormat::SILO, rank );
} else {
ERROR( "Unknown format" );
}
@ -504,5 +558,3 @@ void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStr
}
PROFILE_STOP( "writeData" );
}

23
IO/Writer.h
View File

@ -14,17 +14,18 @@ namespace IO {
/*!
* @brief Initialize the writer
* @details This function initializes the writer to the given path. All subsequent
* writes will occur in this directory. If this is not called, then it will default
* to the current path.
* @details This function initializes the writer to the given path.
* All subsequent writes will occur in this directory.
* If this is not called, then it will default to the current path.
* @param[in] path The path to use for writes
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process
* silo - Silo
* old - Old mesh format
* (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process silo - Silo
* @param[in] append Append any existing data (default is false)
*/
void initialize( const std::string& path="", const std::string& format="silo", bool append=false );
void initialize(
const std::string &path = "", const std::string &format = "silo", bool append = false );
/*!
@ -34,7 +35,8 @@ void initialize( const std::string& path="", const std::string& format="silo", b
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm );
void writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm );
/*!
@ -44,7 +46,8 @@ void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, const Utilities::MPI& comm )
inline void writeData(
int timestep, const std::vector<IO::MeshDataStruct> &meshData, const Utilities::MPI &comm )
{
char subdir[100];
sprintf( subdir, "vis%03i", timestep );
@ -52,6 +55,6 @@ inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& mesh
}
} // IO namespace
} // namespace IO
#endif

111
IO/netcdf.cpp
View File

@ -1,6 +1,6 @@
#include "IO/netcdf.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -56,12 +56,33 @@ static inline VariableType convertType( nc_type type )
// Get nc_type from the template
template<class T> inline nc_type getType();
template<> inline nc_type getType<char>() { return NC_CHAR; }
template<> inline nc_type getType<short>() { return NC_SHORT; }
template<> inline nc_type getType<int>() { return NC_INT; }
template<> inline nc_type getType<float>() { return NC_FLOAT; }
template<> inline nc_type getType<double>() { return NC_DOUBLE; }
template<class T>
inline nc_type getType();
template<>
inline nc_type getType<char>()
{
return NC_CHAR;
}
template<>
inline nc_type getType<short>()
{
return NC_SHORT;
}
template<>
inline nc_type getType<int>()
{
return NC_INT;
}
template<>
inline nc_type getType<float>()
{
return NC_FLOAT;
}
template<>
inline nc_type getType<double>()
{
return NC_DOUBLE;
}
// Function to reverse an array
@ -134,13 +155,16 @@ int open( const std::string& filename, FileMode mode, const Utilities::MPI& comm
}
} else {
if ( mode == READ ) {
int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_open_par(
filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == WRITE ) {
int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_open_par( filename.c_str(), NC_WRITE | NC_MPIPOSIX, comm.getCommunicator(),
MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm.getCommunicator(), MPI_INFO_NULL, &fid );
int err = nc_create_par( filename.c_str(), NC_NETCDF4 | NC_MPIIO,
comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else {
ERROR( "Unknown file mode" );
@ -239,7 +263,6 @@ VariableType getAttType( int fid, const std::string& att )
}
/****************************************************
* Read a variable *
****************************************************/
@ -329,9 +352,8 @@ Array<std::string> getVar<std::string>( int fid, const std::string& var )
PROFILE_STOP( "getVar<std::string>" );
return text;
}
static inline void get_stride_args( const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride,
size_t *startp, size_t *countp, ptrdiff_t *stridep )
static inline void get_stride_args( const std::vector<int> &start, const std::vector<int> &count,
const std::vector<int> &stride, size_t *startp, size_t *countp, ptrdiff_t *stridep )
{
for ( size_t i = 0; i < start.size(); i++ )
startp[i] = start[i];
@ -341,29 +363,29 @@ static inline void get_stride_args( const std::vector<int>& start,
stridep[i] = stride[i];
}
template<class TYPE>
int nc_get_vars_TYPE( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], TYPE *ptr );
int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], TYPE *ptr );
template<>
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], short *ptr )
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], short *ptr )
{
return nc_get_vars_short( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], int *ptr )
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], int *ptr )
{
return nc_get_vars_int( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], float *ptr )
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], float *ptr )
{
return nc_get_vars_float( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], double *ptr )
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], double *ptr )
{
return nc_get_vars_double( fid, varid, start, count, stride, ptr );
}
@ -377,7 +399,8 @@ Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& sta
if ( start[d] < 0 || start[d] + stride[d] * ( count[d] - 1 ) > (int) var_size[d] ) {
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
char tmp[1000];
sprintf(tmp,"%i: Range exceeded array dimension:\n"
sprintf( tmp,
"%i: Range exceeded array dimension:\n"
" start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",
rank, d, start[d], d, count[d], d, stride[d], d, (int) var_size[d] );
ERROR( tmp );
@ -387,15 +410,20 @@ Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& sta
size_t startp[10], countp[10];
ptrdiff_t stridep[10];
get_stride_args( start, count, stride, startp, countp, stridep );
int err = nc_get_vars_TYPE<TYPE>( fid, getVarID(fid,var), startp, countp, stridep, x.data() );
int err =
nc_get_vars_TYPE<TYPE>( fid, getVarID( fid, var ), startp, countp, stridep, x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP( "getVar<> (strided)" );
return x.reverseDim();
}
template Array<short> getVar<short>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<int> getVar<int>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<float> getVar<float>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<double> getVar<double>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<short> getVar<short>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<int> getVar<int>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<float> getVar<float>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<double> getVar<double>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
/****************************************************
@ -427,7 +455,8 @@ Array<std::string> getAtt<std::string>( int fid, const std::string& att )
/****************************************************
* Write an array to a file *
****************************************************/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims )
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims )
{
std::vector<int> dimid( names.size(), 0 );
for ( size_t i = 0; i < names.size(); i++ ) {
@ -453,20 +482,22 @@ void write( int fid, const std::string& var, const std::vector<int>& dimids,
// parallel write: each process writes its subarray to the file
auto x = data.reverseDim();
std::vector<size_t> count = { data.size( 0 ), data.size( 1 ), data.size( 2 ) };
std::vector<size_t> start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) };
std::vector<size_t> start = { info.ix * data.size( 0 ), info.jy * data.size( 1 ),
info.kz * data.size( 2 ) };
nc_put_vara( fid, varid, start.data(), count.data(), x.data() );
}
template void write<short>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<short>& data, const RankInfoStruct& info );
template void write<int>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<int>& data, const RankInfoStruct& info );
template void write<float>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<float>& data, const RankInfoStruct& info );
template void write<double>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<double>& data, const RankInfoStruct& info );
template void write<short>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<short> &data, const RankInfoStruct &info );
template void write<int>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<int> &data, const RankInfoStruct &info );
template void write<float>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<float> &data, const RankInfoStruct &info );
template void write<double>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<double> &data, const RankInfoStruct &info );
}; // netcdf namespace
}; // namespace netcdf
#else
#endif

11
IO/netcdf.h
View File

@ -5,9 +5,8 @@
#include <vector>
#include "common/Array.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/MPI.h"
namespace netcdf {
@ -125,7 +124,8 @@ Array<TYPE> getAtt( int fid, const std::string& att );
* @details This function writes the grid dimensions to netcdf.
* @param fid Handle to the open file
*/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims );
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims );
/*!
@ -134,8 +134,9 @@ std::vector<int> defDim( int fid, const std::vector<std::string>& names, const s
* @param fid Handle to the open file
*/
template<class TYPE>
void write( int fid, const std::string& var, const std::vector<int>& dimids, const Array<TYPE>& data, const RankInfoStruct& rank_info );
void write( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<TYPE> &data, const RankInfoStruct &rank_info );
}; // netcdf namespace
}; // namespace netcdf
#endif

39
IO/silo.cpp
View File

@ -1,6 +1,6 @@
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -10,8 +10,7 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
@ -29,25 +28,22 @@ DBfile* open( const std::string& filename, FileMode mode )
}
return fid;
}
void close( DBfile* fid )
{
DBClose( fid );
}
void close( DBfile *fid ) { DBClose( fid ); }
/****************************************************
* Helper functions *
****************************************************/
VariableDataType varDataType( DBfile *fid, const std::string& name )
DataType varDataType( DBfile *fid, const std::string &name )
{
auto type = DBGetVarType( fid, name.c_str() );
VariableDataType type2 = VariableDataType::UNKNOWN;
DataType type2 = DataType::Null;
if ( type == DB_DOUBLE )
type2 = VariableDataType::DOUBLE;
type2 = DataType::Double;
else if ( type == DB_FLOAT )
type2 = VariableDataType::FLOAT;
type2 = DataType::Float;
else if ( type == DB_INT )
type2 = VariableDataType::INT;
type2 = DataType::Int;
return type2;
}
@ -55,8 +51,8 @@ VariableDataType varDataType( DBfile *fid, const std::string& name )
/****************************************************
* Write/read a uniform mesh to silo *
****************************************************/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N )
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N )
{
DBquadmesh *mesh = DBGetQuadmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
@ -75,8 +71,7 @@ void readUniformMesh( DBfile* fid, const std::string& meshname,
* Write a multimesh *
****************************************************/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string>& meshNames,
const std::vector<int>& meshTypes )
const std::vector<std::string> &meshNames, const std::vector<int> &meshTypes )
{
std::vector<char *> meshnames( meshNames.size() );
for ( size_t i = 0; i < meshNames.size(); ++i )
@ -84,7 +79,8 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
std::string tree_name = meshname + "_tree";
DBoptlist *optList = DBMakeOptlist( 1 );
DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(),
(int *) meshTypes.data(), nullptr );
DBFreeOptlist( optList );
}
@ -93,18 +89,17 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
* Write a multivariable *
****************************************************/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string>& varNames,
const std::vector<int>& varTypes )
const std::vector<std::string> &varNames, const std::vector<int> &varTypes )
{
std::vector<char *> varnames( varNames.size(), nullptr );
for ( size_t j = 0; j < varNames.size(); j++ )
varnames[j] = const_cast<char *>( varNames[j].c_str() );
DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr );
DBPutMultivar(
fid, varname.c_str(), varNames.size(), varnames.data(), (int *) varTypes.data(), nullptr );
}
}; // silo namespace
}; // namespace IO::silo
#else

44
IO/silo.h
View File

@ -1,13 +1,14 @@
#ifndef SILO_INTERFACE
#define SILO_INTERFACE
#include <array>
#include <string>
#include <vector>
#include <array>
#include "IO/Mesh.h"
#include "common/Array.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/MPI.h"
#ifdef USE_SILO
@ -17,16 +18,11 @@
#endif
namespace silo {
namespace IO::silo {
enum FileMode { READ, WRITE, CREATE };
enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
/*!
* @brief Open silo file
@ -52,7 +48,7 @@ void close( DBfile* fid );
* @param[in] fid Handle to the open file
* @param[in] name Name of variable
*/
VariableDataType varDataType( DBfile *dbfile, const std::string& name );
DataType varDataType( DBfile *dbfile, const std::string &name );
/*!
@ -98,8 +94,8 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname,
* @param[out] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
* @param[out] N Number of cells in each direction
*/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N );
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N );
/*!
@ -113,8 +109,9 @@ void readUniformMesh( DBfile* fid, const std::string& meshname,
* @param[in] type Variable type
*/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type );
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type );
/*!
@ -138,8 +135,8 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] coords Coordinates of the points
*/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] );
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] );
/*!
@ -162,8 +159,8 @@ Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
* @param[in] data Variable data
*/
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data );
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data );
/*!
@ -190,8 +187,8 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] tri Coordinates of the points
*/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshname,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] );
void writeTriMesh( DBfile *fid, const std::string &meshname, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] );
/*!
@ -241,8 +238,7 @@ Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] subMeshTypes Type of each submesh
*/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string>& subMeshNames,
const std::vector<int>& subMeshTypes );
const std::vector<std::string> &subMeshNames, const std::vector<int> &subMeshTypes );
/*!
@ -257,12 +253,10 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
* @param[in] nvar Number of subvariables (used to determine suffix)
*/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string>& subVarNames,
const std::vector<int>& subVarTypes );
const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
}; // silo namespace
}; // namespace IO::silo
#endif
#include "IO/silo.hpp"

91
IO/silo.hpp
View File

@ -2,8 +2,8 @@
#define SILO_INTERFACE_HPP
#include "IO/silo.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "ProfilerApp.h"
@ -13,17 +13,29 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
* Helper functions *
****************************************************/
template<class TYPE> static constexpr int getType();
template<> constexpr int getType<double>() { return DB_DOUBLE; }
template<> constexpr int getType<float>() { return DB_FLOAT; }
template<> constexpr int getType<int>() { return DB_INT; }
template<class TYPE>
static constexpr int getType();
template<>
constexpr int getType<double>()
{
return DB_DOUBLE;
}
template<>
constexpr int getType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getType<int>()
{
return DB_INT;
}
template<class TYPE>
inline void copyData( Array<TYPE> &data, int type, const void *src )
{
@ -43,10 +55,23 @@ inline void copyData( Array<TYPE>& data, int type, const void *src )
/****************************************************
* Write/read an arbitrary vector *
****************************************************/
template<class TYPE> constexpr int getSiloType();
template<> constexpr int getSiloType<int>() { return DB_INT; }
template<> constexpr int getSiloType<float>() { return DB_FLOAT; }
template<> constexpr int getSiloType<double>() { return DB_DOUBLE; }
template<class TYPE>
constexpr int getSiloType();
template<>
constexpr int getSiloType<int>()
{
return DB_INT;
}
template<>
constexpr int getSiloType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getSiloType<double>()
{
return DB_DOUBLE;
}
template<class TYPE>
void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data )
{
@ -144,7 +169,11 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname,
z[N[2]] = range[5];
}
float *coords[] = { x, y, z };
int err = DBPutQuadmesh( fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
int err = DBPutQuadmesh(
fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
delete[] x;
delete[] y;
delete[] z;
ASSERT( err == 0 );
PROFILE_STOP( "writeUniformMesh", 2 );
}
@ -154,8 +183,9 @@ void writeUniformMesh( DBfile* fid, const std::string& meshname,
* Write a vector/tensor quad variable *
****************************************************/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type )
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type )
{
PROFILE_START( "writeUniformMeshVariable", 2 );
int nvars = 1, dims[NDIM] = { 1 };
@ -197,8 +227,8 @@ void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const s
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
ASSERT( err == 0 );
PROFILE_STOP( "writeUniformMeshVariable", 2 );
}
@ -227,8 +257,8 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
* Read/write a point mesh/variable to silo *
****************************************************/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] )
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] )
{
int err = DBPutPointmesh( fid, meshname.c_str(), ndim, coords, N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
@ -250,15 +280,16 @@ Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname )
return coords;
}
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data )
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data )
{
int N = data.size( 0 );
int nvars = data.size( 1 );
std::vector<const TYPE *> vars( nvars );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
int err = DBPutPointvar( fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
int err = DBPutPointvar(
fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
}
template<class TYPE>
@ -282,8 +313,8 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
* Read/write a triangle mesh *
****************************************************/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshName,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] )
void writeTriMesh( DBfile *fid, const std::string &meshName, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] )
{
auto zoneName = meshName + "_zones";
std::vector<int> nodelist( ( ndim_tri + 1 ) * N_tri );
@ -302,10 +333,10 @@ void writeTriMesh( DBfile* fid, const std::string& meshName,
ERROR( "Unknown shapetype" );
int shapesize = ndim_tri + 1;
int shapecnt = N_tri;
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(),
nodelist.size(), 0, 0, 0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N,
nodelist.size(), zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(), nodelist.size(), 0, 0,
0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N, nodelist.size(),
zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
}
template<class TYPE>
void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri )
@ -362,8 +393,8 @@ void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, data.size(0), nullptr, 0, getType<TYPE>(), vartype, nullptr );
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
data.size( 0 ), nullptr, 0, getType<TYPE>(), vartype, nullptr );
}
template<class TYPE>
Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
@ -382,7 +413,7 @@ Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname )
}
}; // silo namespace
}; // namespace IO::silo
#endif

1
README.titan
View File

@ -26,7 +26,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CFLAGS="-DCBUB" \
-D CXXFLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

3
StackTrace/ErrorHandlers.h
View File

@ -3,11 +3,10 @@
#include "StackTrace/StackTrace.h"
#include "common/MPI.h"
#include <functional>
#include "mpi.h"
namespace StackTrace
{

181
analysis/FreeEnergy.cpp Normal file
View File

@ -0,0 +1,181 @@
#include "analysis/FreeEnergy.h"
FreeEnergyAnalyzer::FreeEnergyAnalyzer(std::shared_ptr <Domain> dm):
Dm(dm)
{
Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0);
Phi.resize(Nx,Ny,Nz); Phi.fill(0);
Pressure.resize(Nx,Ny,Nz); Pressure.fill(0);
Rho.resize(Nx,Ny,Nz); Rho.fill(0);
Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field
Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0);
Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0);
SDs.resize(Nx,Ny,Nz); SDs.fill(0);
if (Dm->rank()==0){
bool WriteHeader=false;
TIMELOG = fopen("free.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("free.csv","a+");
if (WriteHeader)
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"timestep\n");
}
}
}
FreeEnergyAnalyzer::~FreeEnergyAnalyzer(){
if (Dm->rank()==0){
fclose(TIMELOG);
}
}
void FreeEnergyAnalyzer::SetParams(){
}
void FreeEnergyAnalyzer::Basic(ScaLBL_FreeLeeModel &LeeModel, int timestep){
int i,j,k;
if (Dm->rank()==0){
fprintf(TIMELOG,"%i ",timestep);
/*for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
}
*/
fprintf(TIMELOG,"\n");
fflush(TIMELOG);
}
/* else{
fprintf(TIMELOG,"%i ",timestep);
for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
}
fflush(TIMELOG);
} */
}
void FreeEnergyAnalyzer::WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep){
auto vis_db = input_db->getDatabase( "Visualization" );
char VisName[40];
std::vector<IO::MeshDataStruct> visData;
fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
IO::initialize("","silo","false");
// Create the MeshDataStruct
visData.resize(1);
visData[0].meshName = "domain";
visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
auto VisPhase = std::make_shared<IO::Variable>();
auto VisPressure = std::make_shared<IO::Variable>();
auto VisChemicalPotential = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
auto VyVar = std::make_shared<IO::Variable>();
auto VzVar = std::make_shared<IO::Variable>();
if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
VisPhase->name = "Phase";
VisPhase->type = IO::VariableType::VolumeVariable;
VisPhase->dim = 1;
VisPhase->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPhase);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
VisPressure->name = "Pressure";
VisPressure->type = IO::VariableType::VolumeVariable;
VisPressure->dim = 1;
VisPressure->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPressure);
VisChemicalPotential->name = "ChemicalPotential";
VisChemicalPotential->type = IO::VariableType::VolumeVariable;
VisChemicalPotential->dim = 1;
VisChemicalPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisChemicalPotential);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VxVar);
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VyVar);
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VzVar);
}
if (vis_db->getWithDefault<bool>( "save_phase", true )){
ASSERT(visData[0].vars[0]->name=="Phase");
LeeModel.getPhase(Phi);
Array<double>& PhaseData = visData[0].vars[0]->data;
fillData.copy(Phi,PhaseData);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
ASSERT(visData[0].vars[1]->name=="Pressure");
LeeModel.getPotential(Pressure, ChemicalPotential);
Array<double>& PressureData = visData[0].vars[1]->data;
fillData.copy(Pressure,PressureData);
ASSERT(visData[0].vars[2]->name=="ChemicalPotential");
Array<double>& ChemicalPotentialData = visData[0].vars[2]->data;
fillData.copy(ChemicalPotential,ChemicalPotentialData);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
ASSERT(visData[0].vars[3]->name=="Velocity_x");
ASSERT(visData[0].vars[4]->name=="Velocity_y");
ASSERT(visData[0].vars[5]->name=="Velocity_z");
LeeModel.getVelocity(Vel_x,Vel_y,Vel_z);
Array<double>& VelxData = visData[0].vars[3]->data;
Array<double>& VelyData = visData[0].vars[4]->data;
Array<double>& VelzData = visData[0].vars[5]->data;
fillData.copy(Vel_x,VelxData);
fillData.copy(Vel_y,VelyData);
fillData.copy(Vel_z,VelzData);
}
if (vis_db->getWithDefault<bool>( "write_silo", true ))
IO::writeData( timestep, visData, Dm->Comm );
/* if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
char CurrentIDFilename[40];
sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
Averages.AggregateLabels(CurrentIDFilename);
}
*/
}

54
analysis/FreeEnergy.h Normal file
View File

@ -0,0 +1,54 @@
/*
* averaging tools for electrochemistry
*/
#ifndef FreeEnergyAnalyzer_INC
#define FreeEnergyAnalyzer_INC
#include <vector>
#include "common/Domain.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "analysis/SubPhase.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
#include "models/FreeLeeModel.h"
class FreeEnergyAnalyzer{
public:
std::shared_ptr <Domain> Dm;
double Volume;
// input variables
double rho_n, rho_w;
double nu_n, nu_w;
double gamma_wn, beta;
double Fx, Fy, Fz;
//...........................................................................
int Nx,Ny,Nz;
DoubleArray Rho;
DoubleArray Phi;
DoubleArray ChemicalPotential;
DoubleArray Pressure;
DoubleArray Vel_x;
DoubleArray Vel_y;
DoubleArray Vel_z;
DoubleArray SDs;
FreeEnergyAnalyzer(std::shared_ptr <Domain> Dm);
~FreeEnergyAnalyzer();
void SetParams();
void Basic( ScaLBL_FreeLeeModel &LeeModel, int timestep);
void WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep);
private:
FILE *TIMELOG;
};
#endif

59
analysis/SubPhase.cpp
View File

@ -40,7 +40,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -50,7 +50,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
fprintf(SUBPHASE,"Vwc Awc Hwc Xwc "); // wc region
fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd "); // wd region
fprintf(SUBPHASE,"Vnc Anc Hnc Xnc "); // nc region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd regionin
fprintf(SUBPHASE,"Vi Ai Hi Xi "); // interface region
fprintf(SUBPHASE,"Vic Aic Hic Xic Nic\n"); // interface region
@ -65,7 +65,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
sprintf(LocalRankFilename,"%s%s","subphase.csv.",LocalRankString);
SUBPHASE = fopen(LocalRankFilename,"a+");
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -93,7 +93,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn wet\n");
}
}
}
@ -109,7 +109,7 @@ SubPhase::~SubPhase()
void SubPhase::Write(int timestep)
{
if (Dm->rank()==0){
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction_global);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
@ -125,7 +125,7 @@ void SubPhase::Write(int timestep)
fflush(SUBPHASE);
}
else{
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
@ -172,6 +172,21 @@ void SubPhase::Basic(){
double count_w = 0.0;
double count_n = 0.0;
/* compute the laplacian */
Dm->CommunicateMeshHalo(Phi);
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
// Compute all of the derivatives using finite differences
double fx = 0.5*(Phi(i+1,j,k) - Phi(i-1,j,k));
double fy = 0.5*(Phi(i,j+1,k) - Phi(i,j-1,k));
double fz = 0.5*(Phi(i,j,k+1) - Phi(i,j,k-1));
DelPhi(i,j,k) = sqrt(fx*fx+fy*fy+fz*fz);
}
}
}
Dm->CommunicateMeshHalo(DelPhi);
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
@ -184,6 +199,11 @@ void SubPhase::Basic(){
double phi = (nA-nB)/(nA+nB);
Phi(n) = phi;
}
if (Phi(n) != Phi(n)){
// check for NaN
Phi(n) = 0.0;
//printf("Nan at %i %i %i \n",i,j,k);
}
}
}
}
@ -229,6 +249,31 @@ void SubPhase::Basic(){
}
}
}
total_wetting_interaction = count_wetting_interaction = 0.0;
total_wetting_interaction_global = count_wetting_interaction_global=0.0;
for (k=kmin; k<kmax; k++){
for (j=jmin; j<Ny-1; j++){
for (i=imin; i<Nx-1; i++){
n = k*Nx*Ny + j*Nx + i;
// compute contribution of wetting terms (within two voxels of solid)
if ( Dm->id[n] > 0 && SDs(i,j,k) < 2.0 ){
count_wetting_interaction += 1.0;
total_wetting_interaction += DelPhi(i,j,k);
}
}
}
}
//printf("wetting interaction = %f, count = %f\n",total_wetting_interaction,count_wetting_interaction);
total_wetting_interaction_global=Dm->Comm.sumReduce( total_wetting_interaction);
count_wetting_interaction_global=Dm->Comm.sumReduce( count_wetting_interaction);
/* normalize wetting interactions <-- Don't do this if normalizing laplacian (use solid surface area)
if (count_wetting_interaction > 0.0)
total_wetting_interaction /= count_wetting_interaction;
if (count_wetting_interaction_global > 0.0)
total_wetting_interaction_global /= count_wetting_interaction_global;
*/
gwb.V=Dm->Comm.sumReduce( wb.V);
gnb.V=Dm->Comm.sumReduce( nb.V);
gwb.M=Dm->Comm.sumReduce( wb.M);
@ -303,7 +348,7 @@ void SubPhase::Basic(){
double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
double krw = h*h*nu_w*water_flow_rate / force_mag;
//printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p);
fprintf(TIMELOG,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p, total_wetting_interaction_global);
fflush(TIMELOG);
}
if (err==true){

8
analysis/SubPhase.h
View File

@ -68,12 +68,16 @@ public:
* b - bulk (total)
*/
// local entities
phase wc,wd,wb,nc,nd,nb;
phase wc,wd,wb,nc,nd,nb,solid;
interface iwn,iwnc;
// global entities
phase gwc,gwd,gwb,gnc,gnd,gnb;
phase gwc,gwd,gwb,gnc,gnd,gnb,gsolid;
interface giwn,giwnc;
/* fluid-solid wetting interaction */
double total_wetting_interaction, count_wetting_interaction;
double total_wetting_interaction_global, count_wetting_interaction_global;
//...........................................................................
int Nx,Ny,Nz;
IntArray PhaseID; // Phase ID array (solid=0, non-wetting=1, wetting=2)

3
analysis/morphology.cpp
View File

@ -542,7 +542,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
// Extract only the connected part of NWP
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
Dm->Comm.barrier();
@ -703,10 +702,12 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
if (rank == 0) printf(" delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT);
// Now adjust morph_delta
if (fabs(GrowthEstimate - GrowthPrevious) > 0.0) {
double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
GrowthPrevious = GrowthEstimate;
morph_delta_previous = morph_delta;
morph_delta += step_size;
}
if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0;
//MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25);

431
analysis/runAnalysis.cpp
View File

@ -15,10 +15,8 @@
AnalysisType &operator|=( AnalysisType &lhs, AnalysisType rhs )
{
lhs = static_cast<AnalysisType>(
static_cast<std::underlying_type<AnalysisType>::type>(lhs) |
static_cast<std::underlying_type<AnalysisType>::type>(rhs)
);
lhs = static_cast<AnalysisType>( static_cast<std::underlying_type<AnalysisType>::type>( lhs ) |
static_cast<std::underlying_type<AnalysisType>::type>( rhs ) );
return lhs;
}
bool matches( AnalysisType x, AnalysisType y )
@ -28,10 +26,11 @@ bool matches( AnalysisType x, AnalysisType y )
}
// Create a shared_ptr to an array of values
template<class TYPE>
void DeleteArray( const TYPE *p )
static inline std::shared_ptr<TYPE> make_shared_array( size_t N )
{
delete [] p;
return std::shared_ptr<TYPE>( new TYPE[N], []( const TYPE *p ) { delete[] p; } );
}
@ -39,9 +38,13 @@ void DeleteArray( const TYPE *p )
class WriteRestartWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
WriteRestartWorkItem( const char* filename_, std::shared_ptr<double> cDen_, std::shared_ptr<double> cfq_, int N_ ):
filename(filename_), cfq(cfq_), cDen(cDen_), N(N_) {}
virtual void run() {
WriteRestartWorkItem( const std::string &filename_, std::shared_ptr<double> cDen_,
std::shared_ptr<double> cfq_, int N_ )
: filename( filename_ ), cfq( cfq_ ), cDen( cDen_ ), N( N_ )
{
}
virtual void run()
{
PROFILE_START( "Save Checkpoint", 1 );
double value;
ofstream File( filename, ios::binary );
@ -51,7 +54,6 @@ public:
File.write( (char *) &value, sizeof( value ) );
value = cDen.get()[N + n];
File.write( (char *) &value, sizeof( value ) );
}
for ( int n = 0; n < N; n++ ) {
// Write the distributions
@ -63,43 +65,58 @@ public:
File.close();
PROFILE_STOP( "Save Checkpoint", 1 );
};
private:
WriteRestartWorkItem();
const char* filename;
const std::string filename;
std::shared_ptr<double> cfq, cDen;
// const DoubleArray& phase;
//const DoubleArray& dist;
const int N;
};
// Helper class to compute the blob ids
typedef std::shared_ptr<std::pair<int, IntArray>> BlobIDstruct;
typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
static const std::string id_map_filename = "lbpm_id_map.txt";
class BlobIdentificationWorkItem1 : public ThreadPool::WorkItemRet<void>
{
public:
BlobIdentificationWorkItem1( int timestep_, int Nx_, int Ny_, int Nz_, const RankInfoStruct& rank_info_,
std::shared_ptr<const DoubleArray> phase_, const DoubleArray& dist_,
BlobIDstruct last_id_, BlobIDstruct new_index_, BlobIDstruct new_id_, BlobIDList new_list_, runAnalysis::commWrapper&& comm_ ):
timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_),
phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_), comm(std::move(comm_))
BlobIdentificationWorkItem1( int timestep_, int Nx_, int Ny_, int Nz_,
const RankInfoStruct &rank_info_, std::shared_ptr<const DoubleArray> phase_,
const DoubleArray &dist_, BlobIDstruct last_id_, BlobIDstruct new_index_,
BlobIDstruct new_id_, BlobIDList new_list_, runAnalysis::commWrapper &&comm_ )
: timestep( timestep_ ),
Nx( Nx_ ),
Ny( Ny_ ),
Nz( Nz_ ),
rank_info( rank_info_ ),
phase( phase_ ),
dist( dist_ ),
last_id( last_id_ ),
new_index( new_index_ ),
new_id( new_id_ ),
new_list( new_list_ ),
comm( std::move( comm_ ) )
{
}
~BlobIdentificationWorkItem1() {}
virtual void run() {
virtual void run()
{
// Compute the global blob id and compare to the previous version
PROFILE_START( "Identify blobs", 1 );
double vF = 0.0;
double vS = -1.0; // one voxel buffer region around solid
IntArray &ids = new_index->second;
new_index->first = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,*phase,dist,vF,vS,ids,comm.comm);
new_index->first = ComputeGlobalBlobIDs(
Nx - 2, Ny - 2, Nz - 2, rank_info, *phase, dist, vF, vS, ids, comm.comm );
PROFILE_STOP( "Identify blobs", 1 );
}
private:
BlobIdentificationWorkItem1();
int timestep;
int Nx, Ny, Nz;
const RankInfoStruct& rank_info;
const RankInfoStruct rank_info;
std::shared_ptr<const DoubleArray> phase;
const DoubleArray &dist;
BlobIDstruct last_id, new_index, new_id;
@ -109,15 +126,27 @@ private:
class BlobIdentificationWorkItem2 : public ThreadPool::WorkItemRet<void>
{
public:
BlobIdentificationWorkItem2( int timestep_, int Nx_, int Ny_, int Nz_, const RankInfoStruct& rank_info_,
std::shared_ptr<const DoubleArray> phase_, const DoubleArray& dist_,
BlobIDstruct last_id_, BlobIDstruct new_index_, BlobIDstruct new_id_, BlobIDList new_list_ , runAnalysis::commWrapper&& comm_ ):
timestep(timestep_), Nx(Nx_), Ny(Ny_), Nz(Nz_), rank_info(rank_info_),
phase(phase_), dist(dist_), last_id(last_id_), new_index(new_index_), new_id(new_id_), new_list(new_list_), comm(std::move(comm_))
BlobIdentificationWorkItem2( int timestep_, int Nx_, int Ny_, int Nz_,
const RankInfoStruct &rank_info_, std::shared_ptr<const DoubleArray> phase_,
const DoubleArray &dist_, BlobIDstruct last_id_, BlobIDstruct new_index_,
BlobIDstruct new_id_, BlobIDList new_list_, runAnalysis::commWrapper &&comm_ )
: timestep( timestep_ ),
Nx( Nx_ ),
Ny( Ny_ ),
Nz( Nz_ ),
rank_info( rank_info_ ),
phase( phase_ ),
dist( dist_ ),
last_id( last_id_ ),
new_index( new_index_ ),
new_id( new_id_ ),
new_list( new_list_ ),
comm( std::move( comm_ ) )
{
}
~BlobIdentificationWorkItem2() {}
virtual void run() {
virtual void run()
{
// Compute the global blob id and compare to the previous version
PROFILE_START( "Identify blobs maps", 1 );
const IntArray &ids = new_index->second;
@ -140,11 +169,12 @@ public:
}
PROFILE_STOP( "Identify blobs maps", 1 );
}
private:
BlobIdentificationWorkItem2();
int timestep;
int Nx, Ny, Nz;
const RankInfoStruct& rank_info;
const RankInfoStruct rank_info;
std::shared_ptr<const DoubleArray> phase;
const DoubleArray &dist;
BlobIDstruct last_id, new_index, new_id;
@ -158,14 +188,23 @@ class WriteVisWorkItem: public ThreadPool::WorkItemRet<void>
{
public:
WriteVisWorkItem( int timestep_, std::vector<IO::MeshDataStruct> &visData_,
TwoPhase& Avgerages_, fillHalo<double>& fillData_, runAnalysis::commWrapper&& comm_ ):
timestep(timestep_), visData(visData_), Averages(Avgerages_), fillData(fillData_), comm(std::move(comm_))
TwoPhase &Avgerages_, std::array<int, 3> n_, RankInfoStruct rank_info_,
runAnalysis::commWrapper &&comm_ )
: timestep( timestep_ ),
visData( visData_ ),
Averages( Avgerages_ ),
n( std::move( n_ ) ),
rank_info( std::move( rank_info_ ) ),
comm( std::move( comm_ ) )
{
}
~WriteVisWorkItem() {}
virtual void run() {
virtual void run()
{
PROFILE_START( "Save Vis", 1 );
fillHalo<double> fillData( comm.comm, rank_info, n, { 1, 1, 1 }, 0, 1 );
ASSERT( visData[0].vars[0]->name == "phase" );
Array<double> &PhaseData = visData[0].vars[0]->data;
fillData.copy( Averages.SDn, PhaseData );
@ -195,12 +234,14 @@ public:
PROFILE_STOP( "Save Vis", 1 );
};
private:
WriteVisWorkItem();
int timestep;
std::array<int, 3> n;
RankInfoStruct rank_info;
std::vector<IO::MeshDataStruct> &visData;
TwoPhase &Averages;
fillHalo<double>& fillData;
runAnalysis::commWrapper comm;
};
@ -208,18 +249,28 @@ private:
class IOWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
IOWorkItem(int timestep_, std::shared_ptr<Database> input_db_, std::vector<IO::MeshDataStruct>& visData_,
SubPhase& Averages_, fillHalo<double>& fillData_, runAnalysis::commWrapper&& comm_ ):
timestep(timestep_), input_db(input_db_), visData(visData_), Averages(Averages_), fillData(fillData_), comm(std::move(comm_))
IOWorkItem( int timestep_, std::shared_ptr<Database> input_db_,
std::vector<IO::MeshDataStruct> &visData_, SubPhase &Averages_, std::array<int, 3> n_,
RankInfoStruct rank_info_, runAnalysis::commWrapper &&comm_ )
: timestep( timestep_ ),
input_db( input_db_ ),
visData( visData_ ),
Averages( Averages_ ),
n( std::move( n_ ) ),
rank_info( std::move( rank_info_ ) ),
comm( std::move( comm_ ) )
{
}
~IOWorkItem() {}
virtual void run() {
virtual void run()
{
PROFILE_START( "Save Vis", 1 );
auto color_db = input_db->getDatabase( "Color" );
auto vis_db = input_db->getDatabase( "Visualization" );
// int timestep = color_db->getWithDefault<int>( "timestep", 0 );
PROFILE_START("Save Vis",1);
fillHalo<double> fillData( comm.comm, rank_info, n, { 1, 1, 1 }, 0, 1 );
if ( vis_db->getWithDefault<bool>( "save_phase_field", true ) ) {
ASSERT( visData[0].vars[0]->name == "phase" );
@ -268,13 +319,15 @@ public:
PROFILE_STOP( "Save Vis", 1 );
};
private:
IOWorkItem();
int timestep;
std::array<int, 3> n;
RankInfoStruct rank_info;
std::shared_ptr<Database> input_db;
std::vector<IO::MeshDataStruct> &visData;
SubPhase &Averages;
fillHalo<double>& fillData;
runAnalysis::commWrapper comm;
};
@ -284,12 +337,19 @@ private:
class AnalysisWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
AnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_,
BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
type(type_), timestep(timestep_), Averages(Averages_),
blob_ids(ids), id_list(id_list_), beta(beta_) { }
AnalysisWorkItem( AnalysisType type_, int timestep_, TwoPhase &Averages_, BlobIDstruct ids,
BlobIDList id_list_, double beta_ )
: type( type_ ),
timestep( timestep_ ),
Averages( Averages_ ),
blob_ids( ids ),
id_list( id_list_ ),
beta( beta_ )
{
}
~AnalysisWorkItem() {}
virtual void run() {
virtual void run()
{
Averages.NumberComponents_NWP = blob_ids->first;
Averages.Label_NWP = blob_ids->second;
Averages.Label_NWP_map = *id_list;
@ -316,6 +376,7 @@ public:
PROFILE_STOP( "Compute dist", 1 );
}
}
private:
AnalysisWorkItem();
AnalysisType type;
@ -330,12 +391,19 @@ private:
class TCATWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
TCATWorkItem( AnalysisType type_, int timestep_, TwoPhase& Averages_,
BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
type(type_), timestep(timestep_), Averages(Averages_),
blob_ids(ids), id_list(id_list_), beta(beta_) { }
TCATWorkItem( AnalysisType type_, int timestep_, TwoPhase &Averages_, BlobIDstruct ids,
BlobIDList id_list_, double beta_ )
: type( type_ ),
timestep( timestep_ ),
Averages( Averages_ ),
blob_ids( ids ),
id_list( id_list_ ),
beta( beta_ )
{
}
~TCATWorkItem() {}
virtual void run() {
virtual void run()
{
Averages.NumberComponents_NWP = blob_ids->first;
Averages.Label_NWP = blob_ids->second;
Averages.Label_NWP_map = *id_list;
@ -358,6 +426,7 @@ public:
PROFILE_STOP( "Compute TCAT", 1 );
}
}
private:
TCATWorkItem();
AnalysisType type;
@ -373,11 +442,18 @@ class GanglionTrackingWorkItem: public ThreadPool::WorkItemRet<void>
{
public:
GanglionTrackingWorkItem( AnalysisType type_, int timestep_, TwoPhase &Averages_,
BlobIDstruct ids, BlobIDList id_list_, double beta_ ):
type(type_), timestep(timestep_), Averages(Averages_),
blob_ids(ids), id_list(id_list_), beta(beta_) { }
BlobIDstruct ids, BlobIDList id_list_, double beta_ )
: type( type_ ),
timestep( timestep_ ),
Averages( Averages_ ),
blob_ids( ids ),
id_list( id_list_ ),
beta( beta_ )
{
}
~GanglionTrackingWorkItem() {}
virtual void run() {
virtual void run()
{
Averages.NumberComponents_NWP = blob_ids->first;
Averages.Label_NWP = blob_ids->second;
Averages.Label_NWP_map = *id_list;
@ -400,6 +476,7 @@ public:
PROFILE_STOP( "Compute ganglion", 1 );
}
}
private:
GanglionTrackingWorkItem();
AnalysisType type;
@ -414,10 +491,13 @@ private:
class BasicWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
BasicWorkItem( AnalysisType type_, int timestep_, SubPhase& Averages_ ):
type(type_), timestep(timestep_), Averages(Averages_){ }
BasicWorkItem( AnalysisType type_, int timestep_, SubPhase &Averages_ )
: type( type_ ), timestep( timestep_ ), Averages( Averages_ )
{
}
~BasicWorkItem() {}
virtual void run() {
virtual void run()
{
if ( matches( type, AnalysisType::CopyPhaseIndicator ) ) {
// Averages.ColorToSignedDistance(beta,Averages.Phase,Averages.Phase_tplus);
@ -428,6 +508,7 @@ public:
PROFILE_STOP( "Compute basic averages", 1 );
}
}
private:
BasicWorkItem();
AnalysisType type;
@ -439,16 +520,20 @@ private:
class SubphaseWorkItem : public ThreadPool::WorkItemRet<void>
{
public:
SubphaseWorkItem( AnalysisType type_, int timestep_, SubPhase& Averages_ ):
type(type_), timestep(timestep_), Averages(Averages_){ }
SubphaseWorkItem( AnalysisType type_, int timestep_, SubPhase &Averages_ )
: type( type_ ), timestep( timestep_ ), Averages( Averages_ )
{
}
~SubphaseWorkItem() {}
virtual void run() {
virtual void run()
{
PROFILE_START( "Compute subphase", 1 );
Averages.Full();
Averages.Write( timestep );
PROFILE_STOP( "Compute subphase", 1 );
}
private:
SubphaseWorkItem();
AnalysisType type;
@ -458,20 +543,16 @@ private:
};
/******************************************************************
* MPI comm wrapper for use with analysis *
******************************************************************/
runAnalysis::commWrapper::commWrapper( int tag_, const Utilities::MPI& comm_, runAnalysis* analysis_ ):
comm(comm_),
tag(tag_),
analysis(analysis_)
runAnalysis::commWrapper::commWrapper(
int tag_, const Utilities::MPI &comm_, runAnalysis *analysis_ )
: comm( comm_ ), tag( tag_ ), analysis( analysis_ )
{
}
runAnalysis::commWrapper::commWrapper( commWrapper &&rhs ):
comm(rhs.comm),
tag(rhs.tag),
analysis(rhs.analysis)
runAnalysis::commWrapper::commWrapper( commWrapper &&rhs )
: comm( rhs.comm ), tag( rhs.tag ), analysis( rhs.analysis )
{
rhs.tag = -1;
}
@ -507,14 +588,10 @@ runAnalysis::commWrapper runAnalysis::getComm( )
/******************************************************************
* Constructor/Destructors *
******************************************************************/
runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm,
std::shared_ptr<Domain> Dm,
int Np,
bool Regular,
IntArray Map ):
d_Np( Np ),
runAnalysis::runAnalysis( std::shared_ptr<Database> input_db, const RankInfoStruct &rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> Dm, int Np,
bool Regular, IntArray Map )
: d_Np( Np ),
d_regular( Regular ),
d_rank_info( rank_info ),
d_Map( Map ),
@ -568,7 +645,141 @@ runAnalysis::runAnalysis( std::shared_ptr<Database> input_db,
d_meshData.resize( 1 );
d_meshData[0].meshName = "domain";
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>( d_rank_info,d_n[0],d_n[1],d_n[2],Dm->Lx,Dm->Ly,Dm->Lz );
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>(
d_rank_info, d_n[0], d_n[1], d_n[2], Dm->Lx, Dm->Ly, Dm->Lz );
auto PhaseVar = std::make_shared<IO::Variable>();
auto PressVar = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
auto VyVar = std::make_shared<IO::Variable>();
auto VzVar = std::make_shared<IO::Variable>();
auto SignDistVar = std::make_shared<IO::Variable>();
auto BlobIDVar = std::make_shared<IO::Variable>();
if ( vis_db->getWithDefault<bool>( "save_phase_field", true ) ) {
PhaseVar->name = "phase";
PhaseVar->type = IO::VariableType::VolumeVariable;
PhaseVar->dim = 1;
PhaseVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( PhaseVar );
}
if ( vis_db->getWithDefault<bool>( "save_pressure", false ) ) {
PressVar->name = "Pressure";
PressVar->type = IO::VariableType::VolumeVariable;
PressVar->dim = 1;
PressVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( PressVar );
}
if ( vis_db->getWithDefault<bool>( "save_velocity", false ) ) {
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( VxVar );
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( VyVar );
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( VzVar );
}
if ( vis_db->getWithDefault<bool>( "save_distance", false ) ) {
SignDistVar->name = "SignDist";
SignDistVar->type = IO::VariableType::VolumeVariable;
SignDistVar->dim = 1;
SignDistVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( SignDistVar );
}
if ( vis_db->getWithDefault<bool>( "save_connected_components", false ) ) {
BlobIDVar->name = "BlobID";
BlobIDVar->type = IO::VariableType::VolumeVariable;
BlobIDVar->dim = 1;
BlobIDVar->data.resize( d_n[0], d_n[1], d_n[2] );
d_meshData[0].vars.push_back( BlobIDVar );
}
// Initialize the comms
for ( int i = 0; i < 1024; i++ )
d_comm_used[i] = false;
// Initialize the threads
int N_threads = db->getWithDefault<int>( "N_threads", 4 );
auto method = db->getWithDefault<std::string>( "load_balance", "default" );
createThreads( method, N_threads );
}
runAnalysis::runAnalysis( ScaLBL_ColorModel &ColorModel)
/* std::shared_ptr<Database> input_db, const RankInfoStruct &rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> Dm, int Np,
bool Regular, IntArray Map )
: d_Np( Np ),
d_regular( Regular ),
d_rank_info( rank_info ),
d_Map( Map ),
d_comm( Dm->Comm.dup() ),
d_ScaLBL_Comm( ScaLBL_Comm )*/
{
d_comm = ColorModel.Dm->Comm.dup();
d_Np = ColorModel.Np;
bool Regular = false;
auto input_db = ColorModel.db;
auto db = input_db->getDatabase( "Analysis" );
auto vis_db = input_db->getDatabase( "Visualization" );
// Ids of work items to use for dependencies
ThreadPool::thread_id_t d_wait_blobID;
ThreadPool::thread_id_t d_wait_analysis;
ThreadPool::thread_id_t d_wait_vis;
ThreadPool::thread_id_t d_wait_restart;
ThreadPool::thread_id_t d_wait_subphase;
char rankString[20];
sprintf( rankString, "%05d", ColorModel.Dm->rank() );
d_n[0] = ColorModel.Dm->Nx - 2;
d_n[1] = ColorModel.Dm->Ny - 2;
d_n[2] = ColorModel.Dm->Nz - 2;
d_N[0] = ColorModel.Dm->Nx;
d_N[1] = ColorModel.Dm->Ny;
d_N[2] = ColorModel.Dm->Nz;
d_restart_interval = db->getScalar<int>( "restart_interval" );
d_analysis_interval = db->getScalar<int>( "analysis_interval" );
d_subphase_analysis_interval = INT_MAX;
d_visualization_interval = INT_MAX;
d_blobid_interval = INT_MAX;
if ( db->keyExists( "blobid_interval" ) ) {
d_blobid_interval = db->getScalar<int>( "blobid_interval" );
}
if ( db->keyExists( "visualization_interval" ) ) {
d_visualization_interval = db->getScalar<int>( "visualization_interval" );
}
if ( db->keyExists( "subphase_analysis_interval" ) ) {
d_subphase_analysis_interval = db->getScalar<int>( "subphase_analysis_interval" );
}
auto restart_file = db->getScalar<std::string>( "restart_file" );
d_restartFile = restart_file + "." + rankString;
d_rank = d_comm.getRank();
writeIDMap( ID_map_struct(), 0, id_map_filename );
// Initialize IO for silo
IO::initialize( "", "silo", "false" );
// Create the MeshDataStruct
d_meshData.resize( 1 );
d_meshData[0].meshName = "domain";
d_meshData[0].mesh = std::make_shared<IO::DomainMesh>(
d_rank_info, d_n[0], d_n[1], d_n[2], ColorModel.Dm->Lx, ColorModel.Dm->Ly, ColorModel.Dm->Lz );
auto PhaseVar = std::make_shared<IO::Variable>();
auto PressVar = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
@ -678,7 +889,8 @@ void runAnalysis::createThreads( const std::string& method, int N_threads )
// Check if we have thread support
auto thread_support = Utilities::MPI::queryThreadSupport();
if ( thread_support != Utilities::MPI::ThreadSupport::MULTIPLE && N_threads > 0 )
std::cerr << "Warning: Failed to start MPI with necessary thread support, errors may occur\n";
std::cerr
<< "Warning: Failed to start MPI with necessary thread support, errors may occur\n";
// Create the threads
const auto cores = d_tpool.getProcessAffinity();
if ( N_threads == 0 ) {
@ -758,12 +970,11 @@ AnalysisType runAnalysis::computeAnalysisType( int timestep )
}
/******************************************************************
* Run the analysis *
******************************************************************/
void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase& Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den)
void runAnalysis::run( int timestep, std::shared_ptr<Database> input_db, TwoPhase &Averages,
const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den )
{
int N = d_N[0] * d_N[1] * d_N[2];
NULL_USE( N );
@ -794,7 +1005,7 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
matches(type,AnalysisType::CopySimState) ||
matches(type,AnalysisType::IdentifyBlobs) )
{
phase = std::shared_ptr<DoubleArray>(new DoubleArray(d_N[0],d_N[1],d_N[2]));
phase = std::make_shared<DoubleArray>(d_N[0],d_N[1],d_N[2]);
//ScaLBL_CopyToHost(phase->data(),Phi,N*sizeof(double));
// try 2 d_ScaLBL_Comm.RegulLayout(d_Map,Phi,Averages.Phase);
// memcpy(Averages.Phase.data(),phase->data(),N*sizeof(double));
@ -862,8 +1073,8 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
// if ( matches(type,AnalysisType::CreateRestart) ) {
if ( timestep % d_restart_interval == 0 ) {
// Copy restart data to the CPU
cDen = std::shared_ptr<double>(new double[2*d_Np],DeleteArray<double>);
cfq = std::shared_ptr<double>(new double[19*d_Np],DeleteArray<double>);
cDen = make_shared_array<double>( 2 * d_Np );
cfq = make_shared_array<double>( 19 * d_Np );
ScaLBL_CopyToHost( cfq.get(), fq, 19 * d_Np * sizeof( double ) );
ScaLBL_CopyToHost( cDen.get(), Den, 2 * d_Np * sizeof( double ) );
}
@ -871,15 +1082,15 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
// Spawn threads to do blob identification work
if ( matches( type, AnalysisType::IdentifyBlobs ) ) {
phase = std::shared_ptr<DoubleArray>(new DoubleArray(d_N[0],d_N[1],d_N[2]));
phase = std::make_shared<DoubleArray>( d_N[0], d_N[1], d_N[2] );
if ( d_regular )
d_ScaLBL_Comm->RegularLayout( d_Map, Phi, *phase );
else
ScaLBL_CopyToHost( phase->data(), Phi, N * sizeof( double ) );
BlobIDstruct new_index(new std::pair<int,IntArray>(0,IntArray()));
BlobIDstruct new_ids(new std::pair<int,IntArray>(0,IntArray()));
BlobIDList new_list(new std::vector<BlobIDType>());
auto new_index = std::make_shared<std::pair<int, IntArray>>( 0, IntArray() );
auto new_ids = std::make_shared<std::pair<int, IntArray>>( 0, IntArray() );
auto new_list = std::make_shared<std::vector<BlobIDType>>();
auto work1 = new BlobIdentificationWorkItem1( timestep, d_N[0], d_N[1], d_N[2], d_rank_info,
phase, Averages.SDs, d_last_ids, new_index, new_ids, new_list, getComm() );
auto work2 = new BlobIdentificationWorkItem2( timestep, d_N[0], d_N[1], d_N[2], d_rank_info,
@ -896,7 +1107,8 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
// if (timestep%d_restart_interval==0){
// if ( matches(type,AnalysisType::ComputeAverages) ) {
if ( timestep % d_analysis_interval == 0 ) {
auto work = new AnalysisWorkItem(type,timestep,Averages,d_last_index,d_last_id_map,d_beta);
auto work =
new AnalysisWorkItem( type, timestep, Averages, d_last_index, d_last_id_map, d_beta );
work->add_dependency( d_wait_blobID );
work->add_dependency( d_wait_analysis );
work->add_dependency( d_wait_vis ); // Make sure we are done using analysis before modifying
@ -923,9 +1135,8 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
// if ( matches(type,AnalysisType::CreateRestart) ) {
if ( timestep % d_restart_interval == 0 ) {
// Write the vis files
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work = new WriteVisWorkItem( timestep, d_meshData, Averages, fillData, std::move( comm ) );
auto work =
new WriteVisWorkItem( timestep, d_meshData, Averages, d_n, d_rank_info, getComm() );
work->add_dependency( d_wait_blobID );
work->add_dependency( d_wait_analysis );
work->add_dependency( d_wait_vis );
@ -938,8 +1149,14 @@ void runAnalysis::run(int timestep, std::shared_ptr<Database> input_db, TwoPhase
/******************************************************************
* Run the analysis *
******************************************************************/
void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
void runAnalysis::basic( int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages,
const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den )
{
int Nx = d_N[0];
int Ny = d_N[1];
int Nz = d_N[2];
int N = Nx * Ny * Nz;
NULL_USE( N );
// Check which analysis steps we need to perform
auto color_db = input_db->getDatabase( "Color" );
auto vis_db = input_db->getDatabase( "Visualization" );
@ -973,6 +1190,10 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
PROFILE_START( "Copy-Wait", 1 );
PROFILE_STOP( "Copy-Wait", 1 );
PROFILE_START( "Copy-State", 1 );
/*if (d_regular)
d_ScaLBL_Comm->RegularLayout(d_Map,Phi,Averages.Phi);
else */
ScaLBL_CopyToHost( Averages.Phi.data(), Phi, N * sizeof( double ) );
// copy other variables
d_ScaLBL_Comm->RegularLayout( d_Map, Pressure, Averages.Pressure );
d_ScaLBL_Comm->RegularLayout( d_Map, &Den[0], Averages.Rho_n );
@ -989,7 +1210,8 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
// if ( matches(type,AnalysisType::ComputeAverages) ) {
if ( timestep % d_analysis_interval == 0 ) {
auto work = new BasicWorkItem( type, timestep, Averages );
work->add_dependency(d_wait_subphase); // Make sure we are done using analysis before modifying
work->add_dependency(
d_wait_subphase ); // Make sure we are done using analysis before modifying
work->add_dependency( d_wait_analysis );
work->add_dependency( d_wait_vis );
d_wait_analysis = d_tpool.add_work( work );
@ -997,7 +1219,8 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
if ( timestep % d_subphase_analysis_interval == 0 ) {
auto work = new SubphaseWorkItem( type, timestep, Averages );
work->add_dependency(d_wait_subphase); // Make sure we are done using analysis before modifying
work->add_dependency(
d_wait_subphase ); // Make sure we are done using analysis before modifying
work->add_dependency( d_wait_analysis );
work->add_dependency( d_wait_vis );
d_wait_subphase = d_tpool.add_work( work );
@ -1006,8 +1229,8 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
if ( timestep % d_restart_interval == 0 ) {
std::shared_ptr<double> cfq, cDen;
// Copy restart data to the CPU
cDen = std::shared_ptr<double>(new double[2*d_Np],DeleteArray<double>);
cfq = std::shared_ptr<double>(new double[19*d_Np],DeleteArray<double>);
cDen = make_shared_array<double>( 2 * d_Np );
cfq = make_shared_array<double>( 19 * d_Np );
ScaLBL_CopyToHost( cfq.get(), fq, 19 * d_Np * sizeof( double ) );
ScaLBL_CopyToHost( cDen.get(), Den, 2 * d_Np * sizeof( double ) );
@ -1018,20 +1241,17 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
std::ofstream OutStream( "Restart.db" );
input_db->print( OutStream, "" );
OutStream.close();
}
// Write the restart file (using a seperate thread)
auto work1 = new WriteRestartWorkItem( d_restartFile.c_str(), cDen, cfq, d_Np );
work1->add_dependency( d_wait_restart );
d_wait_restart = d_tpool.add_work( work1 );
}
if ( timestep % d_visualization_interval == 0 ) {
// Write the vis files
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work = new IOWorkItem( timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) );
auto work =
new IOWorkItem( timestep, input_db, d_meshData, Averages, d_n, d_rank_info, getComm() );
work->add_dependency( d_wait_analysis );
work->add_dependency( d_wait_subphase );
work->add_dependency( d_wait_vis );
@ -1041,7 +1261,9 @@ void runAnalysis::basic(int timestep, std::shared_ptr<Database> input_db, SubPha
PROFILE_STOP( "basic" );
}
void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den)
void runAnalysis::WriteVisData( int timestep, std::shared_ptr<Database> input_db,
SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq,
double *Den )
{
auto color_db = input_db->getDatabase( "Color" );
auto vis_db = input_db->getDatabase( "Visualization" );
@ -1064,9 +1286,8 @@ void runAnalysis::WriteVisData(int timestep, std::shared_ptr<Database> input_db,
PROFILE_START( "write vis", 1 );
// if (Averages.WriteVis == true){
commWrapper comm = getComm();
fillHalo<double> fillData( comm.comm, d_rank_info, d_n, {1,1,1}, 0, 1 );
auto work2 = new IOWorkItem(timestep, input_db, d_meshData, Averages, fillData, std::move( comm ) );
auto work2 =
new IOWorkItem( timestep, input_db, d_meshData, Averages, d_n, d_rank_info, getComm() );
work2->add_dependency( d_wait_vis );
d_wait_vis = d_tpool.add_work( work2 );

48
analysis/runAnalysis.h
View File

@ -1,31 +1,39 @@
#ifndef RunAnalysis_H_INC
#define RunAnalysis_H_INC
#include "analysis/analysis.h"
#include "analysis/TwoPhase.h"
#include "analysis/SubPhase.h"
#include "analysis/TwoPhase.h"
#include "analysis/analysis.h"
#include "common/Communication.h"
#include "common/ScaLBL.h"
#include "threadpool/thread_pool.h"
#include "models/ColorModel.h"
#include <limits.h>
typedef std::shared_ptr<std::pair<int,IntArray>> BlobIDstruct;
typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
// Types of analysis
enum class AnalysisType : uint64_t { AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02,
CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20, ComputeSubphase=0x40 };
enum class AnalysisType : uint64_t {
AnalyzeNone = 0,
IdentifyBlobs = 0x01,
CopyPhaseIndicator = 0x02,
CopySimState = 0x04,
ComputeAverages = 0x08,
CreateRestart = 0x10,
WriteVis = 0x20,
ComputeSubphase = 0x40
};
//! Class to run the analysis in multiple threads
class runAnalysis
{
public:
//! Constructor
runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct &rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool Regular, IntArray Map );
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> dm, int Np,
bool Regular, IntArray Map );
runAnalysis( ScaLBL_ColorModel &ColorModel);
//! Destructor
~runAnalysis();
@ -34,8 +42,10 @@ public:
void run( int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData(int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den);
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData( int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages,
const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
//! Finish all active analysis
void finish();
@ -44,7 +54,8 @@ public:
* \brief Set the affinities
* \details This function will create the analysis threads and set the affinity
* of this thread and all analysis threads. If MPI_THREAD_MULTIPLE is not
* enabled, the analysis threads will be disabled and the analysis will run in the current thread.
* enabled, the analysis threads will be disabled and the analysis will run in the current
* thread.
* @param[in] method Method used to control the affinities:
* none - Don't use threads (runs all analysis in the current thread)
* default - Create the specified number of threads, but don't load balance
@ -57,14 +68,12 @@ public:
private:
runAnalysis();
// Determine the analysis to perform
AnalysisType computeAnalysisType( int timestep );
public:
class commWrapper
{
public:
@ -83,9 +92,8 @@ public:
commWrapper getComm();
private:
std::array<int, 3> d_n; // Number of local cells
std::array<int,3> d_N; // NNumber of local cells with ghosts
std::array<int, 3> d_N; // Number of local cells with ghosts
int d_Np;
int d_rank;
int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
@ -95,9 +103,9 @@ private:
ThreadPool d_tpool;
RankInfoStruct d_rank_info;
IntArray d_Map;
BlobIDstruct d_last_ids;
BlobIDstruct d_last_index;
BlobIDList d_last_id_map;
std::shared_ptr<std::pair<int, IntArray>> d_last_ids;
std::shared_ptr<std::pair<int, IntArray>> d_last_index;
std::shared_ptr<std::vector<BlobIDType>> d_last_id_map;
std::vector<IO::MeshDataStruct> d_meshData;
std::string d_restartFile;
Utilities::MPI d_comm;
@ -114,8 +122,6 @@ private:
// Friends
friend commWrapper::~commWrapper();
};
#endif

362
cmake/FindMPI.cmake
View File

@ -1,362 +0,0 @@
# - Message Passing Interface (MPI) module.
#
# The Message Passing Interface (MPI) is a library used to write
# high-performance parallel applications that use message passing, and
# is typically deployed on a cluster. MPI is a standard interface
# (defined by the MPI forum) for which many implementations are
# available. All of these implementations have somewhat different
# compilation approaches (different include paths, libraries to link
# against, etc.), and this module tries to smooth out those differences.
#
# This module will set the following variables:
# MPI_FOUND TRUE if we have found MPI
# MPI_COMPILE_FLAGS Compilation flags for MPI programs
# MPI_INCLUDE_PATH Include path(s) for MPI header
# MPI_LINK_FLAGS Linking flags for MPI programs
# MPI_LIBRARY First MPI library to link against (cached)
# MPI_EXTRA_LIBRARY Extra MPI libraries to link against (cached)
# MPI_LIBRARIES All libraries to link MPI programs against
# MPIEXEC Executable for running MPI programs
# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving it the
# number of processors to run on
# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly before the
# executable to run.
# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after all other flags.
#
# This module will attempt to auto-detect these settings, first by
# looking for a MPI compiler, which many MPI implementations provide
# as a pass-through to the native compiler to simplify the compilation
# of MPI programs. The MPI compiler is stored in the cache variable
# MPI_COMPILER, and will attempt to look for commonly-named drivers
# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
# recognized, it will be used to set all of the module variables. To
# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
# the CMake cache.
#
# If no compiler driver is found or the compiler driver is not
# recognized, this module will then search for common include paths
# and library names to try to detect MPI.
#
# If CMake initially finds a different MPI than was intended, and you
# want to use the MPI compiler auto-detection for a different MPI
# implementation, set MPI_COMPILER to the MPI compiler driver you want
# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
# will run again with the newly-specified MPI_COMPILER.
#
# When using MPIEXEC to execute MPI applications, you should typically
# use all of the MPIEXEC flags as follows:
# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
# ${MPIEXEC_POSTFLAGS} ARGS
# where PROCS is the number of processors on which to execute the program,
# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
# MPI program.
#=============================================================================
# Copyright 2001-2009 Kitware, Inc.
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
# This module is maintained by David Partyka <dave.partyka@kitware.com>.
# A set of directories to search through in addition to the standard system paths
# that find_program will search through.
# Microsoft HPC SDK is automatically added to the system path
# Argonne National Labs MPICH2 sets a registry key that we can use.
set(_MPI_PACKAGE_DIR
mpi
mpich
openmpi
lib/mpi
lib/mpich
lib/openmpi
"MPICH/SDK"
"Microsoft Compute Cluster Pack"
"Microsoft HPC Pack 2008 R2"
)
set(_MPI_PREFIX_PATH)
if(WIN32)
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
endif()
foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
endif()
endforeach(MpiPackageDir)
endforeach(SystemPrefixDir)
# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
find_program(MPIEXEC
NAMES mpiexec mpirun lamexec
PATHS ${_MPI_PREFIX_PATH}
PATH_SUFFIXES bin
DOC "Executable for running MPI programs."
)
# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
# If there is an mpi compiler find it and interogate (farther below) it for the include
# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
find_program(MPI_COMPILER
NAMES mpic++ mpicxx mpiCC mpicc
HINTS "${_MPI_BASE_DIR}"
PATH_SUFFIXES bin
DOC "MPI compiler. Used only to detect MPI compilation flags.")
mark_as_advanced(MPI_COMPILER)
set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILER)
# Check whether the -showme:compile option works. This indicates
# that we have either Open MPI or a newer version of LAM-MPI, and
# implies that -showme:link will also work.
# Note that Windows distros do not have an mpi compiler to interogate.
exec_program(${MPI_COMPILER}
ARGS -showme:compile
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
if (MPI_COMPILER_RETURN EQUAL 0)
# If we appear to have -showme:compile, then we should also have
# -showme:link. Try it.
exec_program(${MPI_COMPILER}
ARGS -showme:link
OUTPUT_VARIABLE MPI_LINK_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
# Note that we probably have -showme:incdirs and -showme:libdirs
# as well.
set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# Older versions of LAM-MPI have "-showme". Try it.
exec_program(${MPI_COMPILER}
ARGS -showme
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# MPICH uses "-show". Try it.
exec_program(${MPI_COMPILER}
ARGS -show
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# We have our command lines, but we might need to copy
# MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
if (NOT MPI_LINK_CMDLINE)
SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
endif (NOT MPI_LINK_CMDLINE)
else (MPI_COMPILER_RETURN EQUAL 0)
message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
endif (MPI_COMPILER_RETURN EQUAL 0)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILE_CMDLINE)
# Extract compile flags from the compile command line.
string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
set(MPI_COMPILE_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
if (MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
else(MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK ${FLAG})
endif(MPI_COMPILE_FLAGS_WORK)
endforeach(FLAG)
# Extract include paths from compile command line
string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
set(MPI_INCLUDE_PATH_WORK)
foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
string(REGEX REPLACE "//" "/" IPATH ${IPATH})
list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
endforeach(IPATH)
if (NOT MPI_INCLUDE_PATH_WORK)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any include paths on it,
# but we may have -showme:incdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:incdirs
OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_INCLUDE_PATH_WORK)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_INCLUDE_PATH_WORK)
if (NOT MPI_INCLUDE_PATH_WORK)
# If all else fails, just search for mpi.h in the normal include
# paths.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include
)
set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
endif (NOT MPI_INCLUDE_PATH_WORK)
# Extract linker paths from the link command line
string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_PATH)
foreach(LPATH ${MPI_ALL_LINK_PATHS})
string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
string(REGEX REPLACE "//" "/" LPATH ${LPATH})
list(APPEND MPI_LINK_PATH ${LPATH})
endforeach(LPATH)
if (NOT MPI_LINK_PATH)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any linking paths on it,
# but we may have -showme:libdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:libdirs
OUTPUT_VARIABLE MPI_LINK_PATH
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_LINK_PATH)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_LINK_PATH)
# Extract linker flags from the link command line
string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_LINK_FLAGS})
if (MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
else(MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK ${FLAG})
endif(MPI_LINK_FLAGS_WORK)
endforeach(FLAG)
if ( MPI_LINK_FLAGS_WORK )
string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
endif ()
# Extract the set of libraries to link against from the link command
# line
string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
# Determine full path names for all of the libraries that one needs
# to link against in an MPI program
set(MPI_LIBRARIES)
foreach(LIB ${MPI_LIBNAMES})
string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
if (MPI_LIB)
list(APPEND MPI_LIBRARIES ${MPI_LIB})
elseif (NOT MPI_FIND_QUIETLY)
message(WARNING "Unable to find MPI library ${LIB}")
endif ()
endforeach(LIB)
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
# Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
# MPI_EXTRA_LIBRARY.
list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
else (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
endif (MPI_NUMLIBS GREATER 1)
# Set up all of the appropriate cache entries
set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
else (MPI_COMPILE_CMDLINE)
# No MPI compiler to interogate so attempt to find everything with find functions.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include Inc
)
# Decide between 32-bit and 64-bit libraries for Microsoft's MPI
if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
set(MS_MPI_ARCH_DIR amd64)
else()
set(MS_MPI_ARCH_DIR i386)
endif()
find_library(MPI_LIBRARY
NAMES mpi mpich msmpi
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
)
find_library(MPI_EXTRA_LIBRARY
NAMES mpi++
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib
DOC "Extra MPI libraries to link against.")
set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Set up extra variables to conform to
if (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
else (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY})
endif (MPI_EXTRA_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND TRUE)
else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND FALSE)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
# handle the QUIETLY and REQUIRED arguments
#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
MPI_EXTRA_LIBRARY)
# unset to cleanup namespace
unset(_MPI_PACKAGE_DIR)
unset(_MPI_PREFIX_PATH)
unset(_MPI_BASE_DIR)

39
cmake/Find_TIMER.cmake
View File

@ -4,7 +4,7 @@
# CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR )
# This function assumes that USE_TIMER is set to indicate if the timer should be used
# If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer
# If USE_TIMER is not set we will create a summy timer that does nothing.
# If USE_TIMER is not set we will create a dummy timer that does nothing.
# The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default.
# The input argument NULL_TIMER_DIR specifies the location to install the dummy timer.
# If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used.
@ -13,7 +13,7 @@
# TIMER_CXXFLAGS - C++ flags for the timer library
# TIMER_LDFLAGS - Linker flags to link the timer library
# TIMER_LDLIBS - Linker libraries to link the timer library
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR QUIET )
# Determine if we want to use the timer utility
CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} )
SET( TIMER_INCLUDE )
@ -33,20 +33,23 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FIND_LIBRARY( TIMER_LIBS NAMES timerutility PATHS ${TIMER_DIRECTORY}/lib NO_DEFAULT_PATH )
SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include )
SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" )
SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib )
SET( TIMER_LDLIBS -ltimerutility )
SET( TIMER_LDFLAGS )
SET( TIMER_LDLIBS "${TIMER_LIBS}" )
ELSE()
MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported. Use -D TIMER_DIRECTORY=" )
ENDIF()
SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
ADD_DEFINITIONS( -DUSE_TIMER )
MESSAGE( "Using timer utility" )
MESSAGE( " TIMER_LIBRARIES = ${TIMER_LIBS}" )
IF ( NOT QUIET )
MESSAGE( STATUS "Using timer utility" )
MESSAGE( STATUS " TIMER_LIBRARIES = ${TIMER_LIBS}" )
ENDIF()
ELSE()
IF ( "${NULL_TIMER_DIR}" STREQUAL "" )
SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" )
ENDIF()
# Write ProfilerApp.h
FILE(WRITE "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...) do {} while(0)\n" )
@ -61,9 +64,25 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" )
# Write MemoryApp.h
FILE(WRITE "${NULL_TIMER_DIR}/MemoryApp.h" "#include <cstring>\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "class MemoryApp final {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "public:\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " struct MemoryStats {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " size_t bytes_new, bytes_delete, N_new, N_delete, tot_bytes_used, system_memory, stack_used, stack_size;\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " MemoryStats() { memset(this,0,sizeof(MemoryStats)); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " };\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline void print( std::ostream& ) {}\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getTotalMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getSystemMemory() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline MemoryStats getMemoryStats() { return MemoryStats(); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "};\n" )
SET( TIMER_INCLUDE "${NULL_TIMER_DIR}" )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
MESSAGE( "Disabling timer utility" )
IF ( NOT QUIET )
MESSAGE( STATUS "Disabling timer utility" )
ENDIF()
ENDIF()
SET( TIMER_INCLUDE "${TIMER_INCLUDE}" PARENT_SCOPE )
SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE )
@ -88,12 +107,12 @@ MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ${FLAG} STREQUAL "" )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "FALSE" ) OR ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
SET( ${FLAG} 0 )
ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "TRUE" ) OR ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
SET( ${FLAG} 1 )
ELSE()
MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
MESSAGE( FATAL_ERROR "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
ENDIF ()
ENDMACRO()

3
cmake/ctest_script.cmake
View File

@ -32,7 +32,6 @@ SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} )
SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} )
SET( LDLIBS $ENV{LDLIBS} )
SET( LDFLAGS $ENV{LDFLAGS} )
SET( MPI_COMPILER $ENV{MPI_COMPILER} )
SET( MPI_DIRECTORY $ENV{MPI_DIRECTORY} )
SET( MPI_INCLUDE $ENV{MPI_INCLUDE} )
SET( MPI_LINK_FLAGS $ENV{MPI_LINK_FLAGS} )
@ -198,7 +197,7 @@ SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLA
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
IF ( USE_MPI )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}")
IF ( NOT USE_VALGRIND )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true")
ENDIF()

124
cmake/libraries.cmake
View File

@ -43,91 +43,59 @@ MACRO( CONFIGURE_MPI )
# Determine if we want to use MPI
CHECK_ENABLE_FLAG( USE_MPI 1 )
IF ( USE_MPI )
# Check if we specified the MPI directory
IF ( MPI_DIRECTORY )
# Check the provided MPI directory for include files
VERIFY_PATH( "${MPI_DIRECTORY}" )
IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" )
ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" )
ELSE()
MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" )
ENDIF ()
INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
# Set MPI libraries
IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} )
ENDIF()
# Set the mpi executable
MESSAGE( "Configuring MPI" )
IF ( MPIEXEC )
# User specified the MPI command directly, use as is
ELSEIF ( MPIEXEC_CMD )
# User specified the name of the MPI executable
SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
IF ( NOT EXISTS ${MPIEXEC} )
MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
SET( MPIEXEC_EXECUTABLE ${MPIEXEC} )
ENDIF()
IF ( NOT MPI_SKIP_SEARCH )
FIND_PACKAGE( MPI )
ELSE()
# Search for the MPI executable in the current directory
FIND_PROGRAM( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH )
IF ( NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
# Write mpi test
SET( MPI_TEST_SRC "${CMAKE_CURRENT_BINARY_DIR}/test_mpi.cpp" )
FILE(WRITE ${MPI_TEST_SRC} "#include <mpi.h>\n" )
FILE(APPEND ${MPI_TEST_SRC} "int main(int argc, char** argv) {\n" )
FILE(APPEND ${MPI_TEST_SRC} " MPI_Init(NULL, NULL);\n")
FILE(APPEND ${MPI_TEST_SRC} " MPI_Finalize();\n" )
FILE(APPEND ${MPI_TEST_SRC} "}\n" )
# Test the compile
IF ( CMAKE_CXX_COMPILER )
SET( TMP_FLAGS -DINCLUDE_DIRECTORIES=${MPI_CXX_INCLUDE_PATH} )
TRY_COMPILE( MPI_TEST_CXX ${CMAKE_CURRENT_BINARY_DIR} ${MPI_TEST_SRC}
CMAKE_FLAGS ${TMP_FLAGS}
LINK_OPTIONS ${MPI_CXX_LINK_FLAGS}
LINK_LIBRARIES ${MPI_CXX_LIBRARIES}
OUTPUT_VARIABLE OUT_TXT)
IF ( NOT ${MPI_TEST} )
MESSAGE( FATAL_ERROR "Skipping MPI search and default compile fails:\n${OUT_TXT}" )
ENDIF()
SET( MPI_C_FOUND TRUE )
SET( MPI_CXX_FOUND TRUE )
SET( MPI_Fortran_FOUND TRUE )
ENDIF()
ENDIF()
# Set MPI flags
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
ENDIF()
ELSEIF ( MPI_COMPILER )
# The mpi compiler should take care of everything
IF ( MPI_INCLUDE )
INCLUDE_DIRECTORIES( ${MPI_INCLUDE} )
ENDIF()
ELSE()
# Perform the default search for MPI
INCLUDE ( FindMPI )
IF ( NOT MPI_FOUND )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
MESSAGE( FATAL_ERROR "Did not find MPI" )
ENDIF ()
INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" )
SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" )
ENDIF()
# Check if we need to use MPI for serial tests
CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
# Set defaults if they have not been set
IF ( NOT MPIEXEC )
SET( MPIEXEC mpirun )
ENDIF()
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
ENDIF()
# Set the definitions
ADD_DEFINITIONS( "-DUSE_MPI" )
MESSAGE( "Using MPI" )
STRING( STRIP "${MPI_CXX_COMPILE_FLAGS}" MPI_CXX_COMPILE_FLAGS )
STRING( STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS )
STRING( STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES )
MESSAGE( " MPI_CXX_FOUND = ${MPI_CXX_FOUND}" )
MESSAGE( " MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}" )
MESSAGE( " MPI_CXX_COMPILE_FLAGS = ${MPI_CXX_COMPILE_FLAGS}" )
MESSAGE( " MPI_CXX_INCLUDE_PATH = ${MPI_CXX_INCLUDE_PATH}" )
MESSAGE( " MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}" )
MESSAGE( " MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}" )
MESSAGE( " MPIEXEC = ${MPIEXEC}" )
MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
ELSE()
SET( USE_MPI_FOR_SERIAL_TESTS 0 )
SET( MPIEXEC "" )
SET( MPIEXEC_NUMPROC_FLAG "" )
SET( MPI_INCLUDE "" )
SET( MPI_LINK_FLAGS "" )
SET( MPI_LIBRARIES "" )
MESSAGE( "Not using MPI, all parallel tests will be disabled" )
MESSAGE( " MPIEXEC_PREFLAGS = ${MPIEXEC_PREFLAGS}" )
MESSAGE( " MPIEXEC_POSTFLAGS = ${MPIEXEC_POSTFLAGS}" )
ADD_DEFINITIONS( -DUSE_MPI )
INCLUDE_DIRECTORIES( ${MPI_CXX_INCLUDE_PATH} )
SET( MPI_LIBRARIES ${MPI_CXX_LIBRARIES} )
SET( MPI_LINK_FLAGS ${MPI_CXX_LINK_FLAGS} )
IF ( NOT MPI_CXX_FOUND )
MESSAGE( FATAL_ERROR "MPI not found" )
ENDIF()
IF ( USE_MPI AND NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Unable to find MPIEXEC, please set it before continuing" )
ENDIF()
ENDIF()
ENDMACRO()

4
cmake/macros.cmake
View File

@ -681,8 +681,8 @@ MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( MPI_LIBRARIES )
TARGET_LINK_LIBRARIES( ${EXE} ${ARGN} ${tmp} )
FOREACH ( tmp ${MPI_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )

7
common/Communication.h
View File

@ -67,6 +67,10 @@ public:
//! Destructor
~fillHalo( );
fillHalo() = delete;
fillHalo(const fillHalo&) = delete;
fillHalo& operator=(const fillHalo&) = delete;
/*!
* @brief Communicate the halos
* @param[in] array The array on which we fill the halos
@ -93,9 +97,6 @@ private:
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor
fillHalo& operator=(const fillHalo&); // Private assignment operator
void pack( const Array<TYPE>& array, int i, int j, int k, TYPE *buffer );
void unpack( Array<TYPE>& array, int i, int j, int k, const TYPE *buffer );
};

26
common/Domain.cpp
View File

@ -558,17 +558,13 @@ void Domain::Decomp( const std::string& Filename )
int64_t z_transition_size = (nprocz*nz - (global_Nz - zStart))/2;
if (z_transition_size < 0) z_transition_size=0;
char LocalRankFilename[40];
char *loc_id;
loc_id = new char [(nx+2)*(ny+2)*(nz+2)];
// Set up the sub-domains
if (RANK==0){
printf("Distributing subdomains across %i processors \n",nprocs);
printf("Process grid: %i x %i x %i \n",nprocx,nprocy,nprocz);
printf("Subdomain size: %i x %i x %i \n",nx,ny,nz);
printf("Size of transition region: %ld \n", z_transition_size);
auto loc_id = new char [(nx+2)*(ny+2)*(nz+2)];
for (int kp=0; kp<nprocz; kp++){
for (int jp=0; jp<nprocy; jp++){
for (int ip=0; ip<nprocx; ip++){
@ -609,6 +605,7 @@ void Domain::Decomp( const std::string& Filename )
Comm.send(loc_id,N,rnk,15);
}
// Write the data for this rank data
char LocalRankFilename[40];
sprintf(LocalRankFilename,"ID.%05i",rnk+rank_offset);
FILE *ID = fopen(LocalRankFilename,"wb");
fwrite(loc_id,1,(nx+2)*(ny+2)*(nz+2),ID);
@ -616,9 +613,8 @@ void Domain::Decomp( const std::string& Filename )
}
}
}
}
else{
delete [] loc_id;
} else {
// Recieve the subdomain from rank = 0
//printf("Ready to recieve data %i at process %i \n", N,rank);
Comm.recv(id.data(),N,0,15);
@ -645,6 +641,7 @@ void Domain::Decomp( const std::string& Filename )
porosity = sum*iVol_global;
if (rank()==0) printf("Media porosity = %f \n",porosity);
//.........................................................
delete [] SegData;
}
void Domain::AggregateLabels( const std::string& filename ){
@ -669,8 +666,7 @@ void Domain::AggregateLabels( const std::string& filename ){
int local_size = (nx-2)*(ny-2)*(nz-2);
long int full_size = long(full_nx)*long(full_ny)*long(full_nz);
signed char *LocalID;
LocalID = new signed char [local_size];
auto LocalID = new signed char [local_size];
//printf("aggregate labels: local size=%i, global size = %i",local_size, full_size);
// assign the ID for the local sub-region
@ -687,8 +683,7 @@ void Domain::AggregateLabels( const std::string& filename ){
// populate the FullID
if (rank() == 0){
signed char *FullID;
FullID = new signed char [full_size];
auto FullID = new signed char [full_size];
// first handle local ID for rank 0
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
@ -727,6 +722,7 @@ void Domain::AggregateLabels( const std::string& filename ){
FILE *OUTFILE = fopen(filename.c_str(),"wb");
fwrite(FullID,1,full_size,OUTFILE);
fclose(OUTFILE);
delete [] FullID;
}
else{
// send LocalID to rank=0
@ -734,8 +730,8 @@ void Domain::AggregateLabels( const std::string& filename ){
int dstrank = 0;
Comm.send(LocalID,local_size,dstrank,tag);
}
delete [] LocalID;
Comm.barrier();
}
/********************************************************
@ -1232,11 +1228,10 @@ void Domain::ReadFromFile(const std::string& Filename,const std::string& Datatyp
// user needs to modify the input file accordingly before LBPM simulator read
// the input file.
//........................................................................................
int rank_offset = 0;
int RANK = rank();
int nprocs, nprocx, nprocy, nprocz, nx, ny, nz;
int64_t global_Nx,global_Ny,global_Nz;
int64_t i,j,k,n;
int64_t i,j,k;
//TODO These offset we may still need them
int64_t xStart,yStart,zStart;
xStart=yStart=zStart=0;
@ -1393,7 +1388,6 @@ void Domain::AggregateLabels( const std::string& filename, DoubleArray &UserData
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
for (int i=1; i<nx-1; i++){
int n = k*nx*ny+j*nx+i;
double local_id_val = UserData(i,j,k);
LocalID[(k-1)*(nx-2)*(ny-2) + (j-1)*(nx-2) + i-1] = local_id_val;
}

85
common/MPI.cpp
View File

@ -401,7 +401,6 @@ MPI_CLASS::MPI_CLASS()
communicator = MPI_CLASS_COMM_NULL;
d_maxTag = mpi_max_tag;
#endif
d_ranks = nullptr;
d_count = nullptr;
d_manage = false;
comm_rank = 0;
@ -435,8 +434,6 @@ void MPI_CLASS::reset()
++N_MPI_Comm_destroyed;
#endif
}
if ( d_ranks != nullptr )
delete[] d_ranks;
delete d_count;
}
if ( d_currentTag == nullptr ) {
@ -448,7 +445,6 @@ void MPI_CLASS::reset()
}
d_manage = false;
d_count = nullptr;
d_ranks = nullptr;
comm_rank = 0;
comm_size = 1;
d_maxTag = 0;
@ -467,7 +463,6 @@ MPI_CLASS::MPI_CLASS( const MPI_CLASS &comm )
d_manage( comm.d_manage ),
comm_rank( comm.comm_rank ),
comm_size( comm.comm_size ),
d_ranks( comm.d_ranks ),
d_maxTag( comm.d_maxTag ),
d_currentTag( comm.d_currentTag )
{
@ -490,7 +485,6 @@ MPI_CLASS::MPI_CLASS( MPI_CLASS &&rhs ) : MPI_CLASS()
std::swap( profile_level, rhs.profile_level );
std::swap( comm_rank, rhs.comm_rank );
std::swap( comm_size, rhs.comm_size );
std::swap( d_ranks, rhs.d_ranks );
std::swap( d_maxTag, rhs.d_maxTag );
std::swap( d_currentTag, rhs.d_currentTag );
std::swap( d_count, rhs.d_count );
@ -511,7 +505,6 @@ MPI_CLASS &MPI_CLASS::operator=( const MPI_CLASS &comm )
this->communicator = comm.communicator;
this->comm_rank = comm.comm_rank;
this->comm_size = comm.comm_size;
this->d_ranks = comm.d_ranks;
this->d_isNull = comm.d_isNull;
this->d_manage = comm.d_manage;
this->d_maxTag = comm.d_maxTag;
@ -537,7 +530,6 @@ MPI_CLASS &MPI_CLASS::operator=( MPI_CLASS &&rhs )
std::swap( profile_level, rhs.profile_level );
std::swap( comm_rank, rhs.comm_rank );
std::swap( comm_size, rhs.comm_size );
std::swap( d_ranks, rhs.d_ranks );
std::swap( d_maxTag, rhs.d_maxTag );
std::swap( d_currentTag, rhs.d_currentTag );
std::swap( d_count, rhs.d_count );
@ -560,7 +552,6 @@ std::atomic_int d_global_count_self = { 1 };
MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
{
d_count = nullptr;
d_ranks = nullptr;
d_manage = false;
tmp_alignment = -1;
// Check if we are using our version of comm_world
@ -623,11 +614,7 @@ MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
}
if ( d_manage )
++N_MPI_Comm_created;
// Create d_ranks
if ( comm_size > 1 ) {
d_ranks = new int[comm_size];
d_ranks[0] = -1;
}
#else
// We are not using MPI, intialize based on the communicator
NULL_USE( manage );
@ -663,34 +650,32 @@ MPI_CLASS::MPI_CLASS( MPI_Comm comm, bool manage )
************************************************************************/
std::vector<int> MPI_CLASS::globalRanks() const
{
// Get my global rank if it has not been set
static int myGlobalRank = -1;
if ( myGlobalRank == -1 ) {
#ifdef USE_MPI
if ( MPI_active() )
MPI_Comm_rank( MPI_CLASS_COMM_WORLD, &myGlobalRank );
#else
myGlobalRank = 0;
#endif
}
// Check if we are dealing with a serial or null communicator
if ( comm_size == 1 )
return std::vector<int>( 1, myGlobalRank );
if ( d_ranks == nullptr || communicator == MPI_COMM_NULL )
if ( d_isNull )
return std::vector<int>();
// Fill d_ranks if necessary
if ( d_ranks[0] == -1 ) {
if ( communicator == MPI_CLASS_COMM_WORLD ) {
for ( int i = 0; i < comm_size; i++ )
d_ranks[i] = i;
} else {
MPI_ASSERT( myGlobalRank != -1 );
this->allGather( myGlobalRank, d_ranks );
#ifdef USE_MPI
// Get my global rank and size if it has not been set
static int globalRank = -1;
static int globalSize = -1;
if ( globalRank == -1 && MPI_active() ) {
MPI_Comm_rank( MPI_CLASS_COMM_WORLD, &globalRank );
MPI_Comm_size( MPI_CLASS_COMM_WORLD, &globalSize );
}
// Check if we are dealing with a serial or global communicator
if ( comm_size == 1 )
return std::vector<int>( 1, globalRank );
if ( comm_size == globalSize ) {
std::vector<int> ranks( globalSize );
for ( int i = 0; i < globalSize; i++ )
ranks[i] = i;
return ranks;
}
// Return d_ranks
return std::vector<int>( d_ranks, d_ranks + comm_size );
// Get the global rank from each rank in the communicator
auto ranks = allGather( globalRank );
std::sort( ranks.begin(), ranks.end() );
return ranks;
#else
return std::vector<int>( 1, 1 );
#endif
}
@ -2806,7 +2791,6 @@ MPI_Request MPI_CLASS::IrecvBytes(
}
/************************************************************************
* sendrecv *
************************************************************************/
@ -2816,9 +2800,8 @@ void MPI_CLASS::sendrecv<char>( const char* sendbuf, int sendcount, int dest, in
char *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<char>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag,
recvbuf, recvcount, MPI_CHAR, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_CHAR, dest, sendtag, recvbuf, recvcount, MPI_CHAR, source,
recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<char>", profile_level );
}
template<>
@ -2826,9 +2809,8 @@ void MPI_CLASS::sendrecv<int>( const int* sendbuf, int sendcount, int dest, int
int *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<int>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag,
recvbuf, recvcount, MPI_INT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_INT, dest, sendtag, recvbuf, recvcount, MPI_INT, source,
recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<int>", profile_level );
}
template<>
@ -2836,9 +2818,8 @@ void MPI_CLASS::sendrecv<float>( const float* sendbuf, int sendcount, int dest,
float *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<float>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag,
recvbuf, recvcount, MPI_FLOAT, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_FLOAT, dest, sendtag, recvbuf, recvcount, MPI_FLOAT,
source, recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<float>", profile_level );
}
template<>
@ -2846,9 +2827,8 @@ void MPI_CLASS::sendrecv<double>( const double* sendbuf, int sendcount, int dest
double *recvbuf, int recvcount, int source, int recvtag ) const
{
PROFILE_START( "sendrecv<double>", profile_level );
MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag,
recvbuf, recvcount, MPI_DOUBLE, source, recvtag,
communicator, MPI_STATUS_IGNORE );
MPI_Sendrecv( sendbuf, sendcount, MPI_DOUBLE, dest, sendtag, recvbuf, recvcount, MPI_DOUBLE,
source, recvtag, communicator, MPI_STATUS_IGNORE );
PROFILE_STOP( "sendrecv<double>", profile_level );
}
#endif
@ -3828,4 +3808,3 @@ MPI MPI::loadBalance( double local, std::vector<double> work )
} // namespace Utilities

20
common/MPI.h
View File

@ -8,10 +8,14 @@ Copyright (c) 2012 UT-Battelle, LLC
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
Collection of administrative costs for redistribution of the source code or binary form is allowed. However, collection of a royalty or other fee in excess of good faith amount for cost recovery for such redistribution is prohibited.
Redistribution and use in source and binary forms, with or without modification, are permitted
provided that the following conditions are met: Redistributions of source code must retain the above
copyright notice, this list of conditions and the following disclaimer. Redistributions in binary
form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution. Collection of
administrative costs for redistribution of the source code or binary form is allowed. However,
collection of a royalty or other fee in excess of good faith amount for cost recovery for such
redistribution is prohibited.
*/
@ -274,6 +278,8 @@ public: // Member functions
* \brief Return the global ranks for the comm
* \details This returns a vector which contains the global ranks for each
* member of the communicator. The global ranks are defined according to WORLD comm.
* Note: this function is a blocking collective on the current communicator
* (unless the current communicator is global, self, or null)
*/
std::vector<int> globalRanks() const;
@ -796,7 +802,8 @@ public: // Member functions
* @brief This function sends and recieves data using a blocking call
*/
template<class type>
void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf, int recvcount, int source, int recvtag ) const;
void sendrecv( const type *sendbuf, int sendcount, int dest, int sendtag, type *recvbuf,
int recvcount, int source, int recvtag ) const;
/*!
@ -1126,9 +1133,6 @@ private: // data members
// The rank and size of the communicator
int comm_rank, comm_size;
// The ranks of the comm in the global comm
mutable int *volatile d_ranks;
// Some attributes
int d_maxTag;
int *volatile d_currentTag;

154
common/ScaLBL.cpp
View File

@ -1,5 +1,8 @@
#include "common/ScaLBL.h"
#include <chrono>
ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
//......................................................................................
Lock=false; // unlock the communicator
@ -306,9 +309,128 @@ ScaLBL_Communicator::ScaLBL_Communicator(std::shared_ptr <Domain> Dm){
}
ScaLBL_Communicator::~ScaLBL_Communicator(){
// destrutor does nothing (bad idea)
// -- note that there needs to be a way to free memory allocated on the device!!!
ScaLBL_Communicator::~ScaLBL_Communicator()
{
ScaLBL_FreeDeviceMemory( sendbuf_x );
ScaLBL_FreeDeviceMemory( sendbuf_X );
ScaLBL_FreeDeviceMemory( sendbuf_y );
ScaLBL_FreeDeviceMemory( sendbuf_Y );
ScaLBL_FreeDeviceMemory( sendbuf_z );
ScaLBL_FreeDeviceMemory( sendbuf_Z );
ScaLBL_FreeDeviceMemory( sendbuf_xy );
ScaLBL_FreeDeviceMemory( sendbuf_xY );
ScaLBL_FreeDeviceMemory( sendbuf_Xy );
ScaLBL_FreeDeviceMemory( sendbuf_XY );
ScaLBL_FreeDeviceMemory( sendbuf_xz );
ScaLBL_FreeDeviceMemory( sendbuf_xZ );
ScaLBL_FreeDeviceMemory( sendbuf_Xz );
ScaLBL_FreeDeviceMemory( sendbuf_XZ );
ScaLBL_FreeDeviceMemory( sendbuf_yz );
ScaLBL_FreeDeviceMemory( sendbuf_yZ );
ScaLBL_FreeDeviceMemory( sendbuf_Yz );
ScaLBL_FreeDeviceMemory( sendbuf_YZ );
ScaLBL_FreeDeviceMemory( recvbuf_x );
ScaLBL_FreeDeviceMemory( recvbuf_X );
ScaLBL_FreeDeviceMemory( recvbuf_y );
ScaLBL_FreeDeviceMemory( recvbuf_Y );
ScaLBL_FreeDeviceMemory( recvbuf_z );
ScaLBL_FreeDeviceMemory( recvbuf_Z );
ScaLBL_FreeDeviceMemory( recvbuf_xy );
ScaLBL_FreeDeviceMemory( recvbuf_xY );
ScaLBL_FreeDeviceMemory( recvbuf_Xy );
ScaLBL_FreeDeviceMemory( recvbuf_XY );
ScaLBL_FreeDeviceMemory( recvbuf_xz );
ScaLBL_FreeDeviceMemory( recvbuf_xZ );
ScaLBL_FreeDeviceMemory( recvbuf_Xz );
ScaLBL_FreeDeviceMemory( recvbuf_XZ );
ScaLBL_FreeDeviceMemory( recvbuf_yz );
ScaLBL_FreeDeviceMemory( recvbuf_yZ );
ScaLBL_FreeDeviceMemory( recvbuf_Yz );
ScaLBL_FreeDeviceMemory( recvbuf_YZ );
ScaLBL_FreeDeviceMemory( dvcSendList_x );
ScaLBL_FreeDeviceMemory( dvcSendList_X );
ScaLBL_FreeDeviceMemory( dvcSendList_y );
ScaLBL_FreeDeviceMemory( dvcSendList_Y );
ScaLBL_FreeDeviceMemory( dvcSendList_z );
ScaLBL_FreeDeviceMemory( dvcSendList_Z );
ScaLBL_FreeDeviceMemory( dvcSendList_xy );
ScaLBL_FreeDeviceMemory( dvcSendList_xY );
ScaLBL_FreeDeviceMemory( dvcSendList_Xy );
ScaLBL_FreeDeviceMemory( dvcSendList_XY );
ScaLBL_FreeDeviceMemory( dvcSendList_xz );
ScaLBL_FreeDeviceMemory( dvcSendList_xZ );
ScaLBL_FreeDeviceMemory( dvcSendList_Xz );
ScaLBL_FreeDeviceMemory( dvcSendList_XZ );
ScaLBL_FreeDeviceMemory( dvcSendList_yz );
ScaLBL_FreeDeviceMemory( dvcSendList_yZ );
ScaLBL_FreeDeviceMemory( dvcSendList_Yz );
ScaLBL_FreeDeviceMemory( dvcSendList_YZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_x );
ScaLBL_FreeDeviceMemory( dvcRecvList_X );
ScaLBL_FreeDeviceMemory( dvcRecvList_y );
ScaLBL_FreeDeviceMemory( dvcRecvList_Y );
ScaLBL_FreeDeviceMemory( dvcRecvList_z );
ScaLBL_FreeDeviceMemory( dvcRecvList_Z );
ScaLBL_FreeDeviceMemory( dvcRecvList_xy );
ScaLBL_FreeDeviceMemory( dvcRecvList_xY );
ScaLBL_FreeDeviceMemory( dvcRecvList_Xy );
ScaLBL_FreeDeviceMemory( dvcRecvList_XY );
ScaLBL_FreeDeviceMemory( dvcRecvList_xz );
ScaLBL_FreeDeviceMemory( dvcRecvList_xZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_Xz );
ScaLBL_FreeDeviceMemory( dvcRecvList_XZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_yz );
ScaLBL_FreeDeviceMemory( dvcRecvList_yZ );
ScaLBL_FreeDeviceMemory( dvcRecvList_Yz );
ScaLBL_FreeDeviceMemory( dvcRecvList_YZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_x );
ScaLBL_FreeDeviceMemory( dvcRecvDist_X );
ScaLBL_FreeDeviceMemory( dvcRecvDist_y );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Y );
ScaLBL_FreeDeviceMemory( dvcRecvDist_z );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Z );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xy );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xY );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Xy );
ScaLBL_FreeDeviceMemory( dvcRecvDist_XY );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_xZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Xz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_XZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_yz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_yZ );
ScaLBL_FreeDeviceMemory( dvcRecvDist_Yz );
ScaLBL_FreeDeviceMemory( dvcRecvDist_YZ );
}
double ScaLBL_Communicator::GetPerformance(int *NeighborList, double *fq, int Np){
/* EACH MPI PROCESS GETS ITS OWN MEASUREMENT*/
/* use MRT kernels to check performance without communication / synchronization */
int TIMESTEPS=500;
double RLX_SETA=1.0;
double RLX_SETB = 8.f*(2.f-RLX_SETA)/(8.f-RLX_SETA);
double FX = 0.0;
double FY = 0.0;
double FZ = 0.0;
ScaLBL_D3Q19_Init(fq, Np);
//.......create and start timer............
Barrier();
auto t1 = std::chrono::system_clock::now();
for (int t=0; t<TIMESTEPS; t++){
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAodd_MRT(NeighborList, fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAeven_MRT(fq, FirstInterior(), LastInterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
ScaLBL_D3Q19_AAeven_MRT(fq, 0, LastExterior(), Np, RLX_SETA, RLX_SETB, FX, FY, FZ);
}
auto t2 = std::chrono::system_clock::now();
Barrier();
// Compute the walltime per timestep
double diff = std::chrono::duration<double>( t2 - t1 ).count();
double cputime = 0.5*diff/TIMESTEPS;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
return MLUPS;
}
int ScaLBL_Communicator::LastExterior(){
return next;
@ -364,7 +486,7 @@ int ScaLBL_Communicator::MemoryOptimizedLayoutAA(IntArray &Map, int *neighborLis
int idx,i,j,k,n;
// Check that Map has size matching sub-domain
if (Map.size(0) != Nx)
if ( (int) Map.size(0) != Nx)
ERROR("ScaLBL_Communicator::MemoryOptimizedLayout: Map array dimensions do not match! \n");
// Initialize Map
@ -1501,35 +1623,31 @@ void ScaLBL_Communicator::SendD3Q7AA(double *Aq, int Component){
// Pack the distributions
//...Packing for x face(2,8,10,12,14)................................
ScaLBL_D3Q19_Pack(2,dvcSendList_x,0,sendCount_x,sendbuf_x,&Aq[Component*7*N],N);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
//...Packing for X face(1,7,9,11,13)................................
ScaLBL_D3Q19_Pack(1,dvcSendList_X,0,sendCount_X,sendbuf_X,&Aq[Component*7*N],N);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
//...Packing for y face(4,8,9,16,18).................................
ScaLBL_D3Q19_Pack(4,dvcSendList_y,0,sendCount_y,sendbuf_y,&Aq[Component*7*N],N);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
//...Packing for Y face(3,7,10,15,17).................................
ScaLBL_D3Q19_Pack(3,dvcSendList_Y,0,sendCount_Y,sendbuf_Y,&Aq[Component*7*N],N);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
//...Packing for z face(6,12,13,16,17)................................
ScaLBL_D3Q19_Pack(6,dvcSendList_z,0,sendCount_z,sendbuf_z,&Aq[Component*7*N],N);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
//...Packing for Z face(5,11,14,15,18)................................
ScaLBL_D3Q19_Pack(5,dvcSendList_Z,0,sendCount_Z,sendbuf_Z,&Aq[Component*7*N],N);
//...................................................................................
// Send all the distributions
//...................................................................................
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x, sendCount_x,rank_x,sendtag);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X, recvCount_X,rank_X,recvtag);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X, sendCount_X,rank_X,sendtag);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x, recvCount_x,rank_x,recvtag);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y, sendCount_y,rank_y,sendtag);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y, recvCount_Y,rank_Y,recvtag);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y, sendCount_Y,rank_Y,sendtag);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y, recvCount_y,rank_y,recvtag);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z, sendCount_z,rank_z,sendtag);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z, recvCount_Z,rank_Z,recvtag);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z, sendCount_Z,rank_Z,sendtag);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z, recvCount_z,rank_z,recvtag);
}

89
common/ScaLBL.h
View File

@ -70,49 +70,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_IMRT(double *dist, int start, int
extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_IMRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
// ION TRANSPORT MODEL
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
// LBM Poisson solver
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
//maybe deprecated
//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
// int strideY, int strideZ,int start, int finish, int Np);
// LBM Stokes Model (adapted from MRT model)
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
@ -129,6 +86,7 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure,
double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
// ION TRANSPORT MODEL
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
@ -197,6 +155,12 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz);
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz);
@ -220,6 +184,44 @@ extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, doub
extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double *ColorGrad, int start, int finish, int Np);
// FREE ENERGY LEE MODEL
extern "C" void ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *mu_phi, double *ColorGrad, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad,
double rhonA, double rhoB, double tauM, double W, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np);
// BOUNDARY CONDITION ROUTINES
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
@ -317,6 +319,7 @@ public:
int FirstInterior();
int LastInterior();
double GetPerformance(int *NeighborList, double *fq, int Np);
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np, int width);
void Barrier(){
ScaLBL_DeviceBarrier();

118
common/WideHalo.cpp
View File

@ -58,7 +58,6 @@ ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain
rank_xyZ = rank_info.rank[0][0][2];
rank_XyZ = rank_info.rank[2][0][2];
rank_xYZ = rank_info.rank[0][2][2];
MPI_COMM_SCALBL.barrier();
/* Fill in communications patterns for the lists */
@ -68,7 +67,7 @@ ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain
sendCount_y =getHaloBlock(width,Nxh-width,width,2*width,width,Nzh-width,dvcSendList_y);
sendCount_Y =getHaloBlock(width,Nxh-width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_Y);
sendCount_z =getHaloBlock(width,Nxh-width,width,Nyh-width,width,2*width,dvcSendList_z);
sendCount_X =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_Z);
sendCount_Z =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_Z);
// xy
sendCount_xy =getHaloBlock(width,2*width,width,2*width,width,Nzh-width,dvcSendList_xy);
sendCount_xY =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_xY);
@ -88,7 +87,7 @@ ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain
sendCount_xyz =getHaloBlock(width,2*width,width,2*width,width,2*width,dvcSendList_xyz);
sendCount_xyZ =getHaloBlock(width,2*width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_xyZ);
sendCount_xYz =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_xYz);
sendCount_xYz =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xYZ);
sendCount_xYZ =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xYZ);
sendCount_Xyz =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,width,2*width,dvcSendList_Xyz);
sendCount_XyZ =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_XyZ);
sendCount_XYz =getHaloBlock(Nxh-2*width,Nxh-width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_XYz);
@ -235,60 +234,61 @@ void ScaLBLWideHalo_Communicator::Send(double *data){
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = MPI_COMM_SCALBL.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = MPI_COMM_SCALBL.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = MPI_COMM_SCALBL.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = MPI_COMM_SCALBL.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = MPI_COMM_SCALBL.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = MPI_COMM_SCALBL.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = MPI_COMM_SCALBL.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = MPI_COMM_SCALBL.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = MPI_COMM_SCALBL.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = MPI_COMM_SCALBL.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = MPI_COMM_SCALBL.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = MPI_COMM_SCALBL.Irecv(&recvCount_z,1,rank_z,recvtag+5);
req1[6] = MPI_COMM_SCALBL.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = MPI_COMM_SCALBL.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = MPI_COMM_SCALBL.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = MPI_COMM_SCALBL.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = MPI_COMM_SCALBL.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = MPI_COMM_SCALBL.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = MPI_COMM_SCALBL.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = MPI_COMM_SCALBL.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
req1[10] = MPI_COMM_SCALBL.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = MPI_COMM_SCALBL.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = MPI_COMM_SCALBL.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = MPI_COMM_SCALBL.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = MPI_COMM_SCALBL.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = MPI_COMM_SCALBL.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = MPI_COMM_SCALBL.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = MPI_COMM_SCALBL.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
req1[14] = MPI_COMM_SCALBL.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = MPI_COMM_SCALBL.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = MPI_COMM_SCALBL.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = MPI_COMM_SCALBL.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = MPI_COMM_SCALBL.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = MPI_COMM_SCALBL.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = MPI_COMM_SCALBL.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = MPI_COMM_SCALBL.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x,sendCount_x,rank_x,sendtag+0);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag+0);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X,sendCount_X,rank_X,sendtag+1);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag+1);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y,sendCount_y,rank_y,sendtag+2);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag+2);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag+3);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag+3);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z,sendCount_z,rank_z,sendtag+4);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag+4);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag+5);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag+5);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag+6);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag+6);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag+7);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag+7);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag+8);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag+8);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag+9);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag+9);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag+10);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag+10);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag+11);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag+11);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag+12);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag+12);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag+13);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag+13);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag+14);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag+14);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag+15);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag+15);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag+16);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag+16);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag+17);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag+17);
/* Corners */
req1[18] = MPI_COMM_SCALBL.Isend(&sendCount_xyz,1,rank_xyz,sendtag+18);
req2[18] = MPI_COMM_SCALBL.Irecv(&recvCount_XYZ,1,rank_XYZ,recvtag+18);
req1[19] = MPI_COMM_SCALBL.Isend(&sendCount_XYz,1,rank_XYz,sendtag+19);
req2[19] = MPI_COMM_SCALBL.Irecv(&recvCount_xyZ,1,rank_xyZ,recvtag+19);
req1[20] = MPI_COMM_SCALBL.Isend(&sendCount_Xyz,1,rank_Xyz,sendtag+20);
req2[20] = MPI_COMM_SCALBL.Irecv(&recvCount_xYZ,1,rank_xYZ,recvtag+20);
req1[21] = MPI_COMM_SCALBL.Isend(&sendCount_xYz,1,rank_xYz,sendtag+21);
req2[21] = MPI_COMM_SCALBL.Irecv(&recvCount_XyZ,1,rank_XyZ,recvtag+21);
req1[22] = MPI_COMM_SCALBL.Isend(&sendCount_xyZ,1,rank_xyZ,sendtag+22);
req2[22] = MPI_COMM_SCALBL.Irecv(&recvCount_XYz,1,rank_XYz,recvtag+22);
req1[23] = MPI_COMM_SCALBL.Isend(&sendCount_XYZ,1,rank_XYZ,sendtag+23);
req2[23] = MPI_COMM_SCALBL.Irecv(&recvCount_xyz,1,rank_xyz,recvtag+23);
req1[24] = MPI_COMM_SCALBL.Isend(&sendCount_XyZ,1,rank_XyZ,sendtag+24);
req2[24] = MPI_COMM_SCALBL.Irecv(&recvCount_xYz,1,rank_xYz,recvtag+24);
req1[25] = MPI_COMM_SCALBL.Isend(&sendCount_xYZ,1,rank_xYZ,sendtag+25);
req2[25] = MPI_COMM_SCALBL.Irecv(&recvCount_Xyz,1,rank_Xyz,recvtag+25);
req1[18] = MPI_COMM_SCALBL.Isend(sendbuf_xyz,sendCount_xyz,rank_xyz,sendtag+18);
req2[18] = MPI_COMM_SCALBL.Irecv(recvbuf_XYZ,recvCount_XYZ,rank_XYZ,recvtag+18);
req1[19] = MPI_COMM_SCALBL.Isend(sendbuf_XYz,sendCount_XYz,rank_XYz,sendtag+19);
req2[19] = MPI_COMM_SCALBL.Irecv(recvbuf_xyZ,recvCount_xyZ,rank_xyZ,recvtag+19);
req1[20] = MPI_COMM_SCALBL.Isend(sendbuf_Xyz,sendCount_Xyz,rank_Xyz,sendtag+20);
req2[20] = MPI_COMM_SCALBL.Irecv(recvbuf_xYZ,recvCount_xYZ,rank_xYZ,recvtag+20);
req1[21] = MPI_COMM_SCALBL.Isend(sendbuf_xYz,sendCount_xYz,rank_xYz,sendtag+21);
req2[21] = MPI_COMM_SCALBL.Irecv(recvbuf_XyZ,recvCount_XyZ,rank_XyZ,recvtag+21);
req1[22] = MPI_COMM_SCALBL.Isend(sendbuf_xyZ,sendCount_xyZ,rank_xyZ,sendtag+22);
req2[22] = MPI_COMM_SCALBL.Irecv(recvbuf_XYz,recvCount_XYz,rank_XYz,recvtag+22);
req1[23] = MPI_COMM_SCALBL.Isend(sendbuf_XYZ,sendCount_XYZ,rank_XYZ,sendtag+23);
req2[23] = MPI_COMM_SCALBL.Irecv(recvbuf_xyz,recvCount_xyz,rank_xyz,recvtag+23);
req1[24] = MPI_COMM_SCALBL.Isend(sendbuf_XyZ,sendCount_XyZ,rank_XyZ,sendtag+24);
req2[24] = MPI_COMM_SCALBL.Irecv(recvbuf_xYz,recvCount_xYz,rank_xYz,recvtag+24);
req1[25] = MPI_COMM_SCALBL.Isend(sendbuf_xYZ,sendCount_xYZ,rank_xYZ,sendtag+25);
req2[25] = MPI_COMM_SCALBL.Irecv(recvbuf_Xyz,recvCount_Xyz,rank_Xyz,recvtag+25);
//...................................................................................
}
@ -298,11 +298,13 @@ ScaLBLWideHalo_Communicator::~ScaLBLWideHalo_Communicator()
void ScaLBLWideHalo_Communicator::Recv(double *data){
//...................................................................................
MPI_Waitall(26,req1,stat1);
MPI_Waitall(26,req2,stat2);
Utilities::MPI::waitAll(26,req1);
Utilities::MPI::waitAll(26,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
//...................................................................................
//printf("Ready to unpack %i to x\n",recvCount_x);
//printf(" print first 10 values...\n");
//for (int idx=0; idx<10; idx++) printf(" recvBuf[%i]=%f \n",idx,recvbuf_x[idx]);
ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh);

7
common/WideHalo.h
View File

@ -4,6 +4,7 @@ This class implements support for halo widths larger than 1
#ifndef WideHalo_H
#define WideHalo_H
#include "common/ScaLBL.h"
#include "common/MPI.h"
class ScaLBLWideHalo_Communicator{
public:
@ -52,9 +53,7 @@ private:
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
MPI_Request req1[26],req2[26];
MPI_Status stat1[26],stat2[26];
//......................................................................................
// MPI ranks for all 18 neighbors
//......................................................................................
@ -95,11 +94,11 @@ private:
int *dvcRecvList_xyZ,*dvcRecvList_XyZ,*dvcRecvList_xYZ,*dvcRecvList_XYZ;
//......................................................................................
inline int getHaloBlock(int imin, int imax, int jmin, int jmax, int kmin, int kmax, int *dvcList){
inline int getHaloBlock(int imin, int imax, int jmin, int jmax, int kmin, int kmax, int *& dvcList){
int count = 0;
int *List;
List = new int [(imax-imin)*(jmax-jmin)*(kmax-kmin)];
for (int k=kmin; k<kmax; k++){
for (k=kmin; k<kmax; k++){
for (j=jmin; j<jmax; j++){
for (i=imin; i<imax; i++){
List[count++] = k*Nxh*Nyh + j*Nxh + i;

194
cpu/Color.cpp
View File

@ -2489,10 +2489,200 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
int nr1,nr2,nr3,nr4,nr5,nr6;
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* neighbors */
nr1 = neighborList[n+0*Np];
nr2 = neighborList[n+1*Np];
nr3 = neighborList[n+2*Np];
nr4 = neighborList[n+3*Np];
nr5 = neighborList[n+4*Np];
nr6 = neighborList[n+5*Np];
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
// compute phase indicator field
phi=(nA-nB)/(nA+nB);
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
// q = 1
//nread = neighborList[n+Np];
Aq[nr2] = a1;
Bq[nr2] = b1;
// q=2
//nread = neighborList[n];
Aq[nr1] = a2;
Bq[nr1] = b2;
//...............................................
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
// q = 3
//nread = neighborList[n+3*Np];
Aq[nr4] = a1;
Bq[nr4] = b1;
// q = 4
//nread = neighborList[n+2*Np];
Aq[nr3] = a2;
Bq[nr3] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
// q = 5
//nread = neighborList[n+5*Np];
Aq[nr6] = a1;
Bq[nr6] = b1;
// q = 6
//nread = neighborList[n+4*Np];
Aq[nr5] = a2;
Bq[nr5] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
Aq[1*Np+n] = a1;
Bq[1*Np+n] = b1;
Aq[2*Np+n] = a2;
Bq[2*Np+n] = b2;
//...............................................
// q = 2
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
Aq[3*Np+n] = a1;
Bq[3*Np+n] = b1;
Aq[4*Np+n] = a2;
Bq[4*Np+n] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
Aq[5*Np+n] = a1;
Bq[5*Np+n] = b1;
Aq[6*Np+n] = a2;
Bq[6*Np+n] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq,
double *Den, double *Phi, int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){
@ -2579,7 +2769,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){

4918
cpu/FreeLee.cpp
View File

File diff suppressed because it is too large Load Diff

2
ctest_script.cmake
View File

@ -164,7 +164,7 @@ SET( CTEST_OPTIONS )
SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
IF ( USE_CUDA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" )
ELSE()

2122
cuda/FreeLee.cu Normal file
View File

File diff suppressed because it is too large Load Diff

76
cuda/MixedGradient.cu Normal file
View File

@ -0,0 +1,76 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
#include <cuda.h>
#include <stdio.h>
#include <cuda_profiler_api.h>
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N,idx;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
N = Nx*Ny*Nz;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = start + S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (idx<finish){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}
}
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
cudaProfilerStart();
dvc_ScaLBL_D3Q19_MixedGradient<<<NBLOCKS,NTHREADS >>>(Map, Phi, Gradient, start, finish, Np, Nx, Ny, Nz);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_MixedGradient: %s \n",cudaGetErrorString(err));
}
cudaProfilerStop();
}

2
example/Piston/input.db
View File

@ -35,4 +35,6 @@ Analysis {
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}
Visualization {
}

15
example/Plates/input.db
View File

@ -7,10 +7,10 @@ Color {
beta = 0.95;
F = 0, 0, 0
Restart = false
timestepMax = 3000
timestepMax = 500
flux = 0.0
ComponentLabels = -2, -1
ComponentAffinity = -1.0, -0.5;
ComponentAffinity = 1.0, 1.0;
}
Domain {
@ -26,13 +26,14 @@ Domain {
}
Analysis {
blobid_interval = 1000 // Frequency to perform blob identification
analysis_interval = 1000 // Frequency to perform analysis
restart_interval = 1000 // Frequency to write restart data
visualization_interval = 1000 // Frequency to write visualization data
analysis_interval = 100 // Frequency to perform analysis
visualization_interval = 500
subphase_analysis_interval = 100
restart_interval = 100000
restart_file = "Restart" // Filename to use for restart file (will append rank)
N_threads = 4 // Number of threads to use
load_balance = "independent" // Load balance method to use: "none", "default", "independent"
}
Visualization {
}

5
hip/CMakeLists.txt
View File

@ -1,6 +1,7 @@
SET( HIP_SEPERABLE_COMPILATION ON )
SET_SOURCE_FILES_PROPERTIES( BGK.cu Color.cu CudaExtras.cu D3Q19.cu D3Q7.cu dfh.cu Extras.cu MRT.hip PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 )
HIP_ADD_LIBRARY( lbpm-hip BGK.cu Color.cu CudaExtras.cu D3Q19.cu D3Q7.cu dfh.cu Extras.cu MRT.cu SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} )
FILE( GLOB HIP_SOURCES "*.cu" )
SET_SOURCE_FILES_PROPERTIES( ${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1 )
HIP_ADD_LIBRARY( lbpm-hip ${HIP_SOURCES} SHARED HIPCC_OPTIONS ${HIP_HIPCC_OPTIONS} HCC_OPTIONS ${HIP_HCC_OPTIONS} NVCC_OPTIONS ${HIP_NVCC_OPTIONS} ${HIP_NVCC_FLAGS} )
#TARGET_LINK_LIBRARIES( lbpm-hip /opt/rocm-3.3.0/lib/libhip_hcc.so )
#TARGET_LINK_LIBRARIES( lbpm-wia lbpm-hip )
#ADD_DEPENDENCIES( lbpm-hip copy-include )

72
hip/D3Q19.cu
View File

@ -89,9 +89,25 @@ __global__ void sum_kernel_block(double *sum, double *input, int n)
__inline__ __device__
double warpReduceSum(double val) {
#if 0
for (int offset = warpSize/2; offset > 0; offset /= 2)
val += __shfl_down_sync(0xFFFFFFFF, val, offset, 32);
return val;
#else
short int id = threadIdx.x % warpSize;
__shared__ double tmp[64];
tmp[id] = val;
__syncthreads();
if ( warpSize == 64) {
tmp[id] += tmp[id+32]; __syncthreads();
}
tmp[id] += tmp[id+16]; __syncthreads();
tmp[id] += tmp[id+8]; __syncthreads();
tmp[id] += tmp[id+4]; __syncthreads();
tmp[id] += tmp[id+2]; __syncthreads();
tmp[id] += tmp[id+1]; __syncthreads();
return tmp[0];
#endif
}
__inline__ __device__
@ -1730,6 +1746,44 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_Pressure_BC_Z(int *list, double *dist,
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f5 = 0.111111111111111111111111 - dist[6*Np+n];
double f11 = 0.05555555555555555555556 - dist[12*Np+n];
double f14 = 0.05555555555555555555556 - dist[13*Np+n];
double f15 = 0.05555555555555555555556 - dist[16*Np+n];
double f18 = 0.05555555555555555555556 - dist[17*Np+n];
dist[6*Np+n] = f5;
dist[12*Np+n] = f11;
dist[13*Np+n] = f14;
dist[16*Np+n] = f15;
dist[17*Np+n] = f18;
}
}
__global__ void dvc_ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int idx, n;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
double f6 = 0.111111111111111111111111 - dist[5*Np+n];
double f12 = 0.05555555555555555555556 - dist[11*Np+n];
double f13 = 0.05555555555555555555556 - dist[14*Np+n] ;
double f16 = 0.05555555555555555555556 - dist[15*Np+n];
double f17 = 0.05555555555555555555556 - dist[18*Np+n];
dist[5*Np+n] = f6;
dist[11*Np+n] = f12;
dist[14*Np+n] = f13;
dist[15*Np+n] = f16;
dist[18*Np+n] = f17;
}
}
__global__ void dvc_ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *d_neighborList, int *list, double *dist, double din, int count, int Np)
{
int idx, n;
@ -2605,6 +2659,24 @@ extern "C" double ScaLBL_D3Q19_Flux_BC_Z(double *disteven, double *distodd, doub
}
extern "C" void ScaLBL_D3Q19_Reflection_BC_z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_z<<<GRID,512>>>(list, dist, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("HIP error in ScaLBL_D3Q19_Reflection_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_Reflection_BC_Z(int *list, double *dist, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q19_Reflection_BC_Z<<<GRID,512>>>(list, dist, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("HIP error in ScaLBL_D3Q19_Reflection_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" double deviceReduce(double *in, double* out, int N) {
int threads = 512;
int blocks = min((N + threads - 1) / threads, 1024);

536
hip/D3Q7BC.cu Normal file
View File

@ -0,0 +1,536 @@
#include <math.h>
#include <stdio.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
}
}
__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = value_q + value_b;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vin;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vout;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
// unknown distributions
nr6 = d_neighborList[n+5*Np];
dist[nr6] = f6;
}
}
//*************************************************************************
extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Dirichlet_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Neumann_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<<GRID,512>>>(list, dist, Vin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<<GRID,512>>>(list, dist, Vout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Vin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Vout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_z<<<GRID,512>>>(list, Map, Psi, Vin, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_Z<<<GRID,512>>>(list, Map, Psi, Vout, count);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<<GRID,512>>>(list, dist, Cin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<<GRID,512>>>(list, dist, Cout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Cin, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Cout, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",hipGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",hipGetErrorString(err));
}
}

2122
hip/FreeLee.cu Normal file
View File

File diff suppressed because it is too large Load Diff

2745
hip/Greyscale.cu Normal file
View File

File diff suppressed because it is too large Load Diff

3038
hip/GreyscaleColor.cu Normal file
View File

File diff suppressed because it is too large Load Diff

392
hip/Ion.cu Normal file
View File

@ -0,0 +1,392 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
int n,nread;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
Ci += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
Ci += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
Ci += fq;
// q=4
nread = neighborList[n+3*Np];
fq = dist[nread];
Ci += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
Ci += fq;
// q=6
nread = neighborList[n+5*Np];
fq = dist[nread];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
int n;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
fq = dist[2*Np+n];
Ci += fq;
// q=2
fq = dist[1*Np+n];
Ci += fq;
// q=3
fq = dist[4*Np+n];
Ci += fq;
// q=4
fq = dist[3*Np+n];
Ci += fq;
// q=5
fq = dist[6*Np+n];
Ci += fq;
// q=6
fq = dist[5*Np+n];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
// q=2
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q=4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q=6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
int n;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
Den[n] = DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
int n;
double DenInit;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
DenInit = Den[n];
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
int n;
double Ci;//ion concentration of species i
double CD;//charge density
double CD_tmp;
double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
Ci = Den[n+ion_component*Np];
CD = ChargeDensity[n];
CD_tmp = F*IonValence*Ci;
ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<<NBLOCKS,NTHREADS >>>(dist,Den,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Ion: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Ion: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init<<<NBLOCKS,NTHREADS >>>(dist,Den,DenInit,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_Init: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init_FromFile<<<NBLOCKS,NTHREADS >>>(dist,Den,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<<NBLOCKS,NTHREADS >>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

77
hip/MixedGradient.cu Normal file
View File

@ -0,0 +1,77 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
#include <stdio.h>
//#include <cuda_profiler_api.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static const int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N,idx;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
N = Nx*Ny*Nz;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = start + S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (idx<finish){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}
}
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
hipProfilerStart();
dvc_ScaLBL_D3Q19_MixedGradient<<<NBLOCKS,NTHREADS >>>(Map, Phi, Gradient, start, finish, Np, Nx, Ny, Nz);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_MixedGradient: %s \n",hipGetErrorString(err));
}
hipProfilerStop();
}

330
hip/Poisson.cu Normal file
View File

@ -0,0 +1,330 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int nread;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
psi += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
psi += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
psi += fq;
// q = 4
nread = neighborList[n+3*Np];
fq = dist[nread];
psi += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
psi += fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
fq = dist[2*Np+n];
psi += fq;
// q=2
fq = dist[1*Np+n];
psi += fq;
// q=3
fq = dist[4*Np+n];
psi += fq;
// q=4
fq = dist[3*Np+n];
psi += fq;
// q=5
fq = dist[6*Np+n];
psi += fq;
// q=6
fq = dist[5*Np+n];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q = 4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q = 6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
int ijk;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
ijk = Map[n];
dist[0*Np+n] = 0.25*Psi[ijk];
dist[1*Np+n] = 0.125*Psi[ijk];
dist[2*Np+n] = 0.125*Psi[ijk];
dist[3*Np+n] = 0.125*Psi[ijk];
dist[4*Np+n] = 0.125*Psi[ijk];
dist[5*Np+n] = 0.125*Psi[ijk];
dist[6*Np+n] = 0.125*Psi[ijk];
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAodd_Poisson: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson<<<NBLOCKS,NTHREADS >>>(Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_AAeven_Poisson: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Poisson_Init<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q7_Poisson_Init: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

996
hip/Stokes.cu Normal file
View File

@ -0,0 +1,996 @@
#include <stdio.h>
#include <math.h>
#include "hip/hip_runtime.h"
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
int nread;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
fq = dist[nread]; // reading the f1 data into register fq
//fp = dist[10*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
fq = dist[nread]; // reading the f2 data into register fq
//fq = dist[Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
nread = neighborList[n+2*Np]; // neighbor 4
fq = dist[nread];
//fq = dist[11*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
nread = neighborList[n+3*Np]; // neighbor 3
fq = dist[nread];
//fq = dist[2*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
//fq = dist[12*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
//fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
nread = neighborList[n+6*Np];
fq = dist[nread];
//fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
nread = neighborList[n+7*Np];
fq = dist[nread];
//fq = dist[4*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
nread = neighborList[n+8*Np];
fq = dist[nread];
//fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
nread = neighborList[n+9*Np];
fq = dist[nread];
//fq = dist[5*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
nread = neighborList[n+10*Np];
fq = dist[nread];
//fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
nread = neighborList[n+11*Np];
fq = dist[nread];
//fq = dist[6*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
nread = neighborList[n+12*Np];
fq = dist[nread];
//fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
nread = neighborList[n+13*Np];
fq = dist[nread];
//fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
nread = neighborList[n+14*Np];
fq = dist[nread];
//fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
nread = neighborList[n+15*Np];
fq = dist[nread];
//fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
//fq = dist[18*Np+n];
nread = neighborList[n+16*Np];
fq = dist[nread];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
nread = neighborList[n+17*Np];
fq = dist[nread];
//fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
nread = neighborList[n+Np];
dist[nread] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
nread = neighborList[n];
dist[nread] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
nread = neighborList[n+3*Np];
dist[nread] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
nread = neighborList[n+2*Np];
dist[nread] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
nread = neighborList[n+5*Np];
dist[nread] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
nread = neighborList[n+4*Np];
dist[nread] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
nread = neighborList[n+7*Np];
dist[nread] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
nread = neighborList[n+6*Np];
dist[nread] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
nread = neighborList[n+9*Np];
dist[nread] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
nread = neighborList[n+8*Np];
dist[nread] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
nread = neighborList[n+11*Np];
dist[nread] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
nread = neighborList[n+10*Np];
dist[nread]= fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
nread = neighborList[n+13*Np];
dist[nread] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
nread = neighborList[n+12*Np];
dist[nread] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
nread = neighborList[n+15*Np];
dist[nread] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
nread = neighborList[n+14*Np];
dist[nread] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
nread = neighborList[n+17*Np];
dist[nread] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
nread = neighborList[n+16*Np];
dist[nread] = fq;
}
}
}
__global__ void dvc_ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
fq = dist[2*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
fq = dist[1*Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
fq = dist[4*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
fq = dist[6*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
fq = dist[5*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
fq = dist[10*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
fq = dist[12*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
fq = dist[11*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
fq = dist[18*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//........................................................................
// READ THE DISTRIBUTIONS
// (read from opposite array due to previous swap operation)
//........................................................................
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
dist[1*Np+n] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
dist[2*Np+n] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
dist[3*Np+n] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
dist[4*Np+n] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
dist[5*Np+n] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
dist[6*Np+n] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
dist[7*Np+n] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
dist[8*Np+n] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
dist[9*Np+n] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
dist[10*Np+n] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
dist[11*Np+n] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
dist[12*Np+n] = fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
dist[13*Np+n] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
dist[14*Np+n] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
dist[15*Np+n] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
dist[16*Np+n] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
dist[17*Np+n] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
dist[18*Np+n] = fq;
//........................................................................
}
}
}
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAodd_StokesMRT<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_AAodd_StokesMRT: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAeven_StokesMRT<<<NBLOCKS,NTHREADS >>>(dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
hipError_t err = hipGetLastError();
if (hipSuccess != err){
printf("hip error in ScaLBL_D3Q19_AAeven_StokesMRT: %s \n",hipGetErrorString(err));
}
//cudaProfilerStop();
}

261
models/ColorModel.cpp
View File

@ -9,15 +9,33 @@ color lattice boltzmann model
#include <stdlib.h>
#include <time.h>
ScaLBL_ColorModel::ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP), Restart(0),timestep(0),timestepMax(0),tauA(0),tauB(0),rhoA(0),rhoB(0),alpha(0),beta(0),
Fx(0),Fy(0),Fz(0),flux(0),din(0),dout(0),inletA(0),inletB(0),outletA(0),outletB(0),
Nx(0),Ny(0),Nz(0),N(0),Np(0),nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),Lx(0),Ly(0),Lz(0),comm(COMM)
rank(RANK), nprocs(NP), Restart(0), timestep(0), timestepMax(0),
tauA(0), tauB(0), rhoA(0), rhoB(0), alpha(0), beta(0),
Fx(0), Fy(0), Fz(0), flux(0), din(0), dout(0),
inletA(0), inletB(0), outletA(0), outletB(0),
Nx(0), Ny(0), Nz(0), N(0), Np(0), nprocx(0), nprocy(0), nprocz(0),
BoundaryCondition(0), Lx(0), Ly(0), Lz(0), id(nullptr),
NeighborList(nullptr), dvcMap(nullptr), fq(nullptr), Aq(nullptr), Bq(nullptr),
Den(nullptr), Phi(nullptr), ColorGrad(nullptr), Velocity(nullptr), Pressure(nullptr),
comm(COMM)
{
REVERSE_FLOW_DIRECTION = false;
}
ScaLBL_ColorModel::~ScaLBL_ColorModel(){
ScaLBL_ColorModel::~ScaLBL_ColorModel()
{
delete [] id;
ScaLBL_FreeDeviceMemory( NeighborList );
ScaLBL_FreeDeviceMemory( dvcMap );
ScaLBL_FreeDeviceMemory( fq );
ScaLBL_FreeDeviceMemory( Aq );
ScaLBL_FreeDeviceMemory( Bq );
ScaLBL_FreeDeviceMemory( Den );
ScaLBL_FreeDeviceMemory( Phi );
ScaLBL_FreeDeviceMemory( Pressure );
ScaLBL_FreeDeviceMemory( Velocity );
ScaLBL_FreeDeviceMemory( ColorGrad );
}
/*void ScaLBL_ColorModel::WriteCheckpoint(const char *FILENAME, const double *cPhi, const double *cfq, int Np)
@ -238,9 +256,26 @@ void ScaLBL_ColorModel::ReadInput(){
}
}
// MeanFilter(Averages->SDs);
Minkowski Solid(Dm);
if (rank==0) printf("Initialized solid phase -- Converting to Signed Distance function \n");
CalcDist(Averages->SDs,id_solid,*Mask);
Solid.ComputeScalar(Averages->SDs,0.0);
/* save averages */
Averages->solid.V = Solid.Vi;
Averages->solid.A = Solid.Ai;
Averages->solid.H = Solid.Ji;
Averages->solid.X = Solid.Xi;
Averages->gsolid.V = Solid.Vi_global;
Averages->gsolid.A = Solid.Ai_global;
Averages->gsolid.H = Solid.Ji_global;
Averages->gsolid.X = Solid.Xi_global;
/* write to file */
if (rank == 0) {
FILE *SOLID = fopen("solid.csv","w");
fprintf(SOLID,"Vs As Hs Xs\n");
fprintf(SOLID,"%.8g %.8g %.8g %.8g\n",Solid.Vi_global,Solid.Ai_global,Solid.Ji_global,Solid.Xi_global);
fclose(SOLID);
}
if (rank == 0) cout << "Domain set." << endl;
Averages->SetParams(rhoA,rhoB,tauA,tauB,Fx,Fy,Fz,alpha,beta);
@ -254,12 +289,17 @@ void ScaLBL_ColorModel::AssignComponentLabels(double *phase)
auto LabelList = color_db->getVector<int>( "ComponentLabels" );
auto AffinityList = color_db->getVector<double>( "ComponentAffinity" );
auto WettingConvention = color_db->getWithDefault<std::string>( "WettingConvention", "none" );
NLABELS=LabelList.size();
if (NLABELS != AffinityList.size()){
ERROR("Error: ComponentLabels and ComponentAffinity must be the same length! \n");
}
if (WettingConvention == "SCAL"){
for (size_t idx=0; idx<NLABELS; idx++) AffinityList[idx] *= -1.0;
}
double label_count[NLABELS];
double label_count_global[NLABELS];
// Assign the labels
@ -391,11 +431,13 @@ void ScaLBL_ColorModel::Create(){
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
delete [] neighborList;
// initialize phi based on PhaseLabel (include solid component labels)
double *PhaseLabel;
PhaseLabel = new double[N];
AssignComponentLabels(PhaseLabel);
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
delete [] PhaseLabel;
}
/********************************************************
@ -489,6 +531,121 @@ void ScaLBL_ColorModel::Initialize(){
ScaLBL_CopyToHost(Averages->Phi.data(),Phi,N*sizeof(double));
}
double ScaLBL_ColorModel::Run(int returntime){
int nprocs=nprocx*nprocy*nprocz;
//************ MAIN ITERATION LOOP ***************************************/
comm.barrier();
PROFILE_START("Loop");
//std::shared_ptr<Database> analysis_db;
bool Regular = false;
auto current_db = db->cloneDatabase();
auto t1 = std::chrono::system_clock::now();
int START_TIMESTEP = timestep;
int EXIT_TIMESTEP = min(timestepMax,returntime);
while (timestep < EXIT_TIMESTEP ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
// *************ODD TIMESTEP*************
timestep++;
// Compute the Phase indicator field
// Read for Aq, Bq happens in this routine (requires communication)
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
ScaLBL_D3Q7_AAodd_PhaseField(NeighborList, dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FROM NORMAL
if (BoundaryCondition > 0 && BoundaryCondition < 5){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
// Halo exchange for phase field
ScaLBL_Comm_Regular->SendHalo(Phi);
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm_Regular->RecvHalo(Phi);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set BCs
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 5){
ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
}
ScaLBL_D3Q19_AAodd_Color(NeighborList, dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->Barrier();
// *************EVEN TIMESTEP*************
timestep++;
// Compute the Phase indicator field
ScaLBL_Comm->BiSendD3Q7AA(Aq,Bq); //READ FROM NORMAL
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->BiRecvD3Q7AA(Aq,Bq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
ScaLBL_D3Q7_AAeven_PhaseField(dvcMap, Aq, Bq, Den, Phi, 0, ScaLBL_Comm->LastExterior(), Np);
// Perform the collision operation
ScaLBL_Comm->SendD3Q19AA(fq); //READ FORM NORMAL
// Halo exchange for phase field
if (BoundaryCondition > 0 && BoundaryCondition < 5){
ScaLBL_Comm->Color_BC_z(dvcMap, Phi, Den, inletA, inletB);
ScaLBL_Comm->Color_BC_Z(dvcMap, Phi, Den, outletA, outletB);
}
ScaLBL_Comm_Regular->SendHalo(Phi);
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm_Regular->RecvHalo(Phi);
ScaLBL_Comm->RecvD3Q19AA(fq); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 3){
ScaLBL_Comm->D3Q19_Pressure_BC_z(NeighborList, fq, din, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 4){
din = ScaLBL_Comm->D3Q19_Flux_BC_z(NeighborList, fq, flux, timestep);
ScaLBL_Comm->D3Q19_Pressure_BC_Z(NeighborList, fq, dout, timestep);
}
else if (BoundaryCondition == 5){
ScaLBL_Comm->D3Q19_Reflection_BC_z(fq);
ScaLBL_Comm->D3Q19_Reflection_BC_Z(fq);
}
ScaLBL_D3Q19_AAeven_Color(dvcMap, fq, Aq, Bq, Den, Phi, Velocity, rhoA, rhoB, tauA, tauB,
alpha, beta, Fx, Fy, Fz, Nx, Nx*Ny, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_Comm->Barrier();
//************************************************************************
}
PROFILE_STOP("Update");
PROFILE_STOP("Loop");
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
// Compute the walltime per timestep
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / (timestep - START_TIMESTEP);
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("CPU time = %f \n", cputime);
if (rank==0) printf("Lattice update rate (per core)= %f MLUPS \n", MLUPS);
return(MLUPS);
MLUPS *= nprocs;
}
void ScaLBL_ColorModel::Run(){
int nprocs=nprocx*nprocy*nprocz;
const RankInfoStruct rank_info(rank,nprocx,nprocy,nprocz);
@ -538,7 +695,6 @@ void ScaLBL_ColorModel::Run(){
if (color_db->keyExists( "krA_morph_factor" )){
KRA_MORPH_FACTOR = color_db->getScalar<double>( "krA_morph_factor" );
}
/* defaults for simulation protocols */
auto protocol = color_db->getWithDefault<std::string>( "protocol", "none" );
if (protocol == "image sequence"){
@ -583,7 +739,7 @@ void ScaLBL_ColorModel::Run(){
if (analysis_db->keyExists( "seed_water" )){
seed_water = analysis_db->getScalar<double>( "seed_water" );
if (rank == 0) printf("Seed water in oil %f (seed_water) \n",seed_water);
USE_SEED = true;
ASSERT(protocol == "seed water");
}
if (analysis_db->keyExists( "morph_delta" )){
morph_delta = analysis_db->getScalar<double>( "morph_delta" );
@ -614,7 +770,6 @@ void ScaLBL_ColorModel::Run(){
MAX_MORPH_TIMESTEPS = analysis_db->getScalar<int>( "max_morph_timesteps" );
}
if (rank==0){
printf("********************************************************\n");
if (protocol == "image sequence"){
@ -651,20 +806,15 @@ void ScaLBL_ColorModel::Run(){
fflush(stdout);
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
comm.barrier();
PROFILE_START("Loop");
//std::shared_ptr<Database> analysis_db;
bool Regular = false;
auto current_db = db->cloneDatabase();
runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
//analysis.createThreads( analysis_method, 4 );
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
@ -997,10 +1147,10 @@ void ScaLBL_ColorModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_Comm->Barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
@ -1080,7 +1230,6 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
ScaLBL_CopyToHost(phase.data(), Phi, N*sizeof(double));
// Extract only the connected part of NWP
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,Averages->SDs,vF,vS,phase_label,Dm->Comm);
comm.barrier();
@ -1202,6 +1351,7 @@ double ScaLBL_ColorModel::MorphOpenConnected(double target_volume_change){
}
return(volume_change);
}
double ScaLBL_ColorModel::SeedPhaseField(const double seed_water_in_oil){
srand(time(NULL));
double mass_loss =0.f;
@ -1283,7 +1433,7 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
double vF = 0.f;
double vS = 0.f;
double delta_volume;
double WallFactor = 0.0;
double WallFactor = 1.0;
bool USE_CONNECTED_NWP = false;
DoubleArray phase(Nx,Ny,Nz);
@ -1306,6 +1456,11 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
}
}
double volume_initial = Dm->Comm.sumReduce( count);
double PoreVolume = Dm->Volume*Dm->Porosity();
/*ensure target isn't an absurdly small fraction of pore volume */
if (volume_initial < target_delta_volume*PoreVolume){
volume_initial = target_delta_volume*PoreVolume;
}
/*
sprintf(LocalRankFilename,"phi_initial.%05i.raw",rank);
FILE *INPUT = fopen(LocalRankFilename,"wb");
@ -1317,7 +1472,6 @@ double ScaLBL_ColorModel::MorphInit(const double beta, const double target_delta
double volume_connected = 0.0;
double second_biggest = 0.0;
if (USE_CONNECTED_NWP){
BlobIDstruct new_index;
ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,rank_info,phase,Averages->SDs,vF,vS,phase_label,comm);
comm.barrier();
@ -1566,3 +1720,68 @@ void ScaLBL_ColorModel::WriteDebug(){
fclose(CGZ_FILE);
*/
}
FlowAdaptor::FlowAdaptor(ScaLBL_ColorModel &M){
Nx = M.Dm->Nx;
Ny = M.Dm->Ny;
Nz = M.Dm->Nz;
timestep=-1;
timestep_previous=-1;
phi.resize(Nx,Ny,Nz); phi.fill(0); // phase indicator field
phi_t.resize(Nx,Ny,Nz); phi_t.fill(0); // time derivative for the phase indicator field
}
FlowAdaptor::~FlowAdaptor(){
}
double FlowAdaptor::MoveInterface(ScaLBL_ColorModel &M){
double INTERFACE_CUTOFF = M.color_db->getWithDefault<double>( "move_interface_cutoff", 0.975 );
double MOVE_INTERFACE_FACTOR = M.color_db->getWithDefault<double>( "move_interface_factor", 10.0 );
ScaLBL_CopyToHost( phi.data(), M.Phi, Nx*Ny*Nz* sizeof( double ) );
/* compute the local derivative of phase indicator field */
double beta = M.beta;
double factor = 0.5/beta;
double total_interface_displacement = 0.0;
double total_interface_sites = 0.0;
for (int n=0; n<Nx*Ny*Nz; n++){
/* compute the distance to the interface */
double value1 = M.Averages->Phi(n);
double dist1 = factor*log((1.0+value1)/(1.0-value1));
double value2 = phi(n);
double dist2 = factor*log((1.0+value2)/(1.0-value2));
phi_t(n) = value2;
if (value1 < INTERFACE_CUTOFF && value1 > -1*INTERFACE_CUTOFF && value2 < INTERFACE_CUTOFF && value2 > -1*INTERFACE_CUTOFF ){
/* time derivative of distance */
double dxdt = 0.125*(dist2-dist1);
/* extrapolate to move the distance further */
double dist3 = dist2 + MOVE_INTERFACE_FACTOR*dxdt;
/* compute the new phase interface */
phi_t(n) = (2.f*(exp(-2.f*beta*(dist3)))/(1.f+exp(-2.f*beta*(dist3))) - 1.f);
total_interface_displacement += fabs(MOVE_INTERFACE_FACTOR*dxdt);
total_interface_sites += 1.0;
}
}
ScaLBL_CopyToDevice( M.Phi, phi_t.data(), Nx*Ny*Nz* sizeof( double ) );
/* ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, 0, ScaLBL_Comm->LastExterior(), Np);
ScaLBL_PhaseField_Init(dvcMap, Phi, Den, Aq, Bq, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
if (BoundaryCondition == 1 || BoundaryCondition == 2 || BoundaryCondition == 3 || BoundaryCondition == 4){
if (Dm->kproc()==0){
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,0);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,1);
ScaLBL_SetSlice_z(Phi,1.0,Nx,Ny,Nz,2);
}
if (Dm->kproc() == nprocz-1){
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-1);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-2);
ScaLBL_SetSlice_z(Phi,-1.0,Nx,Ny,Nz,Nz-3);
}
}
*/
}

20
models/ColorModel.h
View File

@ -16,6 +16,10 @@ Implementation of color lattice boltzmann model
#include "ProfilerApp.h"
#include "threadpool/thread_pool.h"
#ifndef ScaLBL_ColorModel_INC
#define ScaLBL_ColorModel_INC
class ScaLBL_ColorModel{
public:
ScaLBL_ColorModel(int RANK, int NP, const Utilities::MPI& COMM);
@ -29,7 +33,9 @@ public:
void Create();
void Initialize();
void Run();
double Run(int returntime);
void WriteDebug();
void getPhaseField(DoubleArray &f);
bool Restart,pBC;
bool REVERSE_FLOW_DIRECTION;
@ -86,3 +92,17 @@ private:
double MorphOpenConnected(double target_volume_change);
};
class FlowAdaptor{
public:
FlowAdaptor(ScaLBL_ColorModel &M);
~FlowAdaptor();
double MoveInterface(ScaLBL_ColorModel &M);
DoubleArray phi;
DoubleArray phi_t;
private:
int Nx, Ny, Nz;
int timestep;
int timestep_previous;
};
#endif

10
models/DFHModel.cpp
View File

@ -490,14 +490,10 @@ void ScaLBL_DFHModel::Run(){
if (rank==0) printf("********************************************************\n");
if (rank==0) printf("No. of timesteps: %i \n", timestepMax);
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
auto t1 = std::chrono::system_clock::now();
bool Regular = true;
PROFILE_START("Loop");
runAnalysis analysis( analysis_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
@ -589,10 +585,10 @@ void ScaLBL_DFHModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;
if (rank==0) printf("********************************************************\n");

949
models/FreeLeeModel.cpp
View File

File diff suppressed because it is too large Load Diff

33
models/FreeLeeModel.h
View File

@ -16,6 +16,9 @@ Implementation of Lee et al JCP 2016 lattice boltzmann model
#include "common/ScaLBL.h"
#include "common/WideHalo.h"
#ifndef ScaLBL_FreeLeeModel_INC
#define ScaLBL_FreeLeeModel_INC
class ScaLBL_FreeLeeModel{
public:
ScaLBL_FreeLeeModel(int RANK, int NP, const Utilities::MPI& COMM);
@ -26,16 +29,27 @@ public:
void ReadParams(std::shared_ptr<Database> db0);
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run();
void WriteDebug();
void Create_TwoFluid();
void Initialize_TwoFluid();
double Run_TwoFluid(int returntime);
void WriteDebug_TwoFluid();
void Create_SingleFluid();
void Initialize_SingleFluid();
void Run_SingleFluid();
void WriteDebug_SingleFluid();
// test utilities
void Create_DummyPhase_MGTest();
void MGTest();
bool Restart,pBC;
int timestep,timestepMax;
int BoundaryCondition;
double tauA,tauB,rhoA,rhoB;
double W,gamma;
double tau, rho0;//only for single-fluid Lee model
double tauM;//relaxation time for phase field (or mass)
double W,gamma,kappa,beta;
double Fx,Fy,Fz,flux;
double din,dout,inletA,inletB,outletA,outletB;
@ -61,12 +75,16 @@ public:
signed char *id;
int *NeighborList;
int *dvcMap;
double *fq, *hq;
double *gqbar, *hq;
double *mu_phi, *Den, *Phi;
double *ColorGrad;
double *Velocity;
double *Pressure;
void getPhase(DoubleArray &PhaseValues);
void getPotential(DoubleArray &PressureValues, DoubleArray &MuValues);
void getVelocity(DoubleArray &Vx, DoubleArray &Vy, DoubleArray &Vz);
DoubleArray SignDist;
private:
@ -81,6 +99,7 @@ private:
//int rank,nprocs;
void LoadParams(std::shared_ptr<Database> db0);
void AssignComponentLabels_ChemPotential_ColorGrad();
};
#endif

7
models/GreyscaleColorModel.cpp
View File

@ -910,10 +910,8 @@ void ScaLBL_GreyscaleColorModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
//************ MAIN ITERATION LOOP ***************************************/
@ -923,6 +921,7 @@ void ScaLBL_GreyscaleColorModel::Run(){
auto current_db = db->cloneDatabase();
//runAnalysis analysis( current_db, rank_info, ScaLBL_Comm, Dm, Np, Regular, Map );
//analysis.createThreads( analysis_method, 4 );
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax ) {
//if ( rank==0 ) { printf("Running timestep %i (%i MB)\n",timestep+1,(int)(Utilities::getMemoryUsage()/1048576)); }
PROFILE_START("Update");
@ -1319,10 +1318,10 @@ void ScaLBL_GreyscaleColorModel::Run(){
PROFILE_SAVE("lbpm_color_simulator",1);
//************************************************************************
ScaLBL_Comm->Barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

7
models/GreyscaleModel.cpp
View File

@ -485,10 +485,8 @@ void ScaLBL_GreyscaleModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier();
comm.barrier();
starttime = MPI_Wtime();
//.........................................
Minkowski Morphology(Mask);
@ -500,6 +498,7 @@ void ScaLBL_GreyscaleModel::Run(){
double rlx_eff = 1.0/tau_eff;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
// *************ODD TIMESTEP*************//
@ -744,10 +743,10 @@ void ScaLBL_GreyscaleModel::Run(){
//************************************************************************
ScaLBL_DeviceBarrier();
comm.barrier();
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

6
models/IonModel.cpp
View File

@ -784,7 +784,7 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
//.......create and start timer............
//double starttime,stoptime,cputime;
//ScaLBL_Comm->Barrier(); comm.barrier();
//starttime = MPI_Wtime();
//auto t1 = std::chrono::system_clock::now();
for (int ic=0; ic<number_ion_species; ic++){
timestep=0;
@ -886,10 +886,10 @@ void ScaLBL_IonModel::Run(double *Velocity, double *ElectricField){
ScaLBL_D3Q7_Ion_ChargeDensity(Ci, ChargeDensity, IonValence[ic], ic, 0, ScaLBL_Comm->LastExterior(), Np);
}
//************************************************************************/
//stoptime = MPI_Wtime();
//if (rank==0) printf("-------------------------------------------------------------------\n");
//// Compute the walltime per timestep
//cputime = (stoptime - starttime)/timestep;
//auto t2 = std::chrono::system_clock::now();
//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
//// Performance obtained from each node
//double MLUPS = double(Np)/cputime/1000000;

12
models/MRTModel.cpp
View File

@ -26,6 +26,8 @@ void ScaLBL_MRTModel::ReadParams(string filename){
tolerance = 1.0e-8;
Fx = Fy = 0.0;
Fz = 1.0e-5;
dout = 1.0;
din = 1.0;
// Color Model parameters
if (mrt_db->keyExists( "timestepMax" )){
@ -194,7 +196,8 @@ void ScaLBL_MRTModel::Create(){
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
comm.barrier();
double MLUPS = ScaLBL_Comm->GetPerformance(NeighborList,fq,Np);
printf(" MLPUS=%f from rank %i\n",MLUPS,rank);
}
void ScaLBL_MRTModel::Initialize(){
@ -227,14 +230,13 @@ void ScaLBL_MRTModel::Run(){
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_DeviceBarrier(); comm.barrier();
starttime = MPI_Wtime();
if (rank==0) printf("Beginning AA timesteps, timestepMax = %i \n", timestepMax);
if (rank==0) printf("********************************************************\n");
timestep=0;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
timestep++;
@ -351,10 +353,10 @@ void ScaLBL_MRTModel::Run(){
}
}
//************************************************************************/
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

12
models/MultiPhysController.cpp
View File

@ -2,7 +2,7 @@
ScaLBL_Multiphys_Controller::ScaLBL_Multiphys_Controller(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK),nprocs(NP),Restart(0),timestepMax(0),num_iter_Stokes(0),num_iter_Ion(0),
analysis_interval(0),visualization_interval(0),tolerance(0),comm(COMM)
analysis_interval(0),visualization_interval(0),tolerance(0),time_conv_max(0),comm(COMM)
{
}
@ -25,6 +25,7 @@ void ScaLBL_Multiphys_Controller::ReadParams(string filename){
analysis_interval = 500;
visualization_interval = 10000;
tolerance = 1.0e-6;
time_conv_max = 0.0;
// load input parameters
if (study_db->keyExists( "timestepMax" )){
@ -135,3 +136,12 @@ vector<int> ScaLBL_Multiphys_Controller::getIonNumIter_PNP_coupling(double Stoke
}
return num_iter_ion;
}
void ScaLBL_Multiphys_Controller::getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv){
//Return maximum of the time converting factor from Stokes and ion solvers
vector<double> TimeConv;
TimeConv.assign(IonTimeConv.begin(),IonTimeConv.end());
TimeConv.insert(TimeConv.begin(),StokesTimeConv);
time_conv_max = *max_element(TimeConv.begin(),TimeConv.end());
}

2
models/MultiPhysController.h
View File

@ -27,6 +27,7 @@ public:
int getStokesNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
vector<int> getIonNumIter_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
//void getIonNumIter_PNP_coupling(double StokesTimeConv,vector<double> &IonTimeConv,vector<int> &IonTimeMax);
void getTimeConvMax_PNP_coupling(double StokesTimeConv,const vector<double> &IonTimeConv);
bool Restart;
int timestepMax;
@ -35,6 +36,7 @@ public:
int analysis_interval;
int visualization_interval;
double tolerance;
double time_conv_max;
//double SchmidtNum;//Schmidt number = kinematic_viscosity/mass_diffusivity
int rank,nprocs;

195
models/PoissonSolver.cpp
View File

@ -8,8 +8,11 @@
ScaLBL_Poisson::ScaLBL_Poisson(int RANK, int NP, const Utilities::MPI& COMM):
rank(RANK), nprocs(NP),timestep(0),timestepMax(0),tau(0),k2_inv(0),tolerance(0),h(0),
epsilon0(0),epsilon0_LB(0),epsilonR(0),epsilon_LB(0),Vin(0),Vout(0),Nx(0),Ny(0),Nz(0),N(0),Np(0),analysis_interval(0),
chargeDen_dummy(0),WriteLog(0),
nprocx(0),nprocy(0),nprocz(0),BoundaryCondition(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),comm(COMM)
chargeDen_dummy(0),WriteLog(0),nprocx(0),nprocy(0),nprocz(0),
BoundaryConditionInlet(0),BoundaryConditionOutlet(0),BoundaryConditionSolid(0),Lx(0),Ly(0),Lz(0),
Vin0(0),freqIn(0),t0_In(0),Vin_Type(0),Vout0(0),freqOut(0),t0_Out(0),Vout_Type(0),
TestPeriodic(0),TestPeriodicTime(0),TestPeriodicTimeConv(0),TestPeriodicSaveInterval(0),
comm(COMM)
{
}
@ -33,10 +36,12 @@ void ScaLBL_Poisson::ReadParams(string filename){
epsilonR = 78.4;//default dielectric constant of water
epsilon_LB = epsilon0_LB*epsilonR;//electric permittivity
analysis_interval = 1000;
Vin = 1.0; //Boundary-z (inlet) electric potential
Vout = 1.0; //Boundary-Z (outlet) electric potential
chargeDen_dummy = 1.0e-3;//For debugging;unit=[C/m^3]
WriteLog = false;
TestPeriodic = false;
TestPeriodicTime = 1.0;//unit: [sec]
TestPeriodicTimeConv = 0.01; //unit [sec/lt]
TestPeriodicSaveInterval = 0.1; //unit [sec]
// LB-Poisson Model parameters
if (electric_db->keyExists( "timestepMax" )){
@ -57,6 +62,18 @@ void ScaLBL_Poisson::ReadParams(string filename){
if (electric_db->keyExists( "WriteLog" )){
WriteLog = electric_db->getScalar<bool>( "WriteLog" );
}
if (electric_db->keyExists( "TestPeriodic" )){
TestPeriodic = electric_db->getScalar<bool>( "TestPeriodic" );
}
if (electric_db->keyExists( "TestPeriodicTime" )){
TestPeriodicTime = electric_db->getScalar<double>( "TestPeriodicTime" );
}
if (electric_db->keyExists( "TestPeriodicTimeConv" )){
TestPeriodicTimeConv = electric_db->getScalar<double>( "TestPeriodicTimeConv" );
}
if (electric_db->keyExists( "TestPeriodicSaveInterval" )){
TestPeriodicSaveInterval = electric_db->getScalar<double>( "TestPeriodicSaveInterval" );
}
// Read solid boundary condition specific to Poisson equation
BoundaryConditionSolid = 1;
@ -65,10 +82,15 @@ void ScaLBL_Poisson::ReadParams(string filename){
}
// Read boundary condition for electric potential
// BC = 0: normal periodic BC
// BC = 1: fixed inlet and outlet potential
BoundaryCondition = 0;
if (electric_db->keyExists( "BC" )){
BoundaryCondition = electric_db->getScalar<int>( "BC" );
// BC = 1: fixed electric potential
// BC = 2: sine/cosine periodic electric potential (need extra input parameters)
BoundaryConditionInlet = 0;
BoundaryConditionOutlet = 0;
if (electric_db->keyExists( "BC_Inlet" )){
BoundaryConditionInlet = electric_db->getScalar<int>( "BC_Inlet" );
}
if (electric_db->keyExists( "BC_Outlet" )){
BoundaryConditionOutlet = electric_db->getScalar<int>( "BC_Outlet" );
}
// Read domain parameters
@ -117,8 +139,17 @@ void ScaLBL_Poisson::SetDomain(){
for (int i=0; i<Nx*Ny*Nz; i++) Dm->id[i] = 1; // initialize this way
//Averages = std::shared_ptr<TwoPhase> ( new TwoPhase(Dm) ); // TwoPhase analysis object
comm.barrier();
Dm->BoundaryCondition = BoundaryCondition;
Mask->BoundaryCondition = BoundaryCondition;
if (BoundaryConditionInlet==0 && BoundaryConditionOutlet==0){
Dm->BoundaryCondition = 0;
Mask->BoundaryCondition = 0;
}
else if (BoundaryConditionInlet>0 && BoundaryConditionOutlet>0){
Dm->BoundaryCondition = 1;
Mask->BoundaryCondition = 1;
}
else {//i.e. non-periodic and periodic BCs are mixed
ERROR("Error: check the type of inlet and outlet boundary condition! Mixed periodic and non-periodic BCs are found!\n");
}
Dm->CommInit();
comm.barrier();
@ -343,15 +374,91 @@ void ScaLBL_Poisson::Create(){
void ScaLBL_Poisson::Potential_Init(double *psi_init){
if (BoundaryCondition==1){
//set up default boundary input parameters
Vin0 = Vout0 = 1.0; //unit: [V]
freqIn = freqOut = 50.0; //unit: [Hz]
t0_In = t0_Out = 0.0; //unit: [sec]
Vin_Type = Vout_Type = 1; //1->sin; 2->cos
Vin = 1.0; //Boundary-z (inlet) electric potential
Vout = 1.0; //Boundary-Z (outlet) electric potential
if (BoundaryConditionInlet>0){
switch (BoundaryConditionInlet){
case 1:
if (electric_db->keyExists( "Vin" )){
Vin = electric_db->getScalar<double>( "Vin" );
}
if (rank==0) printf("LB-Poisson Solver: inlet boundary; fixed electric potential Vin = %.3g [V]\n",Vin);
break;
case 2:
if (electric_db->keyExists( "Vin0" )){//voltage amplitude; unit: Volt
Vin0 = electric_db->getScalar<double>( "Vin0" );
}
if (electric_db->keyExists( "freqIn" )){//unit: Hz
freqIn = electric_db->getScalar<double>( "freqIn" );
}
if (electric_db->keyExists( "t0_In" )){//timestep shift, unit: lt
t0_In = electric_db->getScalar<double>( "t0_In" );
}
if (electric_db->keyExists( "Vin_Type" )){
//type=1 -> sine
//tyep=2 -> cosine
Vin_Type = electric_db->getScalar<int>( "Vin_Type" );
if (Vin_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vin_Type is currently not supported! \n");
}
if (rank==0){
if (Vin_Type==1){
printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vin0,freqIn,t0_In);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
}
else if (Vin_Type==2){
printf("LB-Poisson Solver: inlet boundary; periodic electric potential Vin = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V] \n",Vin0,freqIn,t0_In);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vin0,freqIn,t0_In);
}
}
break;
}
}
if (BoundaryConditionOutlet>0){
switch (BoundaryConditionOutlet){
case 1:
if (electric_db->keyExists( "Vout" )){
Vout = electric_db->getScalar<double>( "Vout" );
}
if (rank==0) printf("LB-Poisson Solver: outlet boundary; fixed electric potential Vout = %.3g [V] \n",Vout);
break;
case 2:
if (electric_db->keyExists( "Vout0" )){//voltage amplitude; unit: Volt
Vout0 = electric_db->getScalar<double>( "Vout0" );
}
if (electric_db->keyExists( "freqOut" )){//unit: Hz
freqOut = electric_db->getScalar<double>( "freqOut" );
}
if (electric_db->keyExists( "t0_Out" )){//timestep shift, unit: lt
t0_Out = electric_db->getScalar<double>( "t0_Out" );
}
if (electric_db->keyExists( "Vout_Type" )){
//type=1 -> sine
//tyep=2 -> cosine
Vout_Type = electric_db->getScalar<int>( "Vout_Type" );
if (Vout_Type>2 || Vin_Type<=0) ERROR("Error: user-input Vout_Type is currently not supported! \n");
}
if (rank==0){
if (Vout_Type==1){
printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Sin[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
}
else if (Vout_Type==2){
printf("LB-Poisson Solver: outlet boundary; periodic electric potential Vout = %.3g*Cos[2*pi*%.3g*(t+%.3g)] [V]\n",Vout0,freqOut,t0_Out);
printf(" V0 = %.3g [V], frequency = %.3g [Hz], timestep shift = %.3g [sec] \n",Vout0,freqOut,t0_Out);
}
}
break;
}
}
//By default only periodic BC is applied and Vin=Vout=1.0, i.e. there is no potential gradient along Z-axis
if (BoundaryConditionInlet==2) Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,0);
if (BoundaryConditionOutlet==2) Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,0);
double slope = (Vout-Vin)/(Nz-2);
double psi_linearized;
for (int k=0;k<Nz;k++){
@ -375,10 +482,15 @@ void ScaLBL_Poisson::Potential_Init(double *psi_init){
}
}
double ScaLBL_Poisson::getBoundaryVoltagefromPeriodicBC(double V0, double freq, double t0, int V_type, int time_step){
return V0*(V_type==1)*sin(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv))+V0*(V_type==2)*cos(2.0*M_PI*freq*time_conv*(time_step+t0/time_conv));
}
void ScaLBL_Poisson::Initialize(){
void ScaLBL_Poisson::Initialize(double time_conv_from_Study){
/*
* This function initializes model
* "time_conv_from_Study" is the phys to LB time conversion factor, unit=[sec/lt]
* which is used for periodic voltage input for inlet and outlet boundaries
*/
if (rank==0) printf ("LB-Poisson Solver: initializing D3Q7 distributions\n");
//NOTE the initialization involves two steps:
@ -386,6 +498,7 @@ void ScaLBL_Poisson::Initialize(){
//2. Initialize electric potential for pore nodes
double *psi_host;
psi_host = new double [Nx*Ny*Nz];
time_conv = time_conv_from_Study;
AssignSolidBoundary(psi_host);//step1
Potential_Init(psi_host);//step2
ScaLBL_CopyToDevice(Psi, psi_host, Nx*Ny*Nz*sizeof(double));
@ -405,12 +518,12 @@ void ScaLBL_Poisson::Initialize(){
//}
}
void ScaLBL_Poisson::Run(double *ChargeDensity){
void ScaLBL_Poisson::Run(double *ChargeDensity, int timestep_from_Study){
//.......create and start timer............
//double starttime,stoptime,cputime;
//ScaLBL_Comm->Barrier(); comm.barrier();
//starttime = MPI_Wtime();
//comm.barrier();
//auto t1 = std::chrono::system_clock::now();
timestep=0;
double error = 1.0;
@ -420,13 +533,13 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){
// *************ODD TIMESTEP*************//
timestep++;
SolveElectricPotentialAAodd();//update electric potential
SolveElectricPotentialAAodd(timestep_from_Study);//update electric potential
SolvePoissonAAodd(ChargeDensity);//perform collision
ScaLBL_Comm->Barrier(); comm.barrier();
// *************EVEN TIMESTEP*************//
timestep++;
SolveElectricPotentialAAeven();//update electric potential
SolveElectricPotentialAAeven(timestep_from_Study);//update electric potential
SolvePoissonAAeven(ChargeDensity);//perform collision
ScaLBL_Comm->Barrier(); comm.barrier();
//************************************************************************/
@ -466,11 +579,11 @@ void ScaLBL_Poisson::Run(double *ChargeDensity){
}
//************************************************************************/
//stoptime = MPI_Wtime();
////if (rank==0) printf("LB-Poission Solver: a steady-state solution is obtained\n");
////if (rank==0) printf("---------------------------------------------------------------------------\n");
//// Compute the walltime per timestep
//cputime = (stoptime - starttime)/timestep;
//auto t2 = std::chrono::system_clock::now();
//double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
//// Performance obtained from each node
//double MLUPS = double(Np)/cputime/1000000;
@ -506,29 +619,65 @@ void ScaLBL_Poisson::getConvergenceLog(int timestep,double error){
}
}
void ScaLBL_Poisson::SolveElectricPotentialAAodd(){
void ScaLBL_Poisson::SolveElectricPotentialAAodd(int timestep_from_Study){
ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FROM NORMAL
ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 1){
if (BoundaryConditionInlet > 0){
switch (BoundaryConditionInlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
case 2:
Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
}
}
if (BoundaryConditionOutlet > 0){
switch (BoundaryConditionOutlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
case 2:
Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
}
}
//-------------------------//
ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(NeighborList, dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);
}
void ScaLBL_Poisson::SolveElectricPotentialAAeven(){
void ScaLBL_Poisson::SolveElectricPotentialAAeven(int timestep_from_Study){
ScaLBL_Comm->SendD3Q7AA(fq, 0); //READ FORM NORMAL
ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, ScaLBL_Comm->FirstInterior(), ScaLBL_Comm->LastInterior(), Np);
ScaLBL_Comm->RecvD3Q7AA(fq, 0); //WRITE INTO OPPOSITE
ScaLBL_Comm->Barrier();
// Set boundary conditions
if (BoundaryCondition == 1){
if (BoundaryConditionInlet > 0){
switch (BoundaryConditionInlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
case 2:
Vin = getBoundaryVoltagefromPeriodicBC(Vin0,freqIn,t0_In,Vin_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_z(NeighborList, fq, Vin, timestep);
break;
}
}
if (BoundaryConditionOutlet > 0){
switch (BoundaryConditionOutlet){
case 1:
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
case 2:
Vout = getBoundaryVoltagefromPeriodicBC(Vout0,freqOut,t0_Out,Vout_Type,timestep_from_Study);
ScaLBL_Comm->D3Q7_Poisson_Potential_BC_Z(NeighborList, fq, Vout, timestep);
break;
}
}
//-------------------------//
ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(dvcMap, fq, Psi, 0, ScaLBL_Comm->LastExterior(), Np);

21
models/PoissonSolver.h
View File

@ -9,6 +9,7 @@
#include <exception>
#include <stdexcept>
#include <fstream>
#include <cmath>
#include "common/ScaLBL.h"
#include "common/Communication.h"
@ -16,6 +17,7 @@
#include "analysis/Minkowski.h"
#include "ProfilerApp.h"
#define _USE_MATH_DEFINES
#ifndef ScaLBL_POISSON_INC
#define ScaLBL_POISSON_INC
@ -30,8 +32,8 @@ public:
void SetDomain();
void ReadInput();
void Create();
void Initialize();
void Run(double *ChargeDensity);
void Initialize(double time_conv_from_Study);
void Run(double *ChargeDensity,int timestep_from_Study);
void getElectricPotential(DoubleArray &ReturnValues);
void getElectricPotential_debug(int timestep);
void getElectricField(DoubleArray &Values_x, DoubleArray &Values_y, DoubleArray &Values_z);
@ -41,7 +43,8 @@ public:
//bool Restart,pBC;
int timestep,timestepMax;
int analysis_interval;
int BoundaryCondition;
int BoundaryConditionInlet;
int BoundaryConditionOutlet;
int BoundaryConditionSolid;
double tau;
double tolerance;
@ -50,11 +53,18 @@ public:
double Vin, Vout;
double chargeDen_dummy;//for debugging
bool WriteLog;
double Vin0,freqIn,t0_In,Vin_Type;
double Vout0,freqOut,t0_Out,Vout_Type;
bool TestPeriodic;
double TestPeriodicTime;//unit: [sec]
double TestPeriodicTimeConv; //unit [sec/lt]
double TestPeriodicSaveInterval; //unit [sec]
int Nx,Ny,Nz,N,Np;
int rank,nprocx,nprocy,nprocz,nprocs;
double Lx,Ly,Lz;
double h;//image resolution
double time_conv;//phys to LB time converting factor; unit=[sec/lt]
std::shared_ptr<Domain> Dm; // this domain is for analysis
std::shared_ptr<Domain> Mask; // this domain is for lbm
@ -91,12 +101,13 @@ private:
void AssignSolidBoundary(double *poisson_solid);
void Potential_Init(double *psi_init);
void ElectricField_LB_to_Phys(DoubleArray &Efield_reg);
void SolveElectricPotentialAAodd();
void SolveElectricPotentialAAeven();
void SolveElectricPotentialAAodd(int timestep_from_Study);
void SolveElectricPotentialAAeven(int timestep_from_Study);
//void SolveElectricField();
void SolvePoissonAAodd(double *ChargeDensity);
void SolvePoissonAAeven(double *ChargeDensity);
void getConvergenceLog(int timestep,double error);
double getBoundaryVoltagefromPeriodicBC(double V0,double freq,double t0,int V_type,int time_step);
};
#endif

8
models/StokesModel.cpp
View File

@ -573,16 +573,14 @@ void ScaLBL_StokesModel::Run(){
}
}
//.......create and start timer............
double starttime,stoptime,cputime;
ScaLBL_Comm->Barrier(); comm.barrier();
starttime = MPI_Wtime();
if (rank==0) printf("****************************************************************\n");
if (rank==0) printf("LB Single-Fluid Navier-Stokes Solver: timestepMax = %i\n", timestepMax);
if (rank==0) printf("****************************************************************\n");
timestep=0;
double error = 1.0;
double flow_rate_previous = 0.0;
auto t1 = std::chrono::system_clock::now();
while (timestep < timestepMax && error > tolerance) {
//************************************************************************/
timestep++;
@ -700,10 +698,10 @@ void ScaLBL_StokesModel::Run(){
}
}
//************************************************************************/
stoptime = MPI_Wtime();
if (rank==0) printf("-------------------------------------------------------------------\n");
// Compute the walltime per timestep
cputime = (stoptime - starttime)/timestep;
auto t2 = std::chrono::system_clock::now();
double cputime = std::chrono::duration<double>( t2 - t1 ).count() / timestep;
// Performance obtained from each node
double MLUPS = double(Np)/cputime/1000000;

1
sample_scripts/config_build_eos
View File

@ -33,7 +33,6 @@ cmake \
-D CMAKE_CXX_STANDARD=14 \
-D USE_TIMER=false \
-D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_CUDA=0 \

1
sample_scripts/config_build_rhea
View File

@ -27,7 +27,6 @@ cmake \
-D CMAKE_CXX_STD=11 \
-D USE_TIMER=false \
-D TIMER_DIRECTORY=${HOME}/timerutility/build/opt \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_CUDA=0 \

1
sample_scripts/config_build_titan
View File

@ -25,7 +25,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_SILO=1 \

1
sample_scripts/config_build_titan_silo
View File

@ -20,7 +20,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/opt/gcc/6.3.0/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D USE_NETCDF=0 \

1
sample_scripts/config_poplar_hip
View File

@ -36,6 +36,5 @@ cmake \
# MPI_THREAD_MULTIPLE=1 MV2_USE_RDMA_CM=0 MV2_USE_RDMA_CM= MV2_NUM_HCAS=1 MV2_USE_CUDA=1 MV2_ENABLE_AFFINITY=0 srun -n 2 -N 1 --cpu-bind=v -c 1 ./test_MPI
# -D MPI_COMPILER:BOOL=TRUE \
# -D MPIEXEC=mpirun \
# -D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \

1
sample_scripts/config_summit_hip
View File

@ -21,7 +21,6 @@ cmake \
-D HIP_NVCC_OPTIONS="-arch sm_70" \
-D LINK_LIBRARIES="/sw/summit/cuda/9.2.148/lib64/libcudart.so" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D USE_NETCDF=0 \
-D USE_SILO=1 \
-D SILO_DIRECTORY=${TPL_DIR}/silo \

1
sample_scripts/config_titan
View File

@ -9,7 +9,6 @@ cmake \
-D CMAKE_CUDA_FLAGS="-arch sm_35" \
-D CMAKE_CUDA_HOST_COMPILER="/usr/bin/gcc" \
-D USE_MPI=1 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
${HOME}/LBPM-WIA

1
sample_scripts/configure_arc_cluster
View File

@ -10,7 +10,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_arden
View File

@ -5,7 +5,6 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D CMAKE_C_FLAGS="-O3 -fPIC" \
-D CMAKE_CXX_FLAGS="-O3 -fPIC " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_basic_cluster
View File

@ -8,7 +8,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_blueridge
View File

@ -8,7 +8,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="" \
-D CMAKE_CXX_FLAGS="" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_cascades_cpu
View File

@ -11,7 +11,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="-fPIC" \
-D CMAKE_CXX_FLAGS="-fPIC" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/configure_desktop
View File

@ -7,7 +7,6 @@ cmake \
-D CMAKE_C_FLAGS="-g " \
-D CMAKE_CXX_FLAGS="-g " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

3
sample_scripts/configure_huckleberry
View File

@ -12,7 +12,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=mpicxx \
-D CMAKE_C_FLAGS="-fPIC" \
-D CMAKE_CXX_FLAGS="-fPIC" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \
@ -26,5 +25,5 @@ cmake \
-D USE_TIMER=0 \
~/LBPM-WIA
make VERBOSE=1 -j8 && make install
make VERBOSE=1 -j4 && make install

1
sample_scripts/configure_titan_jem
View File

@ -15,7 +15,6 @@ cmake \
-D CMAKE_C_COMPILER:PATH=cc \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

4
sample_scripts/configure_ubuntu
View File

@ -5,9 +5,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D CMAKE_C_FLAGS="-O3 -fPIC" \
-D CMAKE_CXX_FLAGS="-O3 -fPIC " \
-D CMAKE_CXX_STANDARD=14 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \
-D CUDA_FLAGS="-arch sm_35" \
-D CUDA_HOST_COMPILER="/usr/bin/gcc" \
@ -16,7 +14,7 @@ cmake -D CMAKE_C_COMPILER:PATH=/opt/arden/openmpi/3.1.2/bin/mpicc \
-D USE_SILO=1 \
-D SILO_LIB="/opt/arden/silo/4.10.2/lib/libsiloh5.a" \
-D SILO_DIRECTORY="/opt/arden/silo/4.10.2" \
-D USE_NETCDF=1 \
-D USE_NETCDF=0 \
-D NETCDF_DIRECTORY="/opt/arden/netcdf/4.6.1" \
-D USE_CUDA=0 \
-D USE_TIMER=0 \

1
sample_scripts/daedalus_config
View File

@ -12,7 +12,6 @@ i -D CMAKE_CXX_COMPILER:PATH=/home/christopher/openmpi/install_dir/bin/mpicxx
-D USE_DOXYGEN=false \
# -D CMAKE_C_FLAGS="-std=gnu++11 -w" \
# -D CMAKE_CXX_FLAGS="-std=gnu++11 -w" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=/home/christopher/openmpi/install_dir/bin/mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Release \

1
sample_scripts/promethius_config
View File

@ -7,7 +7,6 @@ cmake \
-D CMAKE_C_FLAGS="-g " \
-D CMAKE_CXX_FLAGS="-g -Wno-deprecated-declarations" \
-D CXX_STD=11 \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=mpirun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

10
tests/CMakeLists.txt
View File

@ -4,8 +4,10 @@
ADD_LBPM_EXECUTABLE( lbpm_color_simulator )
ADD_LBPM_EXECUTABLE( lbpm_permeability_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscale_simulator )
ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator )
ADD_LBPM_EXECUTABLE( lbpm_greyscaleColor_simulator )
ADD_LBPM_EXECUTABLE( lbpm_electrokinetic_SingleFluid_simulator )
ADD_LBPM_EXECUTABLE( lbpm_freelee_simulator )
ADD_LBPM_EXECUTABLE( lbpm_freelee_SingleFluidBGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_BGK_simulator )
#ADD_LBPM_EXECUTABLE( lbpm_color_macro_simulator )
ADD_LBPM_EXECUTABLE( lbpm_dfh_simulator )
@ -33,14 +35,15 @@ ADD_LBPM_EXECUTABLE( GenerateSphereTest )
#ADD_LBPM_EXECUTABLE( BlobAnalysis )
#ADD_LBPM_EXECUTABLE( BlobIdentify )
#ADD_LBPM_EXECUTABLE( BlobIdentifyParallel )
#ADD_LBPM_EXECUTABLE( convertIO )
#ADD_LBPM_EXECUTABLE( DataAggregator )
ADD_LBPM_EXECUTABLE( convertIO )
ADD_LBPM_EXECUTABLE( DataAggregator )
#ADD_LBPM_EXECUTABLE( BlobAnalyzeParallel )(
ADD_LBPM_EXECUTABLE( lbpm_minkowski_scalar )
ADD_LBPM_EXECUTABLE( TestPoissonSolver )
ADD_LBPM_EXECUTABLE( TestIonModel )
ADD_LBPM_EXECUTABLE( TestNernstPlanck )
ADD_LBPM_EXECUTABLE( TestPNP_Stokes )
ADD_LBPM_EXECUTABLE( TestMixedGrad )
@ -59,6 +62,7 @@ ADD_LBPM_TEST( TestMap )
ADD_LBPM_TEST( TestWideHalo )
ADD_LBPM_TEST( TestColorGradDFH )
ADD_LBPM_TEST( TestBubbleDFH ../example/Bubble/input.db)
ADD_LBPM_TEST( testGlobalMassFreeLee ../example/Bubble/input.db)
#ADD_LBPM_TEST( TestColorMassBounceback ../example/Bubble/input.db)
ADD_LBPM_TEST( TestPressVel ../example/Bubble/input.db)
ADD_LBPM_TEST( TestPoiseuille ../example/Piston/poiseuille.db)

199
tests/TestMixedGrad.cpp Normal file
View File

@ -0,0 +1,199 @@
#include <exception>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include "common/Utilities.h"
#include "models/FreeLeeModel.h"
inline void Initialize_Mask(ScaLBL_FreeLeeModel &LeeModel){
// initialize a bubble
int i,j,k,n;
int rank = LeeModel.Mask->rank();
int Nx = LeeModel.Mask->Nx;
int Ny = LeeModel.Mask->Ny;
int Nz = LeeModel.Mask->Nz;
if (rank == 0) printf(" initialize mask...\n");
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
LeeModel.Mask->id[n]=1;
LeeModel.id[n] = LeeModel.Mask->id[n];
}
}
}
}
inline void Initialize_DummyPhaseField(ScaLBL_FreeLeeModel &LeeModel, double ax, double ay, double az){
// initialize a bubble
int i,j,k,n;
int rank = LeeModel.Mask->rank();
int Nx = LeeModel.Mask->Nx;
int Ny = LeeModel.Mask->Ny;
int Nz = LeeModel.Mask->Nz;
if (rank == 0) printf("Setting up dummy phase field with gradient {x,y,z} = {%f , %f , %f}...\n",ax,ay,az);
double * Dummy;
int Nh = (Nx+2)*(Ny+2)*(Nz+2);
Dummy = new double [(Nx+2)*(Ny+2)*(Nz+2)];
for (k=0;k<Nz;k++){
for (j=0;j<Ny;j++){
for (i=0;i<Nx;i++){
n = k*Nx*Ny + j*Nz + i;
LeeModel.Mask->id[n]=1;
LeeModel.id[n] = LeeModel.Mask->id[n];
int nh = (k+1)*(Nx+2)*(Ny+2) + (j+1)*(Nx+2) + i+1;
Dummy[nh] = ax*double(i) + ay*double(j) + az*double(k);
}
}
}
ScaLBL_CopyToDevice(LeeModel.Phi, Dummy, sizeof(double)*Nh);
LeeModel.MGTest();
}
inline int MultiHaloNeighborCheck(ScaLBL_FreeLeeModel &LeeModel){
int i,j,k,iq,stride,nread;
int Nxh = LeeModel.Nxh;
int Nyh = LeeModel.Nyh;
int Np = LeeModel.Np;
int *TmpMap;
TmpMap = new int[Np];
ScaLBL_CopyToHost(TmpMap, LeeModel.dvcMap, Np*sizeof(int));
int *neighborList;
neighborList = new int[18*Np];
ScaLBL_CopyToHost(neighborList, LeeModel.NeighborList, 18*Np*sizeof(int));
printf("Check stride for interior neighbors \n");
int count = 0;
for (int n=LeeModel.ScaLBL_Comm->FirstInterior(); n<LeeModel.ScaLBL_Comm->LastInterior(); n++){
// q=0
int idx = TmpMap[n];
k = idx/Nxh/Nyh;
j = (idx-k*Nxh*Nyh)/Nxh;
i = (idx-k*Nxh*Nyh -j*Nxh);
// q=1
nread = neighborList[n];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != 1){
printf(" %i, %i, %i q = 1 stride=%i \n ",i,j,k,stride);
count++;
}
// q=2
nread = neighborList[n+Np];
iq = TmpMap[nread%Np];
stride = iq - idx;
if (stride != 1){
printf(" %i, %i, %i q = 2 stride=%i \n ",i,j,k,stride);
count++;
}
// q=3
nread = neighborList[n+2*Np];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != Nxh){
printf(" %i, %i, %i q = 3 stride=%i \n ",i,j,k,stride);
count++;
}
// q = 4
nread = neighborList[n+3*Np];
iq = TmpMap[nread%Np];
stride = iq-idx;
if (stride != Nxh){
printf(" %i, %i, %i q = 4 stride=%i \n ",i,j,k,stride);
count++;
}
// q=5
nread = neighborList[n+4*Np];
iq = TmpMap[nread%Np];
stride = idx - iq;
if (stride != Nxh*Nyh){
count++;
printf(" %i, %i, %i q = 5 stride=%i \n ",i,j,k,stride);
}
// q = 6
nread = neighborList[n+5*Np];
iq = TmpMap[nread%Np];
stride = iq - idx;
if (stride != Nxh*Nyh){
count++;
printf(" %i, %i, %i q = 6 stride=%i \n ",i,j,k,stride);
}
}
return count;
}
int main( int argc, char **argv )
{
// Initialize
Utilities::startup( argc, argv );
int errors = 0;
// Load the input database
auto db = std::make_shared<Database>( argv[1] );
{ // Limit scope so variables that contain communicators will free before MPI_Finialize
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocs = comm.getSize();
if ( rank == 0 ) {
printf( "********************************************************\n" );
printf( "Running Mixed Gradient Test \n" );
printf( "********************************************************\n" );
}
// Initialize compute device
int device = ScaLBL_SetDevice( rank );
NULL_USE( device );
ScaLBL_DeviceBarrier();
comm.barrier();
PROFILE_ENABLE( 1 );
// PROFILE_ENABLE_TRACE();
// PROFILE_ENABLE_MEMORY();
PROFILE_SYNCHRONIZE();
PROFILE_START( "Main" );
Utilities::setErrorHandlers();
auto filename = argv[1];
ScaLBL_FreeLeeModel LeeModel( rank, nprocs, comm );
LeeModel.ReadParams( filename );
LeeModel.SetDomain();
Initialize_Mask(LeeModel);
//LeeModel.Create_DummyPhase_MGTest();
LeeModel.Create_TwoFluid();
errors=MultiHaloNeighborCheck(LeeModel);
Initialize_DummyPhaseField(LeeModel,1.0, 2.0, 3.0);
LeeModel.WriteDebug_TwoFluid();
PROFILE_STOP( "Main" );
PROFILE_SAVE( file, level );
// ****************************************************
} // Limit scope so variables that contain communicators will free before MPI_Finialize
Utilities::shutdown();
return errors;
}

4
tests/TestNernstPlanck.cpp
View File

@ -66,7 +66,7 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
PoissonSolver.Initialize(0);
int timestep=0;
double error = 1.0;
@ -74,7 +74,7 @@ int main(int argc, char **argv)
while (timestep < Study.timestepMax && error > Study.tolerance){
timestep++;
PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental
PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
IonModel.Run(IonModel.FluidVelocityDummy,PoissonSolver.ElectricField); //solve for ion transport and electric potential
timestep++;//AA operations

9
tests/TestNetcdf.cpp
View File

@ -1,7 +1,7 @@
// Test reading/writing netcdf files
#include "IO/netcdf.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/UnitTest.h"
@ -13,7 +13,8 @@ void load( const std::string& );
void test_NETCDF( UnitTest& ut )
{
const int rank = comm_rank( MPI_COMM_WORLD );
Utilities::MPI comm( MPI_COMM_WORLD );
int rank = comm.getRank();
int nprocx = 2;
int nprocy = 2;
int nprocz = 2;
@ -26,11 +27,11 @@ void test_NETCDF( UnitTest& ut )
size_t z = info.kz*data.size(2);
const char* filename = "test.nc";
std::vector<int> dim = { (int) data.size(0)*nprocx, (int) data.size(1)*nprocy, (int) data.size(2)*nprocz };
int fid = netcdf::open( filename, netcdf::CREATE, MPI_COMM_WORLD );
int fid = netcdf::open( filename, netcdf::CREATE, comm );
auto dims = netcdf::defDim( fid, {"X", "Y", "Z"}, dim );
netcdf::write( fid, "tmp", dims, data, info );
netcdf::close( fid );
MPI_Barrier( MPI_COMM_WORLD );
comm.barrier();
// Read the contents of the file we created
fid = netcdf::open( filename, netcdf::READ );
Array<float> tmp = netcdf::getVar<float>( fid, "tmp" );

4
tests/TestPNP_Stokes.cpp
View File

@ -82,7 +82,7 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
PoissonSolver.Initialize(0);
int timestep=0;
@ -94,7 +94,7 @@ int main(int argc, char **argv)
while (timestep < Study.timestepMax && error > Study.tolerance){
timestep++;
PoissonSolver.Run(IonModel.ChargeDensity);//solve Poisson equtaion to get steady-state electrical potental
PoissonSolver.Run(IonModel.ChargeDensity,0);//solve Poisson equtaion to get steady-state electrical potental
StokesModel.Run_Lite(IonModel.ChargeDensity, PoissonSolver.ElectricField);// Solve the N-S equations to get velocity
IonModel.Run(StokesModel.Velocity,PoissonSolver.ElectricField); //solve for ion transport and electric potential

27
tests/TestPoissonSolver.cpp
View File

@ -51,14 +51,37 @@ int main(int argc, char **argv)
PoissonSolver.SetDomain();
PoissonSolver.ReadInput();
PoissonSolver.Create();
PoissonSolver.Initialize();
if (PoissonSolver.TestPeriodic==true){
PoissonSolver.Initialize(PoissonSolver.TestPeriodicTimeConv);
}
else {
PoissonSolver.Initialize(0);
}
//Initialize dummy charge density for test
PoissonSolver.DummyChargeDensity();
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy);
if (PoissonSolver.TestPeriodic==true){
if (rank==0) printf("Testing periodic voltage input is enabled. Total test time is %.3g[s], saving data every %.3g[s]; user-specified time resolution is %.3g[s/lt]\n",
PoissonSolver.TestPeriodicTime,PoissonSolver.TestPeriodicSaveInterval,PoissonSolver.TestPeriodicTimeConv);
int timestep = 0;
int timeMax = int(PoissonSolver.TestPeriodicTime/PoissonSolver.TestPeriodicTimeConv);
int timeSave = int(PoissonSolver.TestPeriodicSaveInterval/PoissonSolver.TestPeriodicTimeConv);
while (timestep<timeMax){
timestep++;
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,timestep);
if (timestep%timeSave==0){
if (rank==0) printf(" Time = %.3g[s]; saving electric potential and field\n",timestep*PoissonSolver.TestPeriodicTimeConv);
PoissonSolver.getElectricPotential_debug(timestep);
PoissonSolver.getElectricField_debug(timestep);
}
}
}
else {
PoissonSolver.Run(PoissonSolver.ChargeDensityDummy,1);
PoissonSolver.getElectricPotential_debug(1);
PoissonSolver.getElectricField_debug(1);
}
if (rank==0) printf("Maximum timestep is reached and the simulation is completed\n");
if (rank==0) printf("*************************************************************\n");

8
tests/TestWideHalo.cpp
View File

@ -159,6 +159,13 @@ int main(int argc, char **argv)
// copy the neighbor list
ScaLBL_CopyToDevice(NeighborList, neighborList, neighborSize);
// initialize phi based on PhaseLabel (include solid component labels)
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){
for (i=1;i<Nx-1;i++){
PhaseLabel[n] = 0.2;
}
}
}
ScaLBL_CopyToDevice(Phi, PhaseLabel, N*sizeof(double));
//...........................................................................
@ -172,7 +179,6 @@ int main(int argc, char **argv)
int SIZE=3*Np*sizeof(double);
ScaLBL_CopyToHost(&COLORGRAD[0],&ColorGrad[0],SIZE);
double CX,CY,CZ;
for (k=1;k<Nz-1;k++){
for (j=1;j<Ny-1;j++){

Some files were not shown because too many files have changed in this diff Show More