OilfieldToolsNet/LBPM
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix merge conflict

Browse Source
This commit is contained in:
JamesEMcclure 2021-03-24 21:32:08 -04:00
commit 0b03f9bd5c
248 changed files with 51038 additions and 10137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
.clang-format Normal file
View File

@ -0,0 +1,108 @@
# To run clang tools:
# cd to root directory
# To update format only:
# find . -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
# git status -s . | sed s/^...// | grep -E "(\.cpp|\.h|\.cc|\.hpp|\.I)" | xargs -I{} clang-format -i {}
# To run modernize
# export CLANG_PATH=/packages/llvm/build/llvm-60
# export PATH=${CLANG_PATH}/bin:${CLANG_PATH}/share/clang:$PATH
# find src -name "*.cpp" -or -name "*.cc" | xargs -I{} clang-tidy -checks=modernize* -p=/projects/AtomicModel/build/debug -fix {}
# find src -name "*.cpp" -or -name "*.cc" -or -name "*.h" -or -name "*.hpp" -or -name "*.I" | xargs -I{} clang-format -i {}
---
Language: Cpp
# BasedOnStyle: LLVM
AccessModifierOffset: -4
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: true
AlignConsecutiveDeclarations: false
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: true
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: true
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterClass: true
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: false
AfterObjCDeclaration: true
AfterStruct: false
AfterUnion: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
BreakBeforeBinaryOperators: None
#BreakBeforeBraces: Stroustrup
BreakBeforeBraces: Custom
BreakBeforeTernaryOperators: false
BreakConstructorInitializersBeforeComma: false
ColumnLimit: 100
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: false
DerivePointerAlignment: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
IncludeCategories:
- Regex: '^"(llvm|llvm-c|clang|clang-c)/'
Priority: 2
- Regex: '^(<|"(gtest|isl|json)/)'
Priority: 3
- Regex: '.*'
Priority: 1
IndentCaseLabels: false
IndentWidth: 4
IndentWrappedFunctionNames: false
KeepEmptyLinesAtTheStartOfBlocks: true
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 2
NamespaceIndentation: None
ObjCBlockIndentWidth: 4
ObjCSpaceAfterProperty: false
ObjCSpaceBeforeProtocolList: true
PenaltyBreakBeforeFirstCallParameter: 19
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterCStyleCast: true
SpaceAfterTemplateKeyword: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: true
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 4
UseTab: Never
...

362
CMakeLists.txt
View File

@ -1,170 +1,192 @@
# Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
MESSAGE("====================")
MESSAGE("Configuring LBPM-WIA")
MESSAGE("====================")
# Set the project name
SET( PROJ LBPM ) # Set the project name for CMake
SET( LBPM_LIB lbpm-wia ) # Set the final library name
SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...)
SET( TEST_MAX_PROCS 16 )
# Initialize the project
PROJECT( ${PROJ} LANGUAGES CXX )
# Prevent users from building in place
IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )
MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )
ENDIF()
# Set the default C++ standard
SET( CMAKE_CXX_EXTENSIONS OFF )
IF ( NOT CMAKE_CXX_STANDARD )
IF ( CXX_STD )
MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" )
ENDIF()
SET( CMAKE_CXX_STANDARD 14 )
ENDIF()
IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) )
MESSAGE( FATAL_ERROR "C++14 or newer required" )
ENDIF()
# Set source/install paths
SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
IF( ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" )
ELSEIF( PREFIX )
SET( ${PROJ}_INSTALL_DIR "${PREFIX}" )
ELSEIF( NOT ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
ENDIF()
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake )
# Include macros
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" )
# Check if we are only compiling docs
CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
# Set testing paramaters
SET( DROP_METHOD "http" )
SET( DROP_SITE "" )
SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET( TRIGGER_SITE "" )
SET( DROP_SITE_CDASH TRUE )
ENABLE_TESTING()
INCLUDE( CTest )
# Check the compile mode and compile flags
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_SYSTEM()
ENDIF()
# Add some directories to include
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
# Create the target for documentation
ADD_CUSTOM_TARGET( doc )
ADD_CUSTOM_TARGET( latex_docs )
CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
CHECK_ENABLE_FLAG( USE_LATEX 1 )
FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
IF ( USE_DOXYGEN )
SET( DOXYFILE_LATEX YES )
SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )
SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" )
SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" )
SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" )
SET( REL_PACKAGE_HTML "" )
SET( DOXYGEN_MACROS "" )
MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" )
IF ( DOXYGEN_FOUND )
ADD_DEPENDENCIES( doxygen latex_docs )
ADD_DEPENDENCIES( doc latex_docs doxygen )
ELSE()
SET( USE_DOXYGEN 0 )
ENDIF()
ENDIF()
# Create custom targets for build-test, check, and distclean
ADD_CUSTOM_TARGET( build-test )
ADD_CUSTOM_TARGET( build-examples )
ADD_CUSTOM_TARGET( check COMMAND make test )
ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu example common IO threadpool StackTrace )
# Check for CUDA
CHECK_ENABLE_FLAG( USE_CUDA 0 )
NULL_USE( CMAKE_CUDA_FLAGS )
IF ( USE_CUDA )
ADD_DEFINITIONS( -DUSE_CUDA )
ENABLE_LANGUAGE( CUDA )
ENDIF()
# Configure external packages
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_MPI() # MPI must be before other libraries
CONFIGURE_MIC()
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
ENDIF()
# Macro to create 1,2,4 processor tests
MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
ADD_LBPM_TEST( ${EXENAME} ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} )
ENDMACRO()
# Add the src directories
IF ( NOT ONLY_BUILD_DOCS )
BEGIN_PACKAGE_CONFIG( lbpm-wia-library )
ADD_PACKAGE_SUBDIRECTORY( common )
ADD_PACKAGE_SUBDIRECTORY( analysis )
ADD_PACKAGE_SUBDIRECTORY( IO )
ADD_PACKAGE_SUBDIRECTORY( threadpool )
ADD_PACKAGE_SUBDIRECTORY( StackTrace )
ADD_PACKAGE_SUBDIRECTORY( models )
IF ( USE_CUDA )
ADD_PACKAGE_SUBDIRECTORY( gpu )
ELSE()
ADD_PACKAGE_SUBDIRECTORY( cpu )
ENDIF()
INSTALL_LBPM_TARGET( lbpm-wia-library )
ADD_SUBDIRECTORY( tests )
ADD_SUBDIRECTORY( example )
#ADD_SUBDIRECTORY( workflows )
INSTALL_PROJ_LIB()
ENDIF()
# Set some CMake properties
CMAKE_MINIMUM_REQUIRED( VERSION 3.9 )
MESSAGE("====================")
MESSAGE("Configuring LBPM-WIA")
MESSAGE("====================")
# Set the project name
SET( PROJ LBPM ) # Set the project name for CMake
SET( LBPM_LIB lbpm-wia ) # Set the final library name
SET( LBPM_INC ) # Set an optional subfolder for includes (e.g. include/name/...)
SET( TEST_MAX_PROCS 16 )
# Initialize the project
PROJECT( ${PROJ} LANGUAGES CXX )
# Prevent users from building in place
IF ("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}" )
MESSAGE( FATAL_ERROR "Building code in place is a bad idea" )
ENDIF()
# Set the default C++ standard
SET( CMAKE_CXX_EXTENSIONS OFF )
IF ( NOT CMAKE_CXX_STANDARD )
IF ( CXX_STD )
MESSAGE( FATAL_ERROR "CXX_STD is obsolete, please set CMAKE_CXX_STANDARD" )
ENDIF()
SET( CMAKE_CXX_STANDARD 14 )
ENDIF()
IF ( ( "${CMAKE_CXX_STANDARD}" GREATER "90" ) OR ( "${CMAKE_CXX_STANDARD}" LESS "14" ) )
MESSAGE( FATAL_ERROR "C++14 or newer required" )
ENDIF()
# Set source/install paths
SET( ${PROJ}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" )
SET( ${PROJ}_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
IF( ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${${PROJ}_INSTALL_DIR}" )
ELSEIF( PREFIX )
SET( ${PROJ}_INSTALL_DIR "${PREFIX}" )
ELSEIF( NOT ${PROJ}_INSTALL_DIR )
SET( ${PROJ}_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}" )
ENDIF()
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
SET( CMAKE_MODULE_PATH ${${PROJ}_SOURCE_DIR} ${${PROJ}_SOURCE_DIR}/cmake )
# Include macros
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libraries.cmake" )
INCLUDE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/LBPM-macros.cmake" )
# Check if we are only compiling docs
CHECK_ENABLE_FLAG( ONLY_BUILD_DOCS 0 )
# Set testing paramaters
SET( DROP_METHOD "http" )
SET( DROP_SITE "" )
SET( DROP_LOCATION "/CDash/submit.php?project=LBPM-WIA" )
SET( TRIGGER_SITE "" )
SET( DROP_SITE_CDASH TRUE )
ENABLE_TESTING()
INCLUDE( CTest )
# Check the compile mode and compile flags
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_SYSTEM()
ENDIF()
# Add some directories to include
INCLUDE_DIRECTORIES( "${${PROJ}_INSTALL_DIR}/include" )
# Create the target for documentation
ADD_CUSTOM_TARGET( doc )
ADD_CUSTOM_TARGET( latex_docs )
CHECK_ENABLE_FLAG( USE_DOXYGEN 1 )
CHECK_ENABLE_FLAG( USE_LATEX 1 )
FILE( MAKE_DIRECTORY "${${PROJ}_INSTALL_DIR}/doc" )
IF ( USE_DOXYGEN )
SET( DOXYFILE_LATEX YES )
SET( DOXYFILE_IN "${${PROJ}_SOURCE_DIR}/doxygen/Doxyfile.in" )
SET( DOXY_HEADER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/header.html" )
SET( DOXY_FOOTER_FILE "${${PROJ}_SOURCE_DIR}/doxygen/html/footer.html" )
SET( DOXYFILE_OUTPUT_DIR "${${PROJ}_INSTALL_DIR}/doc" )
SET( DOXYFILE_SRC_HTML_DIR "${${PROJ}_SOURCE_DIR}/doxygen/html" )
SET( DOXYFILE_SOURCE_DIR "${${PROJ}_SOURCE_DIR}" )
SET( REL_PACKAGE_HTML "" )
SET( DOXYGEN_MACROS "" )
MESSAGE("DOXYGEN_MACROS = ${DOXYGEN_MACROS}")
INCLUDE( "${${PROJ}_SOURCE_DIR}/cmake/UseDoxygen.cmake" )
IF ( DOXYGEN_FOUND )
ADD_DEPENDENCIES( doxygen latex_docs )
ADD_DEPENDENCIES( doc latex_docs doxygen )
ELSE()
SET( USE_DOXYGEN 0 )
ENDIF()
ENDIF()
# Create custom targets for build-test, check, and distclean
ADD_CUSTOM_TARGET( build-test )
ADD_CUSTOM_TARGET( build-examples )
ADD_CUSTOM_TARGET( check COMMAND make test )
ADD_DISTCLEAN( analysis null_timer tests liblbpm-wia.* cpu gpu cuda hip example common IO threadpool StackTrace )
# Check for CUDA
CHECK_ENABLE_FLAG( USE_CUDA 0 )
CHECK_ENABLE_FLAG( USE_HIP 0 )
NULL_USE( CMAKE_CUDA_FLAGS )
IF ( USE_CUDA )
ADD_DEFINITIONS( -DUSE_CUDA )
ENABLE_LANGUAGE( CUDA )
ELSEIF ( USE_HIP )
IF ( NOT DEFINED HIP_PATH )
IF ( NOT DEFINED ENV{HIP_PATH} )
SET( HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed" )
ELSE()
SET( HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed" )
ENDIF()
ENDIF()
SET( CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH} )
FIND_PACKAGE( HIP REQUIRED )
FIND_PACKAGE( CUDA QUIET )
MESSAGE( "HIP Found")
MESSAGE( " HIP version: ${HIP_VERSION_STRING}")
MESSAGE( " HIP platform: ${HIP_PLATFORM}")
MESSAGE( " HIP Include Path: ${HIP_INCLUDE_DIRS}")
MESSAGE( " HIP Libraries: ${HIP_LIBRARIES}")
ADD_DEFINITIONS( -DUSE_HIP )
ENDIF()
# Configure external packages
IF ( NOT ONLY_BUILD_DOCS )
CONFIGURE_MPI() # MPI must be before other libraries
CONFIGURE_MIC()
CONFIGURE_NETCDF()
CONFIGURE_SILO()
CONFIGURE_LBPM()
CONFIGURE_TIMER( 0 "${${PROJ}_INSTALL_DIR}/null_timer" FALSE )
CONFIGURE_LINE_COVERAGE()
# Set the external library link list
SET( EXTERNAL_LIBS ${EXTERNAL_LIBS} ${TIMER_LIBS} )
ENDIF()
# Macro to create 1,2,4 processor tests
MACRO( ADD_LBPM_TEST_1_2_4 EXENAME ${ARGN} )
ADD_LBPM_TEST( ${EXENAME} ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 2 ${ARGN} )
ADD_LBPM_TEST_PARALLEL( ${EXENAME} 4 ${ARGN} )
ENDMACRO()
# Add the src directories
IF ( NOT ONLY_BUILD_DOCS )
BEGIN_PACKAGE_CONFIG( lbpm-wia-library )
ADD_PACKAGE_SUBDIRECTORY( common )
ADD_PACKAGE_SUBDIRECTORY( analysis )
ADD_PACKAGE_SUBDIRECTORY( IO )
ADD_PACKAGE_SUBDIRECTORY( threadpool )
ADD_PACKAGE_SUBDIRECTORY( StackTrace )
ADD_PACKAGE_SUBDIRECTORY( models )
IF ( USE_CUDA )
ADD_PACKAGE_SUBDIRECTORY( cuda )
ELSEIF ( USE_HIP )
ADD_SUBDIRECTORY( hip )
SET( LBPM_LIBRARIES lbpm-hip lbpm-wia )
ELSE()
ADD_PACKAGE_SUBDIRECTORY( cpu )
ENDIF()
INSTALL_LBPM_TARGET( lbpm-wia-library )
ADD_SUBDIRECTORY( tests )
ADD_SUBDIRECTORY( example )
#ADD_SUBDIRECTORY( workflows )
INSTALL_PROJ_LIB()
ENDIF()

32
IO/IOHelpers.h
View File

@ -25,9 +25,9 @@ namespace IO {
// Find a character in a line
inline size_t find( const char *line, char token )
{
size_t i=0;
size_t i = 0;
while ( 1 ) {
if ( line[i]==token || line[i]<32 || line[i]==0 )
if ( line[i] == token || line[i] < 32 || line[i] == 0 )
break;
++i;
}
@ -36,17 +36,17 @@ inline size_t find( const char *line, char token )
// Remove preceeding/trailing whitespace
inline std::string deblank( const std::string& str )
inline std::string deblank( const std::string &str )
{
size_t i1 = str.size();
size_t i2 = 0;
for (size_t i=0; i<str.size(); i++) {
if ( str[i]!=' ' && str[i]>=32 ) {
i1 = std::min(i1,i);
i2 = std::max(i2,i);
for ( size_t i = 0; i < str.size(); i++ ) {
if ( str[i] != ' ' && str[i] >= 32 ) {
i1 = std::min( i1, i );
i2 = std::max( i2, i );
}
}
return str.substr(i1,i2-i1+1);
return str.substr( i1, i2 - i1 + 1 );
}
@ -57,14 +57,14 @@ inline std::vector<std::string> splitList( const char *line, const char token )
size_t i1 = 0;
size_t i2 = 0;
while ( 1 ) {
if ( line[i2]==token || line[i2]<32 ) {
std::string tmp(&line[i1],i2-i1);
tmp = deblank(tmp);
if ( line[i2] == token || line[i2] < 32 ) {
std::string tmp( &line[i1], i2 - i1 );
tmp = deblank( tmp );
if ( !tmp.empty() )
list.push_back(tmp);
i1 = i2+1;
list.push_back( tmp );
i1 = i2 + 1;
}
if ( line[i2]==0 )
if ( line[i2] == 0 )
break;
i2++;
}
@ -72,8 +72,6 @@ inline std::vector<std::string> splitList( const char *line, const char token )
}
};
}; // namespace IO
#endif

715
IO/Mesh.cpp
View File

@ -29,6 +29,7 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Mesh.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#include <limits>
@ -49,104 +50,110 @@ inline Point nullPoint()
/****************************************************
* Mesh *
****************************************************/
Mesh::Mesh( )
{
}
Mesh::~Mesh( )
{
}
* Mesh *
****************************************************/
Mesh::Mesh() {}
Mesh::~Mesh() {}
/****************************************************
* MeshDataStruct *
****************************************************/
bool MeshDataStruct::check() const
* MeshDataStruct *
****************************************************/
#define checkResult( pass, msg ) \
do { \
if ( !( pass ) ) { \
if ( abort ) \
ERROR( msg ); \
return false; \
} \
} while ( 0 )
bool MeshDataStruct::check( bool abort ) const
{
enum VariableType { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
bool pass = mesh != nullptr;
for ( const auto& var : vars ) {
pass = pass && static_cast<int>(var->type)>=1 && static_cast<int>(var->type)<=3;
pass = pass && !var->data.empty();
for ( const auto &var : vars ) {
checkResult( var->type == VariableType::NodeVariable ||
var->type == VariableType::EdgeVariable ||
var->type == VariableType::SurfaceVariable ||
var->type == VariableType::VolumeVariable,
"Invalid data type" );
checkResult( !var->data.empty(), "Variable data is empty" );
}
if ( !pass )
return false;
const std::string& meshClass = mesh->className();
const std::string &meshClass = mesh->className();
if ( meshClass == "PointList" ) {
const auto mesh2 = dynamic_cast<IO::PointList*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = dynamic_cast<IO::PointList *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
ERROR("Invalid type for PointList");
ERROR( "Invalid type for PointList" );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
}
} else if ( meshClass == "TriMesh" || meshClass == "TriList" ) {
const auto mesh2 = getTriMesh( mesh );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = getTriMesh( mesh );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && var->data.size(0)==mesh2->vertices->points.size() && var->data.size(1)==var->dim;
size_t N_points = mesh2->vertices->points.size();
checkResult( var->data.size( 0 ) == N_points, "sizeof NodeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof NodeVariable" );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && var->data.size(0)==mesh2->A.size() && var->data.size(1)==var->dim;
checkResult( var->data.size( 0 ) == mesh2->A.size(), "sizeof VolumeVariable" );
checkResult( var->data.size( 1 ) == var->dim, "sizeof VolumeVariable" );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
}
} else if ( meshClass == "DomainMesh" ) {
const auto mesh2 = dynamic_cast<IO::DomainMesh*>( mesh.get() );
if ( mesh2 == nullptr )
return false;
for ( const auto& var : vars ) {
auto mesh2 = dynamic_cast<IO::DomainMesh *>( mesh.get() );
ASSERT( mesh2 );
for ( const auto &var : vars ) {
ArraySize varSize;
if ( var->type == IO::VariableType::NodeVariable ) {
pass = pass && (int) var->data.size(0)==(mesh2->nx+1) && (int) var->data.size(1)==(mesh2->ny+1)
&& (int) var->data.size(2)==(mesh2->nz+1) && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx + 1, mesh2->ny + 1, mesh2->nz + 1, var->dim );
} else if ( var->type == IO::VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var->type == IO::VariableType::VolumeVariable ) {
pass = pass && (int) var->data.size(0)==mesh2->nx && (int) var->data.size(1)==mesh2->ny
&& (int) var->data.size(2)==mesh2->nz && var->data.size(3)==var->dim;
varSize = ArraySize( mesh2->nx, mesh2->ny, mesh2->nz, var->dim );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
if ( var->data.size( 0 ) == varSize[0] * varSize[1] * varSize[2] &&
var->data.size( 1 ) == varSize[3] )
var->data.resize( varSize );
for ( int d = 0; d < 4; d++ )
checkResult( var->data.size( d ) == varSize[d], "DomainMesh Variable" );
}
} else {
ERROR("Unknown mesh class: "+mesh->className());
ERROR( "Unknown mesh class: " + mesh->className() );
}
return pass;
return true;
}
/****************************************************
* PointList *
****************************************************/
PointList::PointList( )
{
}
* PointList *
****************************************************/
PointList::PointList() {}
PointList::PointList( size_t N )
{
Point tmp = nullPoint();
points.resize(N,tmp);
}
PointList::~PointList( )
{
points.resize( N, tmp );
}
PointList::~PointList() {}
size_t PointList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
@ -154,174 +161,168 @@ size_t PointList::numberPointsVar( VariableType type ) const
N = points.size();
return N;
}
std::pair<size_t,void*> PointList::pack( int level ) const
std::pair<size_t, void *> PointList::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
data_out.first = (2+3*points.size())*sizeof(double);
double *data_ptr = new double[2+3*points.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
data_int[0] = level;
data_int[1] = points.size();
double *data = &data_ptr[2];
for (size_t i=0; i<points.size(); i++) {
data[3*i+0] = points[i].x;
data[3*i+1] = points[i].y;
data[3*i+2] = points[i].z;
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
data_out.first = ( 2 + 3 * points.size() ) * sizeof( double );
double *data_ptr = new double[2 + 3 * points.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
data_int[0] = level;
data_int[1] = points.size();
double *data = &data_ptr[2];
for ( size_t i = 0; i < points.size(); i++ ) {
data[3 * i + 0] = points[i].x;
data[3 * i + 1] = points[i].y;
data[3 * i + 2] = points[i].z;
}
}
return data_out;
}
void PointList::unpack( const std::pair<size_t,void*>& data_in )
void PointList::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level==0 ) {
ASSERT((2+3*N)*sizeof(double)==data_in.first);
points.resize(N);
for (size_t i=0; i<points.size(); i++) {
points[i].x = data[3*i+0];
points[i].y = data[3*i+1];
points[i].z = data[3*i+2];
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level == 0 ) {
ASSERT( ( 2 + 3 * N ) * sizeof( double ) == data_in.first );
points.resize( N );
for ( size_t i = 0; i < points.size(); i++ ) {
points[i].x = data[3 * i + 0];
points[i].y = data[3 * i + 1];
points[i].z = data[3 * i + 2];
}
}
}
/****************************************************
* TriList *
****************************************************/
TriList::TriList( )
{
}
* TriList *
****************************************************/
TriList::TriList() {}
TriList::TriList( size_t N_tri )
{
Point tmp = nullPoint();
A.resize(N_tri,tmp);
B.resize(N_tri,tmp);
C.resize(N_tri,tmp);
A.resize( N_tri, tmp );
B.resize( N_tri, tmp );
C.resize( N_tri, tmp );
}
TriList::TriList( const TriMesh& mesh )
TriList::TriList( const TriMesh &mesh )
{
Point tmp = nullPoint();
A.resize(mesh.A.size(),tmp);
B.resize(mesh.B.size(),tmp);
C.resize(mesh.C.size(),tmp);
ASSERT(mesh.vertices.get()!=NULL);
const std::vector<Point>& P = mesh.vertices->points;
for (size_t i=0; i<A.size(); i++)
A.resize( mesh.A.size(), tmp );
B.resize( mesh.B.size(), tmp );
C.resize( mesh.C.size(), tmp );
ASSERT( mesh.vertices.get() != NULL );
const std::vector<Point> &P = mesh.vertices->points;
for ( size_t i = 0; i < A.size(); i++ )
A[i] = P[mesh.A[i]];
for (size_t i=0; i<B.size(); i++)
for ( size_t i = 0; i < B.size(); i++ )
B[i] = P[mesh.B[i]];
for (size_t i=0; i<C.size(); i++)
for ( size_t i = 0; i < C.size(); i++ )
C[i] = P[mesh.C[i]];
}
TriList::~TriList( )
{
}
TriList::~TriList() {}
size_t TriList::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
N = 3*A.size();
else if ( type==VariableType::SurfaceVariable || type==VariableType::VolumeVariable )
if ( type == VariableType::NodeVariable )
N = 3 * A.size();
else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
N = A.size();
return N;
}
std::pair<size_t,void*> TriList::pack( int level ) const
std::pair<size_t, void *> TriList::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
data_out.first = (2+9*A.size())*sizeof(double);
double *data_ptr = new double[2+9*A.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_ptr);
data_int[0] = level;
data_int[1] = A.size();
double *data = &data_ptr[2];
for (size_t i=0; i<A.size(); i++) {
data[9*i+0] = A[i].x;
data[9*i+1] = A[i].y;
data[9*i+2] = A[i].z;
data[9*i+3] = B[i].x;
data[9*i+4] = B[i].y;
data[9*i+5] = B[i].z;
data[9*i+6] = C[i].x;
data[9*i+7] = C[i].y;
data[9*i+8] = C[i].z;
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
data_out.first = ( 2 + 9 * A.size() ) * sizeof( double );
double *data_ptr = new double[2 + 9 * A.size()];
data_out.second = data_ptr;
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_ptr );
data_int[0] = level;
data_int[1] = A.size();
double *data = &data_ptr[2];
for ( size_t i = 0; i < A.size(); i++ ) {
data[9 * i + 0] = A[i].x;
data[9 * i + 1] = A[i].y;
data[9 * i + 2] = A[i].z;
data[9 * i + 3] = B[i].x;
data[9 * i + 4] = B[i].y;
data[9 * i + 5] = B[i].z;
data[9 * i + 6] = C[i].x;
data[9 * i + 7] = C[i].y;
data[9 * i + 8] = C[i].z;
}
}
return data_out;
}
void TriList::unpack( const std::pair<size_t,void*>& data_in )
void TriList::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level==0 ) {
ASSERT((2+9*N)*sizeof(double)==data_in.first);
A.resize(N);
B.resize(N);
C.resize(N);
for (size_t i=0; i<A.size(); i++) {
A[i].x = data[9*i+0];
A[i].y = data[9*i+1];
A[i].z = data[9*i+2];
B[i].x = data[9*i+3];
B[i].y = data[9*i+4];
B[i].z = data[9*i+5];
C[i].x = data[9*i+6];
C[i].y = data[9*i+7];
C[i].z = data[9*i+8];
uint64_t *data_int = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int[0];
uint64_t N = data_int[1];
data = &data[2];
if ( level == 0 ) {
ASSERT( ( 2 + 9 * N ) * sizeof( double ) == data_in.first );
A.resize( N );
B.resize( N );
C.resize( N );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i].x = data[9 * i + 0];
A[i].y = data[9 * i + 1];
A[i].z = data[9 * i + 2];
B[i].x = data[9 * i + 3];
B[i].y = data[9 * i + 4];
B[i].z = data[9 * i + 5];
C[i].x = data[9 * i + 6];
C[i].y = data[9 * i + 7];
C[i].z = data[9 * i + 8];
}
}
}
/****************************************************
* TriMesh *
****************************************************/
TriMesh::TriMesh( )
{
}
* TriMesh *
****************************************************/
TriMesh::TriMesh() {}
TriMesh::TriMesh( size_t N_tri, size_t N_point )
{
vertices.reset( new PointList(N_point) );
A.resize(N_tri,-1);
B.resize(N_tri,-1);
C.resize(N_tri,-1);
vertices.reset( new PointList( N_point ) );
A.resize( N_tri, -1 );
B.resize( N_tri, -1 );
C.resize( N_tri, -1 );
}
TriMesh::TriMesh( size_t N_tri, std::shared_ptr<PointList> points )
{
vertices = points;
A.resize(N_tri,-1);
B.resize(N_tri,-1);
C.resize(N_tri,-1);
A.resize( N_tri, -1 );
B.resize( N_tri, -1 );
C.resize( N_tri, -1 );
}
TriMesh::TriMesh( const TriList& mesh )
TriMesh::TriMesh( const TriList &mesh )
{
// For simlicity we will just create a mesh with ~3x the verticies for now
ASSERT(mesh.A.size()==mesh.B.size()&&mesh.A.size()==mesh.C.size());
A.resize(mesh.A.size());
B.resize(mesh.B.size());
C.resize(mesh.C.size());
vertices.reset( new PointList(3*mesh.A.size()) );
for (size_t i=0; i<A.size(); i++) {
A[i] = 3*i+0;
B[i] = 3*i+1;
C[i] = 3*i+2;
ASSERT( mesh.A.size() == mesh.B.size() && mesh.A.size() == mesh.C.size() );
A.resize( mesh.A.size() );
B.resize( mesh.B.size() );
C.resize( mesh.C.size() );
vertices.reset( new PointList( 3 * mesh.A.size() ) );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i] = 3 * i + 0;
B[i] = 3 * i + 1;
C[i] = 3 * i + 2;
vertices->points[A[i]] = mesh.A[i];
vertices->points[B[i]] = mesh.B[i];
vertices->points[C[i]] = mesh.C[i];
}
}
TriMesh::~TriMesh( )
TriMesh::~TriMesh()
{
vertices.reset();
A.clear();
@ -331,181 +332,319 @@ TriMesh::~TriMesh( )
size_t TriMesh::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
if ( type == VariableType::NodeVariable )
N = vertices->points.size();
else if ( type==VariableType::SurfaceVariable || type==VariableType::VolumeVariable )
else if ( type == VariableType::SurfaceVariable || type == VariableType::VolumeVariable )
N = A.size();
return N;
}
std::pair<size_t,void*> TriMesh::pack( int level ) const
std::pair<size_t, void *> TriMesh::pack( int level ) const
{
std::pair<size_t,void*> data_out(0,NULL);
if ( level==0 ) {
const std::vector<Point>& points = vertices->points;
data_out.first = (3+3*points.size())*sizeof(double) + 3*A.size()*sizeof(int);
double *data_ptr = new double[4+3*points.size()+(3*A.size()*sizeof(int))/sizeof(double)];
data_out.second = data_ptr;
uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_ptr);
data_int64[0] = level;
data_int64[1] = points.size();
data_int64[2] = A.size();
double *data = &data_ptr[3];
for (size_t i=0; i<points.size(); i++) {
data[3*i+0] = points[i].x;
data[3*i+1] = points[i].y;
data[3*i+2] = points[i].z;
std::pair<size_t, void *> data_out( 0, NULL );
if ( level == 0 ) {
const std::vector<Point> &points = vertices->points;
data_out.first =
( 3 + 3 * points.size() ) * sizeof( double ) + 3 * A.size() * sizeof( int );
double *data_ptr =
new double[4 + 3 * points.size() + ( 3 * A.size() * sizeof( int ) ) / sizeof( double )];
data_out.second = data_ptr;
uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_ptr );
data_int64[0] = level;
data_int64[1] = points.size();
data_int64[2] = A.size();
double *data = &data_ptr[3];
for ( size_t i = 0; i < points.size(); i++ ) {
data[3 * i + 0] = points[i].x;
data[3 * i + 1] = points[i].y;
data[3 * i + 2] = points[i].z;
}
int *data_int = reinterpret_cast<int*>(&data[3*points.size()]);
for (size_t i=0; i<A.size(); i++) {
data_int[3*i+0] = A[i];
data_int[3*i+1] = B[i];
data_int[3*i+2] = C[i];
int *data_int = reinterpret_cast<int *>( &data[3 * points.size()] );
for ( size_t i = 0; i < A.size(); i++ ) {
data_int[3 * i + 0] = A[i];
data_int[3 * i + 1] = B[i];
data_int[3 * i + 2] = C[i];
}
}
return data_out;
}
void TriMesh::unpack( const std::pair<size_t,void*>& data_in )
void TriMesh::unpack( const std::pair<size_t, void *> &data_in )
{
uint64_t *data_int64 = reinterpret_cast<uint64_t*>(data_in.second);
const double *data = reinterpret_cast<const double*>(data_in.second);
int level = data_int64[0];
uint64_t N_P = data_int64[1];
uint64_t N_A = data_int64[2];
data = &data[3];
if ( level==0 ) {
size_t size = (3+3*N_P)*sizeof(double)+3*N_A*sizeof(int);
ASSERT(size==data_in.first);
vertices.reset( new PointList(N_P) );
std::vector<Point>& points = vertices->points;
for (size_t i=0; i<points.size(); i++) {
points[i].x = data[3*i+0];
points[i].y = data[3*i+1];
points[i].z = data[3*i+2];
uint64_t *data_int64 = reinterpret_cast<uint64_t *>( data_in.second );
const double *data = reinterpret_cast<const double *>( data_in.second );
int level = data_int64[0];
uint64_t N_P = data_int64[1];
uint64_t N_A = data_int64[2];
data = &data[3];
if ( level == 0 ) {
size_t size = ( 3 + 3 * N_P ) * sizeof( double ) + 3 * N_A * sizeof( int );
ASSERT( size == data_in.first );
vertices.reset( new PointList( N_P ) );
std::vector<Point> &points = vertices->points;
for ( size_t i = 0; i < points.size(); i++ ) {
points[i].x = data[3 * i + 0];
points[i].y = data[3 * i + 1];
points[i].z = data[3 * i + 2];
}
const int *data_int = reinterpret_cast<const int*>(&data[3*N_P]);
A.resize(N_A);
B.resize(N_A);
C.resize(N_A);
for (size_t i=0; i<A.size(); i++) {
A[i] = data_int[3*i+0];
B[i] = data_int[3*i+1];
C[i] = data_int[3*i+2];
const int *data_int = reinterpret_cast<const int *>( &data[3 * N_P] );
A.resize( N_A );
B.resize( N_A );
C.resize( N_A );
for ( size_t i = 0; i < A.size(); i++ ) {
A[i] = data_int[3 * i + 0];
B[i] = data_int[3 * i + 1];
C[i] = data_int[3 * i + 2];
}
}
}
/****************************************************
* Domain mesh *
****************************************************/
DomainMesh::DomainMesh():
nprocx(0), nprocy(0), nprocz(0), rank(0),
nx(0), ny(0), nz(0),
Lx(0), Ly(0), Lz(0)
* Domain mesh *
****************************************************/
DomainMesh::DomainMesh()
: nprocx( 0 ),
nprocy( 0 ),
nprocz( 0 ),
rank( 0 ),
nx( 0 ),
ny( 0 ),
nz( 0 ),
Lx( 0 ),
Ly( 0 ),
Lz( 0 )
{
}
DomainMesh::DomainMesh( RankInfoStruct data,
int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 ):
nprocx(data.nx), nprocy(data.ny), nprocz(data.nz), rank(data.rank[1][1][1]),
nx(nx2), ny(ny2), nz(nz2),
Lx(Lx2), Ly(Ly2), Lz(Lz2)
{
}
DomainMesh::~DomainMesh()
DomainMesh::DomainMesh(
RankInfoStruct data, int nx2, int ny2, int nz2, double Lx2, double Ly2, double Lz2 )
: nprocx( data.nx ),
nprocy( data.ny ),
nprocz( data.nz ),
rank( data.rank[1][1][1] ),
nx( nx2 ),
ny( ny2 ),
nz( nz2 ),
Lx( Lx2 ),
Ly( Ly2 ),
Lz( Lz2 )
{
}
DomainMesh::~DomainMesh() {}
size_t DomainMesh::numberPointsVar( VariableType type ) const
{
size_t N = 0;
if ( type==VariableType::NodeVariable )
N = (nx+1)*(ny+1)*(nz+1);
else if ( type==VariableType::SurfaceVariable )
N = (nx+1)*ny*nz + nx*(ny+1)*nz + nx*ny*(nz+1);
else if ( type==VariableType::VolumeVariable )
N = nx*ny*nz;
if ( type == VariableType::NodeVariable )
N = ( nx + 1 ) * ( ny + 1 ) * ( nz + 1 );
else if ( type == VariableType::SurfaceVariable )
N = ( nx + 1 ) * ny * nz + nx * ( ny + 1 ) * nz + nx * ny * ( nz + 1 );
else if ( type == VariableType::VolumeVariable )
N = nx * ny * nz;
return N;
}
std::pair<size_t,void*> DomainMesh::pack( int level ) const
std::pair<size_t, void *> DomainMesh::pack( int level ) const
{
std::pair<size_t,void*> data(0,NULL);
data.first = 7*sizeof(double);
std::pair<size_t, void *> data( 0, NULL );
data.first = 7 * sizeof( double );
data.second = new double[7];
memset(data.second,0,7*sizeof(double));
int *data_int = reinterpret_cast<int*>(data.second);
double *data_double = &reinterpret_cast<double*>(data.second)[4];
data_int[0] = nprocx;
data_int[1] = nprocy;
data_int[2] = nprocz;
data_int[3] = rank;
data_int[4] = nx;
data_int[5] = ny;
data_int[6] = nz;
data_double[0] = Lx;
data_double[1] = Ly;
data_double[2] = Lz;
memset( data.second, 0, 7 * sizeof( double ) );
int *data_int = reinterpret_cast<int *>( data.second );
double *data_double = &reinterpret_cast<double *>( data.second )[4];
data_int[0] = nprocx;
data_int[1] = nprocy;
data_int[2] = nprocz;
data_int[3] = rank;
data_int[4] = nx;
data_int[5] = ny;
data_int[6] = nz;
data_double[0] = Lx;
data_double[1] = Ly;
data_double[2] = Lz;
return data;
}
void DomainMesh::unpack( const std::pair<size_t,void*>& data )
void DomainMesh::unpack( const std::pair<size_t, void *> &data )
{
const int *data_int = reinterpret_cast<const int*>(data.second);
const double *data_double = &reinterpret_cast<const double*>(data.second)[4];
nprocx = data_int[0];
nprocy = data_int[1];
nprocz = data_int[2];
rank = data_int[3];
nx = data_int[4];
ny = data_int[5];
nz = data_int[6];
Lx = data_double[0];
Ly = data_double[1];
Lz = data_double[2];
const int *data_int = reinterpret_cast<const int *>( data.second );
const double *data_double = &reinterpret_cast<const double *>( data.second )[4];
nprocx = data_int[0];
nprocy = data_int[1];
nprocz = data_int[2];
rank = data_int[3];
nx = data_int[4];
ny = data_int[5];
nz = data_int[6];
Lx = data_double[0];
Ly = data_double[1];
Lz = data_double[2];
}
/****************************************************
* Converters *
****************************************************/
* Converters *
****************************************************/
std::shared_ptr<PointList> getPointList( std::shared_ptr<Mesh> mesh )
{
return std::dynamic_pointer_cast<PointList>(mesh);
return std::dynamic_pointer_cast<PointList>( mesh );
}
std::shared_ptr<TriMesh> getTriMesh( std::shared_ptr<Mesh> mesh )
{
std::shared_ptr<TriMesh> mesh2;
if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriMesh>(mesh);
} else if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>(mesh);
ASSERT(trilist.get()!=NULL);
mesh2.reset( new TriMesh(*trilist) );
if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriMesh>( mesh );
} else if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
std::shared_ptr<TriList> trilist = std::dynamic_pointer_cast<TriList>( mesh );
ASSERT( trilist.get() != NULL );
mesh2.reset( new TriMesh( *trilist ) );
}
return mesh2;
}
std::shared_ptr<TriList> getTriList( std::shared_ptr<Mesh> mesh )
{
std::shared_ptr<TriList> mesh2;
if ( std::dynamic_pointer_cast<TriList>(mesh).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriList>(mesh);
} else if ( std::dynamic_pointer_cast<TriMesh>(mesh).get() != NULL ) {
std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>(mesh);
ASSERT(trimesh.get()!=NULL);
mesh2.reset( new TriList(*trimesh) );
if ( std::dynamic_pointer_cast<TriList>( mesh ).get() != NULL ) {
mesh2 = std::dynamic_pointer_cast<TriList>( mesh );
} else if ( std::dynamic_pointer_cast<TriMesh>( mesh ).get() != NULL ) {
std::shared_ptr<TriMesh> trimesh = std::dynamic_pointer_cast<TriMesh>( mesh );
ASSERT( trimesh.get() != NULL );
mesh2.reset( new TriList( *trimesh ) );
}
return mesh2;
}
std::shared_ptr<const PointList> getPointList( std::shared_ptr<const Mesh> mesh )
{
return getPointList( std::const_pointer_cast<Mesh>(mesh) );
return getPointList( std::const_pointer_cast<Mesh>( mesh ) );
}
std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh )
{
return getTriMesh( std::const_pointer_cast<Mesh>(mesh) );
return getTriMesh( std::const_pointer_cast<Mesh>( mesh ) );
}
std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh )
{
return getTriList( std::const_pointer_cast<Mesh>(mesh) );
return getTriList( std::const_pointer_cast<Mesh>( mesh ) );
}
} // IO namespace
/****************************************************
* Convert enum values *
****************************************************/
std::string getString( VariableType type )
{
if ( type == VariableType::NodeVariable )
return "node";
else if ( type == VariableType::EdgeVariable )
return "edge";
else if ( type == VariableType::SurfaceVariable )
return "face";
else if ( type == VariableType::VolumeVariable )
return "cell";
else if ( type == VariableType::NullVariable )
return "null";
else
ERROR( "Invalid type" );
return "";
}
VariableType getVariableType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "node" )
return VariableType::NodeVariable;
else if ( type == "edge" || type == "1" )
return VariableType::EdgeVariable;
else if ( type == "face" )
return VariableType::SurfaceVariable;
else if ( type == "cell" || type == "3" )
return VariableType::VolumeVariable;
else if ( type == "null" )
return VariableType::NullVariable;
else
ERROR( "Invalid type: " + type );
return VariableType::NullVariable;
}
std::string getString( DataType type )
{
if ( type == DataType::Double )
return "double";
else if ( type == DataType::Float )
return "float";
else if ( type == DataType::Int )
return "int";
else if ( type == DataType::Null )
return "null";
else
ERROR( "Invalid type" );
return "";
}
DataType getDataType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "double" )
return DataType::Double;
else if ( type == "float" )
return DataType::Float;
else if ( type == "int" )
return DataType::Int;
else if ( type == "null" )
return DataType::Null;
else
ERROR( "Invalid type: " + type );
return DataType::Null;
}
std::string getString( MeshType type )
{
if ( type == MeshType::PointMesh )
return "PointMesh";
else if ( type == MeshType::SurfaceMesh )
return "SurfaceMesh";
else if ( type == MeshType::VolumeMesh )
return "VolumeMesh";
else if ( type == MeshType::Unknown )
return "unknown";
else
ERROR( "Invalid type" );
return "";
}
MeshType getMeshType( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "PointMesh" || type == "1" )
return MeshType::PointMesh;
else if ( type == "SurfaceMesh" || type == "2" )
return MeshType::SurfaceMesh;
else if ( type == "VolumeMesh" || type == "3" )
return MeshType::VolumeMesh;
else if ( type == "unknown" || type == "-1" )
return MeshType::Unknown;
else
ERROR( "Invalid type: " + type );
return MeshType::Unknown;
}
std::string getString( FileFormat type )
{
if ( type == FileFormat::OLD )
return "old";
else if ( type == FileFormat::NEW )
return "new";
else if ( type == FileFormat::NEW_SINGLE )
return "new(single)";
else if ( type == FileFormat::SILO )
return "silo";
else
ERROR( "Invalid type" );
return "";
}
FileFormat getFileFormat( const std::string &type_in )
{
auto type = deblank( type_in );
if ( type == "old" || type == "1" )
return FileFormat::OLD;
else if ( type == "new" || type == "2" )
return FileFormat::NEW;
else if ( type == "new(single)" || type == "3" )
return FileFormat::NEW_SINGLE;
else if ( type == "silo" || type == "4" )
return FileFormat::SILO;
else
ERROR( "Invalid type: " + type );
return FileFormat::SILO;
}
} // namespace IO

133
IO/Mesh.h
View File

@ -21,17 +21,36 @@
#include <string.h>
#include <vector>
#include "analysis/PointList.h"
#include "common/Array.h"
#include "common/Communication.h"
#include "analysis/PointList.h"
namespace IO {
//! Possible variable types
enum class VariableType: unsigned char { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class DataType: unsigned char { Double=1, Float=2, Int=2, Null=0 };
//! Enums to define types
enum class VariableType {
NodeVariable,
EdgeVariable,
SurfaceVariable,
VolumeVariable,
NullVariable
};
enum class DataType { Double, Float, Int, Null };
enum class MeshType { PointMesh, SurfaceMesh, VolumeMesh, Unknown };
enum class FileFormat { OLD, NEW, NEW_SINGLE, SILO };
//! Convert enums to/from strings (more future-proof than static_cast<int>)
std::string getString( VariableType );
std::string getString( DataType );
std::string getString( MeshType );
std::string getString( FileFormat );
VariableType getVariableType( const std::string & );
DataType getDataType( const std::string & );
MeshType getMeshType( const std::string & );
FileFormat getFileFormat( const std::string & );
/*! \class Mesh
@ -47,21 +66,22 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const = 0;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const = 0;
virtual std::pair<size_t, void *> pack( int level ) const = 0;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data ) = 0;
virtual void unpack( const std::pair<size_t, void *> &data ) = 0;
protected:
//! Empty constructor
Mesh();
Mesh(const Mesh&);
Mesh& operator=(const Mesh&);
Mesh( const Mesh & );
Mesh &operator=( const Mesh & );
};
/*! \class PointList
\brief A class used to hold a list of verticies
*/
class PointList: public Mesh
class PointList : public Mesh
{
public:
//! Empty constructor
@ -75,13 +95,14 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
//! Access the points
const std::vector<Point>& getPoints() const { return points; }
const std::vector<Point> &getPoints() const { return points; }
public:
std::vector<Point> points; //!< List of points vertex
std::vector<Point> points; //!< List of points vertex
};
@ -89,7 +110,7 @@ public:
\brief A class used to hold a list of triangles specified by their vertex coordinates
*/
class TriMesh;
class TriList: public Mesh
class TriList : public Mesh
{
public:
//! Empty constructor
@ -97,7 +118,7 @@ public:
//! Constructor for N triangles
TriList( size_t N_tri );
//! Constructor from TriMesh
TriList( const TriMesh& );
TriList( const TriMesh & );
//! Destructor
virtual ~TriList();
//! Mesh class name
@ -105,20 +126,22 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::vector<Point> A; //!< First vertex
std::vector<Point> B; //!< Second vertex
std::vector<Point> C; //!< Third vertex
std::vector<Point> A; //!< First vertex
std::vector<Point> B; //!< Second vertex
std::vector<Point> C; //!< Third vertex
};
/*! \class TriMesh
\brief A class used to hold a list of trianges specified by their vertex number and list of coordiantes
\brief A class used to hold a list of trianges specified by their vertex number and list of
coordiantes
*/
class TriMesh: public Mesh
class TriMesh : public Mesh
{
public:
//! TriMesh constructor
@ -128,7 +151,7 @@ public:
//! Constructor for Nt triangles and the given points
TriMesh( size_t N_tri, std::shared_ptr<PointList> points );
//! Constructor from TriList
TriMesh( const TriList& );
TriMesh( const TriList & );
//! Destructor
virtual ~TriMesh();
//! Mesh class name
@ -136,21 +159,22 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
std::shared_ptr<PointList> vertices; //!< List of verticies
std::vector<int> A; //!< First vertex
std::vector<int> B; //!< Second vertex
std::vector<int> C; //!< Third vertex
std::shared_ptr<PointList> vertices; //!< List of verticies
std::vector<int> A; //!< First vertex
std::vector<int> B; //!< Second vertex
std::vector<int> C; //!< Third vertex
};
/*! \class Domain
\brief A class used to hold the domain
*/
class DomainMesh: public Mesh
class DomainMesh : public Mesh
{
public:
//! Empty constructor
@ -164,9 +188,10 @@ public:
//! Number of points for the given variable type
virtual size_t numberPointsVar( VariableType type ) const;
//! Pack the data
virtual std::pair<size_t,void*> pack( int level ) const;
virtual std::pair<size_t, void *> pack( int level ) const;
//! Unpack the data
virtual void unpack( const std::pair<size_t,void*>& data );
virtual void unpack( const std::pair<size_t, void *> &data );
public:
int nprocx, nprocy, nprocz, rank;
int nx, ny, nz;
@ -174,37 +199,40 @@ public:
};
/*! \class Variable
\brief A base class for variables
*/
struct Variable
{
struct Variable {
public:
// Internal variables
unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
VariableType type; //!< Variable type
DataType precision; //!< Variable precision to use for IO
std::string name; //!< Variable name
Array<double> data; //!< Variable data
unsigned char dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
VariableType type; //!< Variable type
DataType precision; //!< Variable precision to use for IO
std::string name; //!< Variable name
Array<double> data; //!< Variable data
//! Empty constructor
Variable(): dim(0), type(VariableType::NullVariable), precision(DataType::Double) {}
Variable() : dim( 0 ), type( VariableType::NullVariable ), precision( DataType::Double ) {}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_) {}
Variable( int dim_, IO::VariableType type_, const std::string &name_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ )
{
}
//! Constructor
Variable( int dim_, IO::VariableType type_, const std::string& name_, const Array<double>& data_ ):
dim(dim_), type(type_), precision(DataType::Double), name(name_), data(data_) {}
Variable(
int dim_, IO::VariableType type_, const std::string &name_, const Array<double> &data_ )
: dim( dim_ ), type( type_ ), precision( DataType::Double ), name( name_ ), data( data_ )
{
}
//! Destructor
virtual ~Variable() {}
protected:
//! Empty constructor
Variable(const Variable&);
Variable& operator=(const Variable&);
Variable( const Variable & );
Variable &operator=( const Variable & );
};
/*! \class MeshDataStruct
\brief A class used to hold database info for saving a mesh
*/
@ -212,11 +240,11 @@ struct MeshDataStruct {
DataType precision; //!< Precision to use for IO (mesh)
std::string meshName; //!< Mesh name
std::shared_ptr<Mesh> mesh; //!< Mesh data
std::vector<std::shared_ptr<Variable> > vars;
std::vector<std::shared_ptr<Variable>> vars;
//! Empty constructor
MeshDataStruct(): precision(DataType::Double) {}
MeshDataStruct() : precision( DataType::Double ) {}
//! Check the data
bool check() const;
bool check( bool abort = true ) const;
};
@ -229,7 +257,6 @@ std::shared_ptr<const TriMesh> getTriMesh( std::shared_ptr<const Mesh> mesh );
std::shared_ptr<const TriList> getTriList( std::shared_ptr<const Mesh> mesh );
} // IO namespace
} // namespace IO
#endif

602
IO/MeshDatabase.cpp
View File

@ -29,133 +29,146 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/MeshDatabase.h"
#include "IO/Mesh.h"
#include "IO/IOHelpers.h"
#include "common/MPI_Helpers.h"
#include "IO/Mesh.h"
#include "IO/PackData.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <vector>
#include <cstdio>
#include <map>
#include <set>
#include <cstdio>
#include <vector>
#include <ProfilerApp.h>
// Default pack/unpack
// clang-format off
#define INSTANTIATE_PACK( TYPE ) \
template<> \
size_t packsize<TYPE>( const TYPE &rhs ) \
{ \
return sizeof( TYPE ); \
} \
template<> \
void pack<TYPE>( const TYPE &rhs, char *buffer ) \
{ \
memcpy( buffer, &rhs, sizeof( IO::MeshType ) ); \
} \
template<> \
void unpack<TYPE>( TYPE &data, const char *buffer ) \
{ \
memcpy( &data, buffer, sizeof( IO::MeshType ) ); \
}
INSTANTIATE_PACK( IO::VariableType )
INSTANTIATE_PACK( IO::DataType )
INSTANTIATE_PACK( IO::MeshType )
INSTANTIATE_PACK( IO::FileFormat )
// clang-format on
/****************************************************
****************************************************/
// MeshType
template<>
size_t packsize<IO::MeshType>( const IO::MeshType& rhs )
{
return sizeof(IO::MeshType);
}
template<>
void pack<IO::MeshType>( const IO::MeshType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::MeshType));
}
template<>
void unpack<IO::MeshType>( IO::MeshType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::MeshType));
}
// Variable::VariableType
template<>
size_t packsize<IO::VariableType>( const IO::VariableType& rhs )
{
return sizeof(IO::VariableType);
}
template<>
void pack<IO::VariableType>( const IO::VariableType& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(IO::VariableType));
}
template<>
void unpack<IO::VariableType>( IO::VariableType& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(IO::VariableType));
}
// DatabaseEntry
template<>
size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs )
size_t packsize<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs )
{
return packsize(rhs.name)+packsize(rhs.file)+packsize(rhs.offset);
return packsize( rhs.name ) + packsize( rhs.file ) + packsize( rhs.offset );
}
template<>
void pack<IO::DatabaseEntry>( const IO::DatabaseEntry& rhs, char *buffer )
void pack<IO::DatabaseEntry>( const IO::DatabaseEntry &rhs, char *buffer )
{
size_t i=0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.file,&buffer[i]); i+=packsize(rhs.file);
pack(rhs.offset,&buffer[i]); i+=packsize(rhs.offset);
size_t i = 0;
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.file, &buffer[i] );
i += packsize( rhs.file );
pack( rhs.offset, &buffer[i] );
i += packsize( rhs.offset );
}
template<>
void unpack<IO::DatabaseEntry>( IO::DatabaseEntry& data, const char *buffer )
void unpack<IO::DatabaseEntry>( IO::DatabaseEntry &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.file,&buffer[i]); i+=packsize(data.file);
unpack(data.offset,&buffer[i]); i+=packsize(data.offset);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.file, &buffer[i] );
i += packsize( data.file );
unpack( data.offset, &buffer[i] );
i += packsize( data.offset );
}
// VariableDatabase
template<>
size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase& rhs )
size_t packsize<IO::VariableDatabase>( const IO::VariableDatabase &rhs )
{
return packsize(rhs.name)+packsize(rhs.type)+packsize(rhs.dim);
return packsize( rhs.name ) + packsize( rhs.type ) + packsize( rhs.dim );
}
template<>
void pack<IO::VariableDatabase>( const IO::VariableDatabase& rhs, char *buffer )
void pack<IO::VariableDatabase>( const IO::VariableDatabase &rhs, char *buffer )
{
size_t i=0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.dim,&buffer[i]); i+=packsize(rhs.dim);
size_t i = 0;
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.dim, &buffer[i] );
i += packsize( rhs.dim );
}
template<>
void unpack<IO::VariableDatabase>( IO::VariableDatabase& data, const char *buffer )
void unpack<IO::VariableDatabase>( IO::VariableDatabase &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.dim,&buffer[i]); i+=packsize(data.dim);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.dim, &buffer[i] );
i += packsize( data.dim );
}
// MeshDatabase
template<>
size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase& data )
size_t packsize<IO::MeshDatabase>( const IO::MeshDatabase &data )
{
return packsize(data.name)
+ packsize(data.type)
+ packsize(data.meshClass)
+ packsize(data.format)
+ packsize(data.domains)
+ packsize(data.variables)
+ packsize(data.variable_data);
return packsize( data.name ) + packsize( data.type ) + packsize( data.meshClass ) +
packsize( data.format ) + packsize( data.domains ) + packsize( data.variables ) +
packsize( data.variable_data );
}
template<>
void pack<IO::MeshDatabase>( const IO::MeshDatabase& rhs, char *buffer )
void pack<IO::MeshDatabase>( const IO::MeshDatabase &rhs, char *buffer )
{
size_t i = 0;
pack(rhs.name,&buffer[i]); i+=packsize(rhs.name);
pack(rhs.type,&buffer[i]); i+=packsize(rhs.type);
pack(rhs.meshClass,&buffer[i]); i+=packsize(rhs.meshClass);
pack(rhs.format,&buffer[i]); i+=packsize(rhs.format);
pack(rhs.domains,&buffer[i]); i+=packsize(rhs.domains);
pack(rhs.variables,&buffer[i]); i+=packsize(rhs.variables);
pack(rhs.variable_data,&buffer[i]); i+=packsize(rhs.variable_data);
pack( rhs.name, &buffer[i] );
i += packsize( rhs.name );
pack( rhs.type, &buffer[i] );
i += packsize( rhs.type );
pack( rhs.meshClass, &buffer[i] );
i += packsize( rhs.meshClass );
pack( rhs.format, &buffer[i] );
i += packsize( rhs.format );
pack( rhs.domains, &buffer[i] );
i += packsize( rhs.domains );
pack( rhs.variables, &buffer[i] );
i += packsize( rhs.variables );
pack( rhs.variable_data, &buffer[i] );
i += packsize( rhs.variable_data );
}
template<>
void unpack<IO::MeshDatabase>( IO::MeshDatabase& data, const char *buffer )
void unpack<IO::MeshDatabase>( IO::MeshDatabase &data, const char *buffer )
{
size_t i=0;
unpack(data.name,&buffer[i]); i+=packsize(data.name);
unpack(data.type,&buffer[i]); i+=packsize(data.type);
unpack(data.meshClass,&buffer[i]); i+=packsize(data.meshClass);
unpack(data.format,&buffer[i]); i+=packsize(data.format);
unpack(data.domains,&buffer[i]); i+=packsize(data.domains);
unpack(data.variables,&buffer[i]); i+=packsize(data.variables);
unpack(data.variable_data,&buffer[i]); i+=packsize(data.variable_data);
size_t i = 0;
unpack( data.name, &buffer[i] );
i += packsize( data.name );
unpack( data.type, &buffer[i] );
i += packsize( data.type );
unpack( data.meshClass, &buffer[i] );
i += packsize( data.meshClass );
unpack( data.format, &buffer[i] );
i += packsize( data.format );
unpack( data.domains, &buffer[i] );
i += packsize( data.domains );
unpack( data.variables, &buffer[i] );
i += packsize( data.variables );
unpack( data.variable_data, &buffer[i] );
i += packsize( data.variable_data );
}
@ -163,79 +176,72 @@ namespace IO {
/****************************************************
* VariableDatabase *
****************************************************/
bool VariableDatabase::operator==(const VariableDatabase& rhs ) const
* VariableDatabase *
****************************************************/
bool VariableDatabase::operator==( const VariableDatabase &rhs ) const
{
return type==rhs.type && dim==rhs.dim && name==rhs.name;
return type == rhs.type && dim == rhs.dim && name == rhs.name;
}
bool VariableDatabase::operator!=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator!=( const VariableDatabase &rhs ) const
{
return type!=rhs.type || dim!=rhs.dim || name!=rhs.name;
return type != rhs.type || dim != rhs.dim || name != rhs.name;
}
bool VariableDatabase::operator>=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator>=( const VariableDatabase &rhs ) const
{
return operator>(rhs) || operator==(rhs);
return operator>( rhs ) || operator==( rhs );
}
bool VariableDatabase::operator<=(const VariableDatabase& rhs ) const
bool VariableDatabase::operator<=( const VariableDatabase &rhs ) const { return !operator>( rhs ); }
bool VariableDatabase::operator>( const VariableDatabase &rhs ) const
{
return !operator>(rhs);
}
bool VariableDatabase::operator>(const VariableDatabase& rhs ) const
{
if ( name>rhs.name )
if ( name > rhs.name )
return true;
else if ( name<rhs.name )
else if ( name < rhs.name )
return false;
if ( type>rhs.type )
if ( type > rhs.type )
return true;
else if ( type<rhs.type )
else if ( type < rhs.type )
return false;
if ( dim>rhs.dim )
if ( dim > rhs.dim )
return true;
else if ( dim<rhs.dim )
else if ( dim < rhs.dim )
return false;
return false;
}
bool VariableDatabase::operator<(const VariableDatabase& rhs ) const
bool VariableDatabase::operator<( const VariableDatabase &rhs ) const
{
return !operator>(rhs) && operator!=(rhs);
return !operator>( rhs ) && operator!=( rhs );
}
/****************************************************
* MeshDatabase *
****************************************************/
MeshDatabase::MeshDatabase()
* MeshDatabase *
****************************************************/
MeshDatabase::MeshDatabase() {}
MeshDatabase::~MeshDatabase() {}
MeshDatabase::MeshDatabase( const MeshDatabase &rhs )
{
}
MeshDatabase::~MeshDatabase()
{
}
MeshDatabase::MeshDatabase(const MeshDatabase& rhs)
{
name = rhs.name;
type = rhs.type;
meshClass = rhs.meshClass;
format = rhs.format;
domains = rhs.domains;
variables = rhs.variables;
name = rhs.name;
type = rhs.type;
meshClass = rhs.meshClass;
format = rhs.format;
domains = rhs.domains;
variables = rhs.variables;
variable_data = rhs.variable_data;
}
MeshDatabase& MeshDatabase::operator=(const MeshDatabase& rhs)
MeshDatabase &MeshDatabase::operator=( const MeshDatabase &rhs )
{
this->name = rhs.name;
this->type = rhs.type;
this->meshClass = rhs.meshClass;
this->format = rhs.format;
this->domains = rhs.domains;
this->variables = rhs.variables;
this->name = rhs.name;
this->type = rhs.type;
this->meshClass = rhs.meshClass;
this->format = rhs.format;
this->domains = rhs.domains;
this->variables = rhs.variables;
this->variable_data = rhs.variable_data;
return *this;
}
VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname ) const
VariableDatabase MeshDatabase::getVariableDatabase( const std::string &varname ) const
{
for (size_t i=0; i<variables.size(); i++) {
for ( size_t i = 0; i < variables.size(); i++ ) {
if ( variables[i].name == varname )
return variables[i];
}
@ -244,221 +250,219 @@ VariableDatabase MeshDatabase::getVariableDatabase( const std::string& varname )
/****************************************************
* DatabaseEntry *
****************************************************/
std::string DatabaseEntry::write( ) const
* DatabaseEntry *
****************************************************/
std::string DatabaseEntry::write() const
{
char tmp[1000];
sprintf(tmp,"%s; %s; %lu",name.c_str(),file.c_str(),offset);
return std::string(tmp);
sprintf( tmp, "%s; %s; %lu", name.c_str(), file.c_str(), offset );
return std::string( tmp );
}
DatabaseEntry::DatabaseEntry( const char* line )
DatabaseEntry::DatabaseEntry( const char *line )
{
std::vector<std::string> list = splitList(line,';');
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const char* line )
void DatabaseEntry::read( const char *line )
{
std::vector<std::string> list = splitList(line,';');
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
auto list = splitList( line, ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
}
void DatabaseEntry::read( const std::string& line )
void DatabaseEntry::read( const std::string &line )
{
std::vector<std::string> list = splitList(line.c_str(),';');
name = list[0];
file = list[1];
offset = atol(list[2].c_str());
auto list = splitList( line.c_str(), ';' );
name = list[0];
file = list[1];
offset = atol( list[2].c_str() );
}
// Gather the mesh databases from all processors
inline int tod( int N ) { return (N+7)/sizeof(double); }
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm )
inline int tod( int N ) { return ( N + 7 ) / sizeof( double ); }
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm )
{
#ifdef USE_MPI
PROFILE_START("gatherAll");
PROFILE_START("gatherAll-pack",2);
int size = MPI_WORLD_SIZE();
// First pack the mesh data to local buffers
int localsize = 0;
for (size_t i=0; i<meshes.size(); i++)
localsize += tod(packsize(meshes[i]));
auto localbuf = new double[localsize];
int pos = 0;
for (size_t i=0; i<meshes.size(); i++) {
pack( meshes[i], (char*) &localbuf[pos] );
pos += tod(packsize(meshes[i]));
}
PROFILE_STOP("gatherAll-pack",2);
// Get the number of bytes each processor will be sending/recieving
PROFILE_START("gatherAll-send1",2);
auto recvsize = new int[size];
MPI_Allgather(&localsize,1,MPI_INT,recvsize,1,MPI_INT,comm);
int globalsize = recvsize[0];
auto disp = new int[size];
disp[0] = 0;
for (int i=1; i<size; i++) {
disp[i] = disp[i-1] + recvsize[i];
globalsize += recvsize[i];
}
PROFILE_STOP("gatherAll-send1",2);
// Send/recv the global data
PROFILE_START("gatherAll-send2",2);
auto globalbuf = new double[globalsize];
MPI_Allgatherv(localbuf,localsize,MPI_DOUBLE,globalbuf,recvsize,disp,MPI_DOUBLE,comm);
PROFILE_STOP("gatherAll-send2",2);
// Unpack the data
PROFILE_START("gatherAll-unpack",2);
std::map<std::string,MeshDatabase> data;
pos = 0;
while ( pos < globalsize ) {
MeshDatabase tmp;
unpack(tmp,(char*)&globalbuf[pos]);
pos += tod(packsize(tmp));
std::map<std::string,MeshDatabase>::iterator it = data.find(tmp.name);
if ( it==data.end() ) {
data[tmp.name] = tmp;
} else {
for (size_t i=0; i<tmp.domains.size(); i++)
it->second.domains.push_back(tmp.domains[i]);
for (size_t i=0; i<tmp.variables.size(); i++)
it->second.variables.push_back(tmp.variables[i]);
it->second.variable_data.insert(tmp.variable_data.begin(),tmp.variable_data.end());
}
}
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it) {
// Get the unique variables
std::set<VariableDatabase> data2(it->second.variables.begin(),it->second.variables.end());
it->second.variables = std::vector<VariableDatabase>(data2.begin(),data2.end());
}
// Free temporary memory
delete [] localbuf;
delete [] recvsize;
delete [] disp;
delete [] globalbuf;
// Return the results
std::vector<MeshDatabase> data2(data.size());
size_t i=0;
for (std::map<std::string,MeshDatabase>::iterator it=data.begin(); it!=data.end(); ++it, ++i)
data2[i] = it->second;
PROFILE_STOP("gatherAll-unpack",2);
PROFILE_STOP("gatherAll");
return data2;
#else
if ( comm.getSize() == 1 )
return meshes;
#endif
PROFILE_START( "gatherAll" );
PROFILE_START( "gatherAll-pack", 2 );
int size = comm.getSize();
// First pack the mesh data to local buffers
int localsize = 0;
for ( size_t i = 0; i < meshes.size(); i++ )
localsize += tod( packsize( meshes[i] ) );
auto localbuf = new double[localsize];
int pos = 0;
for ( size_t i = 0; i < meshes.size(); i++ ) {
pack( meshes[i], (char *) &localbuf[pos] );
pos += tod( packsize( meshes[i] ) );
}
PROFILE_STOP( "gatherAll-pack", 2 );
// Get the number of bytes each processor will be sending/recieving
PROFILE_START( "gatherAll-send1", 2 );
auto recvsize = comm.allGather( localsize );
int globalsize = recvsize[0];
auto disp = new int[size];
disp[0] = 0;
for ( int i = 1; i < size; i++ ) {
disp[i] = disp[i - 1] + recvsize[i];
globalsize += recvsize[i];
}
PROFILE_STOP( "gatherAll-send1", 2 );
// Send/recv the global data
PROFILE_START( "gatherAll-send2", 2 );
auto globalbuf = new double[globalsize];
comm.allGather( localbuf, localsize, globalbuf, recvsize.data(), disp, true );
PROFILE_STOP( "gatherAll-send2", 2 );
// Unpack the data
PROFILE_START( "gatherAll-unpack", 2 );
std::map<std::string, MeshDatabase> data;
pos = 0;
while ( pos < globalsize ) {
MeshDatabase tmp;
unpack( tmp, (char *) &globalbuf[pos] );
pos += tod( packsize( tmp ) );
std::map<std::string, MeshDatabase>::iterator it = data.find( tmp.name );
if ( it == data.end() ) {
data[tmp.name] = tmp;
} else {
for ( size_t i = 0; i < tmp.domains.size(); i++ )
it->second.domains.push_back( tmp.domains[i] );
for ( size_t i = 0; i < tmp.variables.size(); i++ )
it->second.variables.push_back( tmp.variables[i] );
it->second.variable_data.insert( tmp.variable_data.begin(), tmp.variable_data.end() );
}
}
for ( auto it = data.begin(); it != data.end(); ++it ) {
// Get the unique variables
std::set<VariableDatabase> data2(
it->second.variables.begin(), it->second.variables.end() );
it->second.variables = std::vector<VariableDatabase>( data2.begin(), data2.end() );
}
// Free temporary memory
delete[] localbuf;
delete[] disp;
delete[] globalbuf;
// Return the results
std::vector<MeshDatabase> data2( data.size() );
size_t i = 0;
for ( auto it = data.begin(); it != data.end(); ++it, ++i )
data2[i] = it->second;
PROFILE_STOP( "gatherAll-unpack", 2 );
PROFILE_STOP( "gatherAll" );
return data2;
}
//! Write the mesh databases to a file
void write( const std::vector<MeshDatabase>& meshes, const std::string& filename )
void write( const std::vector<MeshDatabase> &meshes, const std::string &filename )
{
PROFILE_START("write");
FILE *fid = fopen(filename.c_str(),"wb");
for (size_t i=0; i<meshes.size(); i++) {
fprintf(fid,"%s\n",meshes[i].name.c_str());
fprintf(fid," type: %i\n",static_cast<int>(meshes[i].type));
fprintf(fid," meshClass: %s\n",meshes[i].meshClass.c_str());
fprintf(fid," format: %i\n",static_cast<int>(meshes[i].format));
for (size_t j=0; j<meshes[i].domains.size(); j++)
fprintf(fid," domain: %s\n",meshes[i].domains[j].write().c_str());
fprintf(fid," variables: ");
for (size_t j=0; j<meshes[i].variables.size(); j++) {
const VariableDatabase& var = meshes[i].variables[j];
fprintf(fid,"%s|%i|%i; ",var.name.c_str(),static_cast<int>(var.type),var.dim);
PROFILE_START( "write" );
FILE *fid = fopen( filename.c_str(), "wb" );
for ( size_t i = 0; i < meshes.size(); i++ ) {
fprintf( fid, "%s\n", meshes[i].name.c_str() );
fprintf( fid, " type: %s\n", getString( meshes[i].type ).data() );
fprintf( fid, " meshClass: %s\n", meshes[i].meshClass.c_str() );
fprintf( fid, " format: %s\n", getString( meshes[i].format ).data() );
for ( size_t j = 0; j < meshes[i].domains.size(); j++ )
fprintf( fid, " domain: %s\n", meshes[i].domains[j].write().c_str() );
fprintf( fid, " variables: " );
for ( size_t j = 0; j < meshes[i].variables.size(); j++ ) {
const VariableDatabase &var = meshes[i].variables[j];
fprintf( fid, "%s|%s|%i; ", var.name.data(), getString( var.type ).data(), var.dim );
}
fprintf(fid,"\n");
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
for (it=meshes[i].variable_data.begin(); it!=meshes[i].variable_data.end(); ++it) {
const char* domain = it->first.first.c_str();
const char* variable = it->first.second.c_str();
fprintf(fid," variable(%s,%s): %s\n",domain,variable,it->second.write().c_str());
fprintf( fid, "\n" );
for ( auto it = meshes[i].variable_data.begin(); it != meshes[i].variable_data.end();
++it ) {
const char *domain = it->first.first.c_str();
const char *variable = it->first.second.c_str();
fprintf(
fid, " variable(%s,%s): %s\n", domain, variable, it->second.write().c_str() );
}
}
fclose(fid);
PROFILE_STOP("write");
fclose( fid );
PROFILE_STOP( "write" );
}
//! Read the mesh databases from a file
std::vector<MeshDatabase> read( const std::string& filename )
std::vector<MeshDatabase> read( const std::string &filename )
{
std::vector<MeshDatabase> meshes;
PROFILE_START("read");
FILE *fid = fopen(filename.c_str(),"rb");
if ( fid==NULL )
ERROR("Error opening file");
PROFILE_START( "read" );
FILE *fid = fopen( filename.c_str(), "rb" );
if ( fid == NULL )
ERROR( "Error opening file" );
char *line = new char[10000];
while ( std::fgets(line,1000,fid) != NULL ) {
if ( line[0]<32 ) {
while ( std::fgets( line, 1000, fid ) != NULL ) {
if ( line[0] < 32 ) {
// Empty line
continue;
} else if ( line[0] != ' ' ) {
meshes.resize(meshes.size()+1);
std::string name(line);
name.resize(name.size()-1);
meshes.resize( meshes.size() + 1 );
std::string name( line );
name.resize( name.size() - 1 );
meshes.back().name = name;
} else if ( strncmp(line," format:",10)==0 ) {
meshes.back().format = static_cast<unsigned char>(atoi(&line[10]));
} else if ( strncmp(line," type:",8)==0 ) {
meshes.back().type = static_cast<MeshType>(atoi(&line[8]));
} else if ( strncmp(line," meshClass:",13)==0 ) {
meshes.back().meshClass = deblank(std::string(&line[13]));
} else if ( strncmp(line," domain:",10)==0 ) {
DatabaseEntry data(&line[10]);
meshes.back().domains.push_back(data);
} else if ( strncmp(line," variables:",13)==0 ) {
MeshDatabase& mesh = meshes.back();
std::vector<std::string> variables = splitList(&line[13],';');
mesh.variables.resize(variables.size());
for (size_t i=0; i<variables.size(); i++) {
std::vector<std::string> tmp = splitList(variables[i].c_str(),'|');
ASSERT(tmp.size()==3);
} else if ( strncmp( line, " format:", 10 ) == 0 ) {
meshes.back().format = getFileFormat( &line[10] );
} else if ( strncmp( line, " type:", 8 ) == 0 ) {
meshes.back().type = getMeshType( &line[8] );
} else if ( strncmp( line, " meshClass:", 13 ) == 0 ) {
meshes.back().meshClass = deblank( std::string( &line[13] ) );
} else if ( strncmp( line, " domain:", 10 ) == 0 ) {
DatabaseEntry data( &line[10] );
meshes.back().domains.push_back( data );
} else if ( strncmp( line, " variables:", 13 ) == 0 ) {
MeshDatabase &mesh = meshes.back();
std::vector<std::string> variables = splitList( &line[13], ';' );
mesh.variables.resize( variables.size() );
for ( size_t i = 0; i < variables.size(); i++ ) {
std::vector<std::string> tmp = splitList( variables[i].c_str(), '|' );
ASSERT( tmp.size() == 3 );
mesh.variables[i].name = tmp[0];
mesh.variables[i].type = static_cast<VariableType>(atoi(tmp[1].c_str()));
mesh.variables[i].dim = atoi(tmp[2].c_str());
mesh.variables[i].type = getVariableType( tmp[1] );
mesh.variables[i].dim = atoi( tmp[2].c_str() );
}
} else if ( strncmp(line," variable(",12)==0 ) {
size_t i1 = find(line,',');
size_t i2 = find(line,':');
std::string domain = deblank(std::string(line,12,i1-12));
std::string variable = deblank(std::string(line,i1+1,i2-i1-2));
std::pair<std::string,std::string> key(domain,variable);
DatabaseEntry data(&line[i2+1]);
meshes.back().variable_data.insert(
std::pair<std::pair<std::string,std::string>,DatabaseEntry>(key,data) );
} else if ( strncmp( line, " variable(", 12 ) == 0 ) {
size_t i1 = find( line, ',' );
size_t i2 = find( line, ':' );
std::string domain = deblank( std::string( line, 12, i1 - 12 ) );
std::string variable = deblank( std::string( line, i1 + 1, i2 - i1 - 2 ) );
std::pair<std::string, std::string> key( domain, variable );
DatabaseEntry data( &line[i2 + 1] );
meshes.back().variable_data.insert(
std::pair<std::pair<std::string, std::string>, DatabaseEntry>( key, data ) );
} else {
ERROR("Error reading line");
ERROR( "Error reading line" );
}
}
fclose(fid);
delete [] line;
PROFILE_STOP("read");
fclose( fid );
delete[] line;
PROFILE_STOP( "read" );
return meshes;
}
// Return the mesh type
IO::MeshType meshType( const IO::Mesh& mesh )
IO::MeshType meshType( const IO::Mesh &mesh )
{
IO::MeshType type = IO::Unknown;
IO::MeshType type = IO::MeshType::Unknown;
const std::string meshClass = mesh.className();
if ( meshClass=="PointList" ) {
type = IO::PointMesh;
} else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
type = IO::SurfaceMesh;
} else if ( meshClass=="DomainMesh" ) {
type = IO::VolumeMesh;
if ( meshClass == "PointList" ) {
type = IO::MeshType::PointMesh;
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
type = IO::MeshType::SurfaceMesh;
} else if ( meshClass == "DomainMesh" ) {
type = IO::MeshType::VolumeMesh;
} else {
ERROR("Unknown mesh");
ERROR( "Unknown mesh" );
}
return type;
}
} // IO namespace
} // namespace IO

75
IO/MeshDatabase.h
View File

@ -16,90 +16,85 @@
#ifndef MeshDatabase_INC
#define MeshDatabase_INC
#include "IO/Mesh.h"
#include "common/MPI_Helpers.h"
#include "IO/Mesh.h"
#include "common/MPI.h"
#include <iostream>
#include <map>
#include <memory>
#include <string.h>
#include <vector>
#include <map>
namespace IO {
class Mesh;
//! Enum to identify mesh type
//enum class MeshType : char { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
enum MeshType { PointMesh=1, SurfaceMesh=2, VolumeMesh=3, Unknown=-1 };
//! Helper struct for containing offsets for the mesh info
struct DatabaseEntry {
std::string name; //!< Name of the entry
std::string file; //!< Name of the file containing the entry
size_t offset; //!< Offset in the file to start reading
std::string write( ) const; //!< Convert the data to a string
void read( const char* line ); //!< Convert the string to data
void read( const std::string& line ); //!< Convert the string to data
DatabaseEntry( ) {} //!< Empty constructor
DatabaseEntry( const char* line ); //!< Convert the string to data
~DatabaseEntry() {} //!< Destructor
std::string name; //!< Name of the entry
std::string file; //!< Name of the file containing the entry
size_t offset; //!< Offset in the file to start reading
std::string write() const; //!< Convert the data to a string
void read( const char *line ); //!< Convert the string to data
void read( const std::string &line ); //!< Convert the string to data
DatabaseEntry() {} //!< Empty constructor
DatabaseEntry( const char *line ); //!< Convert the string to data
~DatabaseEntry() {} //!< Destructor
};
//! Structure to hold the info about the variables
struct VariableDatabase {
std::string name; //!< Name of the variable
IO::VariableType type; //!< Variable
unsigned int dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
std::string name; //!< Name of the variable
IO::VariableType type; //!< Variable
unsigned int dim; //!< Number of points per grid point (1: scalar, 3: vector, ...)
// Overload key operators
bool operator==(const VariableDatabase& rhs ) const;
bool operator!=(const VariableDatabase& rhs ) const;
bool operator>=(const VariableDatabase& rhs ) const;
bool operator<=(const VariableDatabase& rhs ) const;
bool operator> (const VariableDatabase& rhs ) const;
bool operator< (const VariableDatabase& rhs ) const;
bool operator==( const VariableDatabase &rhs ) const;
bool operator!=( const VariableDatabase &rhs ) const;
bool operator>=( const VariableDatabase &rhs ) const;
bool operator<=( const VariableDatabase &rhs ) const;
bool operator>( const VariableDatabase &rhs ) const;
bool operator<( const VariableDatabase &rhs ) const;
};
//! Structure to hold the info about the meshes
struct MeshDatabase {
typedef std::pair<std::string,std::string> variable_id;
typedef std::pair<std::string, std::string> variable_id;
std::string name; //!< Name of the mesh
MeshType type; //!< Mesh type
std::string meshClass; //!< Mesh class
unsigned char format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
FileFormat format; //!< Data format (1: old, 2: new, 3: new (single), 4: silo)
std::vector<DatabaseEntry> domains; //!< List of the domains
std::vector<VariableDatabase> variables; //!< List of the variables
std::map<variable_id,DatabaseEntry> variable_data; //!< Data for the variables
VariableDatabase getVariableDatabase( const std::string& varname ) const;
std::vector<VariableDatabase> variables; //!< List of the variables
std::map<variable_id, DatabaseEntry> variable_data; //!< Data for the variables
VariableDatabase getVariableDatabase( const std::string &varname ) const;
public:
MeshDatabase();
~MeshDatabase();
MeshDatabase(const MeshDatabase&);
MeshDatabase& operator=(const MeshDatabase&);
MeshDatabase( const MeshDatabase & );
MeshDatabase &operator=( const MeshDatabase & );
};
//! Gather the mesh databases from all processors
std::vector<MeshDatabase> gatherAll( const std::vector<MeshDatabase>& meshes, MPI_Comm comm );
std::vector<MeshDatabase> gatherAll(
const std::vector<MeshDatabase> &meshes, const Utilities::MPI &comm );
//! Write the mesh databases to a file
void write( const std::vector<MeshDatabase>& meshes, const std::string& filename );
void write( const std::vector<MeshDatabase> &meshes, const std::string &filename );
//! Read the mesh databases from a file
std::vector<MeshDatabase> read( const std::string& filename );
std::vector<MeshDatabase> read( const std::string &filename );
//! Return the mesh type
IO::MeshType meshType( const IO::Mesh& mesh );
IO::MeshType meshType( const IO::Mesh &mesh );
} // IO namespace
} // namespace IO
#endif

131
IO/PIO.cpp
View File

@ -14,12 +14,12 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/PIO.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include <cstring>
#include <fstream>
#include <string>
#include <cstring>
namespace IO {
@ -30,19 +30,18 @@ static ParallelStreamBuffer perr_buffer;
static ParallelStreamBuffer plog_buffer;
std::ostream pout(&pout_buffer);
std::ostream perr(&perr_buffer);
std::ostream plog(&plog_buffer);
std::ostream pout( &pout_buffer );
std::ostream perr( &perr_buffer );
std::ostream plog( &plog_buffer );
/****************************************************************************
* Functions to control logging *
****************************************************************************/
std::ofstream *global_filestream=NULL;
static void shutdownFilestream( )
* Functions to control logging *
****************************************************************************/
std::ofstream *global_filestream = NULL;
static void shutdownFilestream()
{
if ( global_filestream!=NULL ) {
if ( global_filestream != NULL ) {
global_filestream->flush();
global_filestream->close();
delete global_filestream;
@ -52,16 +51,16 @@ static void shutdownFilestream( )
void Utilities::logOnlyNodeZero( const std::string &filename )
{
int rank = 0;
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
if ( rank == 0 )
logAllNodes(filename,true);
logAllNodes( filename, true );
}
void Utilities::logAllNodes( const std::string &filename, bool singleStream )
{
if ( singleStream )
ERROR("Not implimented yet");
ERROR( "Not implimented yet" );
// If the filestream was open, then close it and reset streams
shutdownFilestream();
@ -70,33 +69,33 @@ void Utilities::logAllNodes( const std::string &filename, bool singleStream )
std::string full_filename = filename;
if ( !singleStream ) {
int rank = 0;
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
#ifdef USE_MPI
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
#endif
char tmp[100];
sprintf(tmp,".%04i",rank);
full_filename += std::string(tmp);
sprintf( tmp, ".%04i", rank );
full_filename += std::string( tmp );
}
global_filestream = new std::ofstream(full_filename.c_str());
global_filestream = new std::ofstream( full_filename.c_str() );
if ( !(*global_filestream) ) {
if ( !( *global_filestream ) ) {
delete global_filestream;
global_filestream = NULL;
perr << "PIO: Could not open log file ``" << full_filename << "''\n";
} else {
pout_buffer.setOutputStream(global_filestream);
pout_buffer.setOutputStream(&std::cout);
perr_buffer.setOutputStream(global_filestream);
perr_buffer.setOutputStream(&std::cerr);
plog_buffer.setOutputStream(global_filestream);
pout_buffer.setOutputStream( global_filestream );
pout_buffer.setOutputStream( &std::cout );
perr_buffer.setOutputStream( global_filestream );
perr_buffer.setOutputStream( &std::cerr );
plog_buffer.setOutputStream( global_filestream );
}
}
/****************************************************************************
* ParallelStreamBuffer class *
****************************************************************************/
void Utilities::stopLogging( )
* ParallelStreamBuffer class *
****************************************************************************/
void Utilities::stopLogging()
{
pout_buffer.reset();
perr_buffer.reset();
@ -108,77 +107,71 @@ void Utilities::stopLogging( )
/****************************************************************************
* ParallelStreamBuffer class *
****************************************************************************/
ParallelStreamBuffer::ParallelStreamBuffer( ):
d_rank(0), d_size(0), d_buffer_size(0), d_buffer(NULL)
* ParallelStreamBuffer class *
****************************************************************************/
ParallelStreamBuffer::ParallelStreamBuffer()
: d_rank( 0 ), d_size( 0 ), d_buffer_size( 0 ), d_buffer( NULL )
{
}
ParallelStreamBuffer:: ~ParallelStreamBuffer()
{
delete [] d_buffer;
}
void ParallelStreamBuffer::setOutputStream( std::ostream *stream )
{
d_stream.push_back( stream );
}
ParallelStreamBuffer::~ParallelStreamBuffer() { delete[] d_buffer; }
void ParallelStreamBuffer::setOutputStream( std::ostream *stream ) { d_stream.push_back( stream ); }
int ParallelStreamBuffer::sync()
{
for (size_t i=0; i<d_stream.size(); i++) {
std::ostream& stream = *d_stream[i];
for ( size_t i = 0; i < d_stream.size(); i++ ) {
std::ostream &stream = *d_stream[i];
stream << d_buffer;
}
d_size = 0;
memset(d_buffer,0,d_buffer_size);
memset( d_buffer, 0, d_buffer_size );
return 0;
}
void ParallelStreamBuffer::reserve( size_t size )
{
if ( size > d_buffer_size ) {
if ( d_buffer_size==0 ) {
if ( d_buffer_size == 0 ) {
d_buffer_size = 1024;
d_buffer = new char[d_buffer_size];
memset(d_buffer,0,d_buffer_size);
d_buffer = new char[d_buffer_size];
memset( d_buffer, 0, d_buffer_size );
}
while ( size > d_buffer_size ) {
char *tmp = d_buffer;
d_buffer_size *= 2;
d_buffer = new char[d_buffer_size];
memset(d_buffer,0,d_buffer_size);
memcpy(d_buffer,tmp,d_size);
delete [] tmp;
memset( d_buffer, 0, d_buffer_size );
memcpy( d_buffer, tmp, d_size );
delete[] tmp;
}
}
}
std::streamsize ParallelStreamBuffer::xsputn( const char* text, std::streamsize n )
std::streamsize ParallelStreamBuffer::xsputn( const char *text, std::streamsize n )
{
reserve(d_size+n);
memcpy(&d_buffer[d_size],text,n);
reserve( d_size + n );
memcpy( &d_buffer[d_size], text, n );
d_size += n;
if ( text[n-1]==0 || text[n-1]==10 ) { sync(); }
if ( text[n - 1] == 0 || text[n - 1] == 10 ) {
sync();
}
return n;
}
int ParallelStreamBuffer::overflow(int ch)
int ParallelStreamBuffer::overflow( int ch )
{
reserve(d_size+1);
reserve( d_size + 1 );
d_buffer[d_size] = ch;
d_size++;
if ( ch==0 || ch==10 ) { sync(); }
return std::char_traits<char>::to_int_type(ch);
if ( ch == 0 || ch == 10 ) {
sync();
}
return std::char_traits<char>::to_int_type( ch );
}
int ParallelStreamBuffer::underflow()
{
return -1;
}
void ParallelStreamBuffer::reset()
int ParallelStreamBuffer::underflow() { return -1; }
void ParallelStreamBuffer::reset()
{
sync();
d_stream.clear();
delete [] d_buffer;
d_buffer = NULL;
delete[] d_buffer;
d_buffer = NULL;
d_buffer_size = 0;
}
} // IO namespace
} // namespace IO

47
IO/PIO.h
View File

@ -32,7 +32,7 @@ extern std::ostream pout;
/*!
* Parallel output stream perr writes to the standard error from all nodes.
* Output is prepended with the processor number.
* Output is prepended with the processor number.
*/
extern std::ostream perr;
@ -60,12 +60,11 @@ inline int printp( const char *format, ... );
class ParallelStreamBuffer : public std::streambuf
{
public:
/*!
* Create a parallel buffer class. The object will require further
* initialization to set up the I/O streams and prefix string.
*/
ParallelStreamBuffer( );
ParallelStreamBuffer();
/*!
* Set the output file stream (multiple output streams are supported)
@ -75,26 +74,26 @@ public:
/*!
* The destructor simply deallocates any internal data
* buffers. It does not modify the output streams.
* buffers. It does not modify the output streams.
*/
virtual ~ParallelStreamBuffer();
/*!
* Synchronize the parallel buffer (called from streambuf).
*/
virtual int sync();
virtual int sync();
/**
* Write the specified number of characters into the output stream (called
* from streambuf).
*/
virtual std::streamsize xsputn(const char* text, std::streamsize n);
*/
virtual std::streamsize xsputn( const char *text, std::streamsize n );
/*!
* Write an overflow character into the parallel buffer (called from
* streambuf).
*/
virtual int overflow(int ch);
virtual int overflow( int ch );
/*!
* Read an overflow character from the parallel buffer (called from
@ -113,30 +112,30 @@ private:
size_t d_size;
size_t d_buffer_size;
char *d_buffer;
std::vector<std::ostream*> d_stream;
std::vector<std::ostream *> d_stream;
inline void reserve( size_t size );
};
namespace Utilities {
/*!
* Log messages for node zero only to the specified filename. All output
* to pout, perr, and plog on node zero will go to the log file.
*/
void logOnlyNodeZero( const std::string &filename );
/*!
* Log messages for node zero only to the specified filename. All output
* to pout, perr, and plog on node zero will go to the log file.
*/
void logOnlyNodeZero( const std::string &filename );
/*!
* Log messages from all nodes. The diagnostic data for processor XXXXX
* will be sent to a file with the name filename.XXXXX, where filename is
* the function argument.
*/
void logAllNodes( const std::string &filename, bool singleStream=false );
/*!
* Log messages from all nodes. The diagnostic data for processor XXXXX
* will be sent to a file with the name filename.XXXXX, where filename is
* the function argument.
*/
void logAllNodes( const std::string &filename, bool singleStream = false );
/*!
* Stop logging messages, flush buffers, and reset memory.
*/
void stopLogging( );
/*!
* Stop logging messages, flush buffers, and reset memory.
*/
void stopLogging();
} // namespace Utilities

12
IO/PIO.hpp
View File

@ -33,9 +33,9 @@
#include "IO/PIO.h"
#include <cstdio>
#include <iostream>
#include <stdarg.h>
#include <cstdio>
namespace IO {
@ -43,17 +43,17 @@ namespace IO {
inline int printp( const char *format, ... )
{
va_list ap;
va_start(ap,format);
va_list ap;
va_start( ap, format );
char tmp[1024];
int n = vsprintf(tmp,format,ap);
va_end(ap);
int n = vsprintf( tmp, format, ap );
va_end( ap );
pout << tmp;
pout.flush();
return n;
}
} // IO namespace
} // namespace IO
#endif

104
IO/PackData.cpp Normal file
View File

@ -0,0 +1,104 @@
#include "IO/PackData.h"
#include <string.h>
/********************************************************
* Concrete implimentations for packing/unpacking *
********************************************************/
// unsigned char
template<>
size_t packsize<unsigned char>( const unsigned char &rhs )
{
return sizeof( unsigned char );
}
template<>
void pack<unsigned char>( const unsigned char &rhs, char *buffer )
{
memcpy( buffer, &rhs, sizeof( unsigned char ) );
}
template<>
void unpack<unsigned char>( unsigned char &data, const char *buffer )
{
memcpy( &data, buffer, sizeof( unsigned char ) );
}
// char
template<>
size_t packsize<char>( const char &rhs )
{
return sizeof( char );
}
template<>
void pack<char>( const char &rhs, char *buffer )
{
memcpy( buffer, &rhs, sizeof( char ) );
}
template<>
void unpack<char>( char &data, const char *buffer )
{
memcpy( &data, buffer, sizeof( char ) );
}
// int
template<>
size_t packsize<int>( const int &rhs )
{
return sizeof( int );
}
template<>
void pack<int>( const int &rhs, char *buffer )
{
memcpy( buffer, &rhs, sizeof( int ) );
}
template<>
void unpack<int>( int &data, const char *buffer )
{
memcpy( &data, buffer, sizeof( int ) );
}
// unsigned int
template<>
size_t packsize<unsigned int>( const unsigned int &rhs )
{
return sizeof( unsigned int );
}
template<>
void pack<unsigned int>( const unsigned int &rhs, char *buffer )
{
memcpy( buffer, &rhs, sizeof( int ) );
}
template<>
void unpack<unsigned int>( unsigned int &data, const char *buffer )
{
memcpy( &data, buffer, sizeof( int ) );
}
// size_t
template<>
size_t packsize<size_t>( const size_t &rhs )
{
return sizeof( size_t );
}
template<>
void pack<size_t>( const size_t &rhs, char *buffer )
{
memcpy( buffer, &rhs, sizeof( size_t ) );
}
template<>
void unpack<size_t>( size_t &data, const char *buffer )
{
memcpy( &data, buffer, sizeof( size_t ) );
}
// std::string
template<>
size_t packsize<std::string>( const std::string &rhs )
{
return rhs.size() + 1;
}
template<>
void pack<std::string>( const std::string &rhs, char *buffer )
{
memcpy( buffer, rhs.c_str(), rhs.size() + 1 );
}
template<>
void unpack<std::string>( std::string &data, const char *buffer )
{
data = std::string( buffer );
}

77
IO/PackData.h Normal file
View File

@ -0,0 +1,77 @@
// This file contains unctions to pack/unpack data structures
#ifndef included_PackData
#define included_PackData
#include <map>
#include <set>
#include <vector>
//! Template function to return the buffer size required to pack a class
template<class TYPE>
size_t packsize( const TYPE &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const TYPE &rhs, char *buffer );
//! Template function to unpack a class from a buffer
template<class TYPE>
void unpack( TYPE &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::vector
template<class TYPE>
size_t packsize( const std::vector<TYPE> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::vector<TYPE> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::vector<TYPE> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::pair
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1, TYPE2> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::map
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1, TYPE2> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer );
//! Template function to return the buffer size required to pack a std::set
template<class TYPE>
size_t packsize( const std::set<TYPE> &rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::set<TYPE> &rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::set<TYPE> &data, const char *buffer );
#include "IO/PackData.hpp"
#endif

159
IO/PackData.hpp Normal file
View File

@ -0,0 +1,159 @@
// This file functions to pack/unpack data structures
#ifndef included_PackData_hpp
#define included_PackData_hpp
#include "IO/PackData.h"
#include <map>
#include <set>
#include <string.h>
#include <vector>
/********************************************************
* Default instantiations for std::vector *
********************************************************/
template<class TYPE>
size_t packsize( const std::vector<TYPE> &rhs )
{
size_t bytes = sizeof( size_t );
for ( size_t i = 0; i < rhs.size(); i++ )
bytes += packsize( rhs[i] );
return bytes;
}
template<class TYPE>
void pack( const std::vector<TYPE> &rhs, char *buffer )
{
size_t size = rhs.size();
memcpy( buffer, &size, sizeof( size_t ) );
size_t pos = sizeof( size_t );
for ( size_t i = 0; i < rhs.size(); i++ ) {
pack( rhs[i], &buffer[pos] );
pos += packsize( rhs[i] );
}
}
template<class TYPE>
void unpack( std::vector<TYPE> &data, const char *buffer )
{
size_t size;
memcpy( &size, buffer, sizeof( size_t ) );
data.clear();
data.resize( size );
size_t pos = sizeof( size_t );
for ( size_t i = 0; i < data.size(); i++ ) {
unpack( data[i], &buffer[pos] );
pos += packsize( data[i] );
}
}
/********************************************************
* Default instantiations for std::pair *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1, TYPE2> &rhs )
{
return packsize( rhs.first ) + packsize( rhs.second );
}
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1, TYPE2> &rhs, char *buffer )
{
pack( rhs.first, buffer );
pack( rhs.second, &buffer[packsize( rhs.first )] );
}
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1, TYPE2> &data, const char *buffer )
{
unpack( data.first, buffer );
unpack( data.second, &buffer[packsize( data.first )] );
}
/********************************************************
* Default instantiations for std::map *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1, TYPE2> &rhs )
{
size_t bytes = sizeof( size_t );
typename std::map<TYPE1, TYPE2>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
bytes += packsize( it->first );
bytes += packsize( it->second );
}
return bytes;
}
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1, TYPE2> &rhs, char *buffer )
{
size_t N = rhs.size();
pack( N, buffer );
size_t pos = sizeof( size_t );
typename std::map<TYPE1, TYPE2>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack( it->first, &buffer[pos] );
pos += packsize( it->first );
pack( it->second, &buffer[pos] );
pos += packsize( it->second );
}
}
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1, TYPE2> &data, const char *buffer )
{
size_t N = 0;
unpack( N, buffer );
size_t pos = sizeof( size_t );
data.clear();
for ( size_t i = 0; i < N; i++ ) {
std::pair<TYPE1, TYPE2> tmp;
unpack( tmp.first, &buffer[pos] );
pos += packsize( tmp.first );
unpack( tmp.second, &buffer[pos] );
pos += packsize( tmp.second );
data.insert( tmp );
}
}
/********************************************************
* Default instantiations for std::set *
********************************************************/
template<class TYPE>
size_t packsize( const std::set<TYPE> &rhs )
{
size_t bytes = sizeof( size_t );
typename std::set<TYPE>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
bytes += packsize( *it );
}
return bytes;
}
template<class TYPE>
void pack( const std::set<TYPE> &rhs, char *buffer )
{
size_t N = rhs.size();
pack( N, buffer );
size_t pos = sizeof( size_t );
typename std::set<TYPE>::const_iterator it;
for ( it = rhs.begin(); it != rhs.end(); ++it ) {
pack( *it );
pos += packsize( *it );
}
}
template<class TYPE>
void unpack( std::set<TYPE> &data, const char *buffer )
{
size_t N = 0;
unpack( N, buffer );
size_t pos = sizeof( size_t );
data.clear();
for ( size_t i = 0; i < N; i++ ) {
TYPE tmp;
unpack( tmp, &buffer[pos] );
pos += packsize( tmp );
data.insert( tmp );
}
}
#endif

468
IO/Reader.cpp
View File

@ -14,9 +14,9 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/Reader.h"
#include "IO/IOHelpers.h"
#include "IO/Mesh.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "common/Utilities.h"
#ifdef USE_SILO
@ -25,60 +25,139 @@
#include <ProfilerApp.h>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <map>
#include <string.h>
#include <vector>
#include <map>
#include <cstdio>
// Inline function to read line without a return argument
static inline void fgetl( char * str, int num, FILE * stream )
static inline void fgetl( char *str, int num, FILE *stream )
{
char* ptr = fgets( str, num, stream );
if ( 0 ) {char *temp = (char *)&ptr; temp++;}
char *ptr = fgets( str, num, stream );
if ( 0 ) {
char *temp = (char *) &ptr;
temp++;
}
}
// Check if the file exists
bool fileExists( const std::string &filename )
{
std::ifstream ifile( filename.c_str() );
return ifile.good();
}
// Get the path to a file
std::string IO::getPath( const std::string& filename )
std::string IO::getPath( const std::string &filename )
{
std::string file(filename);
size_t k1 = file.rfind(47);
size_t k2 = file.rfind(92);
if ( k1==std::string::npos ) { k1=0; }
if ( k2==std::string::npos ) { k2=0; }
return file.substr(0,std::max(k1,k2));
std::string file( filename );
size_t k1 = file.rfind( 47 );
size_t k2 = file.rfind( 92 );
if ( k1 == std::string::npos ) {
k1 = 0;
}
if ( k2 == std::string::npos ) {
k2 = 0;
}
return file.substr( 0, std::max( k1, k2 ) );
}
// List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string& filename )
// List the timesteps in the given directory (dumps.LBPM)
std::vector<std::string> IO::readTimesteps( const std::string &path, const std::string &format )
{
PROFILE_START("readTimesteps");
FILE *fid= fopen(filename.c_str(),"rb");
if ( fid==NULL )
ERROR("Error opening file");
// Get the name of the summary filename
std::string filename = path + "/";
if ( format == "old" || format == "new" ) {
filename += "summary.LBM";
} else if ( format == "silo" ) {
filename += "LBM.visit";
} else if ( format == "auto" ) {
bool test_old = fileExists( path + "/summary.LBM" );
bool test_silo = fileExists( path + "/LBM.visit" );
if ( test_old && test_silo ) {
ERROR( "Unable to determine format (both summary.LBM and LBM.visit exist)" );
} else if ( test_old ) {
filename += "summary.LBM";
} else if ( test_silo ) {
filename += "LBM.visit";
} else {
ERROR( "Unable to determine format (neither summary.LBM or LBM.visit exist)" );
}
} else {
ERROR( "Unknown format: " + format );
}
PROFILE_START( "readTimesteps" );
// Read the data
FILE *fid = fopen( filename.c_str(), "rb" );
if ( fid == NULL )
ERROR( "Error opening file" );
std::vector<std::string> timesteps;
char buf[1000];
while (fgets(buf,sizeof(buf),fid) != NULL) {
std::string line(buf);
line.resize(line.size()-1);
while ( fgets( buf, sizeof( buf ), fid ) != NULL ) {
std::string line( buf );
line.resize( line.size() - 1 );
auto pos = line.find( "summary.silo" );
if ( pos != std::string::npos )
line.resize(pos);
line.resize( pos );
if ( line.empty() )
continue;
timesteps.push_back(line);
timesteps.push_back( line );
}
fclose(fid);
PROFILE_STOP("readTimesteps");
fclose( fid );
PROFILE_STOP( "readTimesteps" );
return timesteps;
return timesteps;
}
// Get the maximum number of domains
int IO::maxDomains( const std::string &path, const std::string &format, const Utilities::MPI &comm )
{
int rank = comm.getRank();
int n_domains = 0;
if ( rank == 0 ) {
// Get the timesteps
auto timesteps = IO::readTimesteps( path, format );
ASSERT( !timesteps.empty() );
// Get the database for the first domain
auto db = IO::getMeshList( path, timesteps[0] );
for ( size_t i = 0; i < db.size(); i++ )
n_domains = std::max<int>( n_domains, db[i].domains.size() );
}
return comm.bcast( n_domains, 0 );
}
// Read the data for the given timestep
std::vector<IO::MeshDataStruct> IO::readData(
const std::string &path, const std::string &timestep, int rank )
{
// Get the mesh databases
auto db = IO::getMeshList( path, timestep );
// Create the data
std::vector<IO::MeshDataStruct> data( db.size() );
for ( size_t i = 0; i < data.size(); i++ ) {
data[i].precision = IO::DataType::Double;
data[i].meshName = db[i].name;
data[i].mesh = getMesh( path, timestep, db[i], rank );
data[i].vars.resize( db[i].variables.size() );
for ( size_t j = 0; j < db[i].variables.size(); j++ )
data[i].vars[j] = getVariable( path, timestep, db[i], rank, db[i].variables[j].name );
INSIST( data[i].check(), "Failed check of " + data[i].meshName );
}
return data;
}
// Read the list of variables for the given timestep
std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const std::string& timestep )
std::vector<IO::MeshDatabase> IO::getMeshList(
const std::string &path, const std::string &timestep )
{
std::string filename = path + "/" + timestep + "/LBM.summary";
return IO::read( filename );
@ -86,270 +165,271 @@ std::vector<IO::MeshDatabase> IO::getMeshList( const std::string& path, const st
// Read the given mesh domain
std::shared_ptr<IO::Mesh> IO::getMesh( const std::string& path, const std::string& timestep,
const IO::MeshDatabase& meshDatabase, int domain )
std::shared_ptr<IO::Mesh> IO::getMesh( const std::string &path, const std::string &timestep,
const IO::MeshDatabase &meshDatabase, int domain )
{
PROFILE_START("getMesh");
PROFILE_START( "getMesh" );
std::shared_ptr<IO::Mesh> mesh;
if ( meshDatabase.format==1 ) {
if ( meshDatabase.format == FileFormat::OLD ) {
// Old format (binary doubles)
std::string filename = path + "/" + timestep + "/" + meshDatabase.domains[domain].file;
FILE *fid = fopen(filename.c_str(),"rb");
INSIST(fid!=NULL,"Error opening file");
FILE *fid = fopen( filename.c_str(), "rb" );
INSIST( fid != NULL, "Error opening file" );
fseek( fid, 0, SEEK_END );
size_t bytes = ftell(fid);
size_t N_max = bytes/sizeof(double)+1000;
size_t bytes = ftell( fid );
size_t N_max = bytes / sizeof( double ) + 1000;
double *data = new double[N_max];
fseek(fid,0,SEEK_SET);
size_t count = fread(data,sizeof(double),N_max,fid);
fclose(fid);
if ( count%3 != 0 )
ERROR("Error reading file");
if ( meshDatabase.type==IO::PointMesh ) {
size_t N = count/3;
std::shared_ptr<PointList> pointlist( new PointList(N) );
std::vector<Point>& P = pointlist->points;
for (size_t i=0; i<N; i++) {
P[i].x = data[3*i+0];
P[i].y = data[3*i+1];
P[i].z = data[3*i+2];
fseek( fid, 0, SEEK_SET );
size_t count = fread( data, sizeof( double ), N_max, fid );
fclose( fid );
if ( count % 3 != 0 )
ERROR( "Error reading file" );
if ( meshDatabase.type == IO::MeshType::PointMesh ) {
size_t N = count / 3;
std::shared_ptr<PointList> pointlist( new PointList( N ) );
std::vector<Point> &P = pointlist->points;
for ( size_t i = 0; i < N; i++ ) {
P[i].x = data[3 * i + 0];
P[i].y = data[3 * i + 1];
P[i].z = data[3 * i + 2];
}
mesh = pointlist;
} else if ( meshDatabase.type==IO::SurfaceMesh ) {
if ( count%9 != 0 )
ERROR("Error reading file (2)");
size_t N_tri = count/9;
std::shared_ptr<TriList> trilist( new TriList(N_tri) );
std::vector<Point>& A = trilist->A;
std::vector<Point>& B = trilist->B;
std::vector<Point>& C = trilist->C;
for (size_t i=0; i<N_tri; i++) {
A[i].x = data[9*i+0];
A[i].y = data[9*i+1];
A[i].z = data[9*i+2];
B[i].x = data[9*i+3];
B[i].y = data[9*i+4];
B[i].z = data[9*i+5];
C[i].x = data[9*i+6];
C[i].y = data[9*i+7];
C[i].z = data[9*i+8];
} else if ( meshDatabase.type == IO::MeshType::SurfaceMesh ) {
if ( count % 9 != 0 )
ERROR( "Error reading file (2)" );
size_t N_tri = count / 9;
std::shared_ptr<TriList> trilist( new TriList( N_tri ) );
std::vector<Point> &A = trilist->A;
std::vector<Point> &B = trilist->B;
std::vector<Point> &C = trilist->C;
for ( size_t i = 0; i < N_tri; i++ ) {
A[i].x = data[9 * i + 0];
A[i].y = data[9 * i + 1];
A[i].z = data[9 * i + 2];
B[i].x = data[9 * i + 3];
B[i].y = data[9 * i + 4];
B[i].z = data[9 * i + 5];
C[i].x = data[9 * i + 6];
C[i].y = data[9 * i + 7];
C[i].z = data[9 * i + 8];
}
mesh = trilist;
} else if ( meshDatabase.type==IO::VolumeMesh ) {
} else if ( meshDatabase.type == IO::MeshType::VolumeMesh ) {
// this was never supported in the old format
mesh = std::shared_ptr<DomainMesh>( new DomainMesh() );
} else {
ERROR("Unknown mesh type");
ERROR( "Unknown mesh type" );
}
delete [] data;
} else if ( meshDatabase.format==2 ) {
const DatabaseEntry& database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen(filename.c_str(),"rb");
fseek(fid,database.offset,SEEK_SET);
delete[] data;
} else if ( meshDatabase.format == FileFormat::NEW ||
meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen( filename.c_str(), "rb" );
fseek( fid, database.offset, SEEK_SET );
char line[1000];
fgetl(line,1000,fid);
size_t i1 = find(line,':');
size_t i2 = find(&line[i1+1],':')+i1+1;
size_t bytes = atol(&line[i2+1]);
char *data = new char[bytes];
size_t count = fread(data,1,bytes,fid);
fclose(fid);
ASSERT(count==bytes);
if ( meshDatabase.meshClass=="PointList" ) {
fgetl( line, 1000, fid );
size_t i1 = find( line, ':' );
size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1;
size_t bytes = atol( &line[i2 + 1] );
char *data = new char[bytes];
size_t count = fread( data, 1, bytes, fid );
fclose( fid );
ASSERT( count == bytes );
if ( meshDatabase.meshClass == "PointList" ) {
mesh.reset( new IO::PointList() );
} else if ( meshDatabase.meshClass=="TriMesh" ) {
} else if ( meshDatabase.meshClass == "TriMesh" ) {
mesh.reset( new IO::TriMesh() );
} else if ( meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriList" ) {
mesh.reset( new IO::TriList() );
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
mesh.reset( new IO::DomainMesh() );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
mesh->unpack( std::pair<size_t,void*>(bytes,data) );
delete [] data;
} else if ( meshDatabase.format==4 ) {
mesh->unpack( std::pair<size_t, void *>( bytes, data ) );
delete[] data;
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const DatabaseEntry& database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
if ( meshDatabase.meshClass=="PointList" ) {
const DatabaseEntry &database = meshDatabase.domains[domain];
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
if ( meshDatabase.meshClass == "PointList" ) {
Array<double> coords = silo::readPointMesh<double>( fid, database.name );
ASSERT(coords.size(1)==3);
std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size(0) ) );
for (size_t i=0; i<coords.size(1); i++) {
mesh2->points[i].x = coords(i,0);
mesh2->points[i].y = coords(i,1);
mesh2->points[i].z = coords(i,2);
ASSERT( coords.size( 1 ) == 3 );
std::shared_ptr<IO::PointList> mesh2( new IO::PointList( coords.size( 0 ) ) );
for ( size_t i = 0; i < coords.size( 1 ); i++ ) {
mesh2->points[i].x = coords( i, 0 );
mesh2->points[i].y = coords( i, 1 );
mesh2->points[i].z = coords( i, 2 );
}
mesh = mesh2;
} else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
Array<double> coords;
Array<int> tri;
silo::readTriMesh( fid, database.name, coords, tri );
ASSERT( tri.size(1)==3 && coords.size(1)==3 );
int N_tri = tri.size(0);
int N_point = coords.size(0);
ASSERT( tri.size( 1 ) == 3 && coords.size( 1 ) == 3 );
int N_tri = tri.size( 0 );
int N_point = coords.size( 0 );
std::shared_ptr<IO::TriMesh> mesh2( new IO::TriMesh( N_tri, N_point ) );
for (int i=0; i<N_point; i++) {
mesh2->vertices->points[i].x = coords(i,0);
mesh2->vertices->points[i].y = coords(i,1);
mesh2->vertices->points[i].z = coords(i,2);
for ( int i = 0; i < N_point; i++ ) {
mesh2->vertices->points[i].x = coords( i, 0 );
mesh2->vertices->points[i].y = coords( i, 1 );
mesh2->vertices->points[i].z = coords( i, 2 );
}
for (int i=0; i<N_tri; i++) {
mesh2->A[i] = tri(i,0);
mesh2->B[i] = tri(i,1);
mesh2->C[i] = tri(i,2);
for ( int i = 0; i < N_tri; i++ ) {
mesh2->A[i] = tri( i, 0 );
mesh2->B[i] = tri( i, 1 );
mesh2->C[i] = tri( i, 2 );
}
if ( meshDatabase.meshClass=="TriMesh" ) {
if ( meshDatabase.meshClass == "TriMesh" ) {
mesh = mesh2;
} else if ( meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriList" ) {
auto trilist = IO::getTriList( std::dynamic_pointer_cast<IO::Mesh>( mesh2 ) );
mesh = trilist;
mesh = trilist;
}
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
std::vector<double> range;
std::vector<int> N;
silo::readUniformMesh( fid, database.name, range, N );
auto rankinfo = silo::read<int>( fid, database.name+"_rankinfo" );
auto rankinfo = silo::read<int>( fid, database.name + "_rankinfo" );
RankInfoStruct rank_data( rankinfo[0], rankinfo[1], rankinfo[2], rankinfo[3] );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1]-range[0], range[3]-range[2], range[5]-range[4] ) );
mesh.reset( new IO::DomainMesh( rank_data, N[0], N[1], N[2], range[1] - range[0],
range[3] - range[2], range[5] - range[4] ) );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
silo::close( fid );
#else
ERROR("Build without silo support");
ERROR( "Build without silo support" );
#endif
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
PROFILE_STOP("getMesh");
PROFILE_STOP( "getMesh" );
return mesh;
}
// Read the given variable for the given mesh domain
std::shared_ptr<IO::Variable> IO::getVariable( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain, const std::string& variable )
std::shared_ptr<IO::Variable> IO::getVariable( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain, const std::string &variable )
{
std::pair<std::string,std::string> key(meshDatabase.domains[domain].name,variable);
std::map<std::pair<std::string,std::string>,DatabaseEntry>::const_iterator it;
it = meshDatabase.variable_data.find(key);
if ( it==meshDatabase.variable_data.end() )
std::pair<std::string, std::string> key( meshDatabase.domains[domain].name, variable );
auto it = meshDatabase.variable_data.find( key );
if ( it == meshDatabase.variable_data.end() )
return std::shared_ptr<IO::Variable>();
std::shared_ptr<IO::Variable> var;
if ( meshDatabase.format == 2 ) {
const DatabaseEntry& database = it->second;
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen(filename.c_str(),"rb");
fseek(fid,database.offset,SEEK_SET);
if ( meshDatabase.format == FileFormat::NEW || meshDatabase.format == FileFormat::NEW_SINGLE ) {
const DatabaseEntry &database = it->second;
std::string filename = path + "/" + timestep + "/" + database.file;
FILE *fid = fopen( filename.c_str(), "rb" );
fseek( fid, database.offset, SEEK_SET );
char line[1000];
fgetl(line,1000,fid);
size_t i1 = find(line,':');
size_t i2 = find(&line[i1+1],':')+i1+1;
std::vector<std::string> values = splitList(&line[i2+1],',');
ASSERT(values.size()==5);
int dim = atoi(values[0].c_str());
int type = atoi(values[1].c_str());
size_t N = atol(values[2].c_str());
size_t bytes = atol(values[3].c_str());
std::string precision = values[4];
var = std::shared_ptr<IO::Variable>( new IO::Variable() );
var->dim = dim;
var->type = static_cast<IO::VariableType>(type);
var->name = variable;
var->data.resize(N*dim);
if ( precision=="double" ) {
size_t count = fread(var->data.data(),sizeof(double),N*dim,fid);
ASSERT(count*sizeof(double)==bytes);
fgetl( line, 1000, fid );
size_t i1 = find( line, ':' );
size_t i2 = find( &line[i1 + 1], ':' ) + i1 + 1;
std::vector<std::string> values = splitList( &line[i2 + 1], ',' );
ASSERT( values.size() == 5 );
int dim = atoi( values[0].c_str() );
auto type = values[1];
size_t N = atol( values[2].c_str() );
size_t bytes = atol( values[3].c_str() );
std::string precision = values[4];
var = std::shared_ptr<IO::Variable>( new IO::Variable() );
var->dim = dim;
var->type = getVariableType( type );
var->name = variable;
var->data.resize( N, dim );
if ( precision == "double" ) {
size_t count = fread( var->data.data(), sizeof( double ), N * dim, fid );
ASSERT( count * sizeof( double ) == bytes );
} else {
ERROR("Format not implimented");
ERROR( "Format not implimented" );
}
fclose(fid);
} else if ( meshDatabase.format == 4 ) {
fclose( fid );
} else if ( meshDatabase.format == FileFormat::SILO ) {
// Reading a silo file
#ifdef USE_SILO
const auto& database = meshDatabase.domains[domain];
const auto &database = meshDatabase.domains[domain];
auto variableDatabase = meshDatabase.getVariableDatabase( variable );
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
std::string filename = path + "/" + timestep + "/" + database.file;
auto fid = silo::open( filename, silo::READ );
var.reset( new Variable( variableDatabase.dim, variableDatabase.type, variable ) );
if ( meshDatabase.meshClass=="PointList" ) {
if ( meshDatabase.meshClass == "PointList" ) {
var->data = silo::readPointMeshVariable<double>( fid, variable );
} else if ( meshDatabase.meshClass=="TriMesh" || meshDatabase.meshClass=="TriList" ) {
} else if ( meshDatabase.meshClass == "TriMesh" || meshDatabase.meshClass == "TriList" ) {
var->data = silo::readTriMeshVariable<double>( fid, variable );
} else if ( meshDatabase.meshClass=="DomainMesh" ) {
} else if ( meshDatabase.meshClass == "DomainMesh" ) {
var->data = silo::readUniformMeshVariable<double>( fid, variable );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
silo::close( fid );
#else
ERROR("Build without silo support");
ERROR( "Build without silo support" );
#endif
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
return var;
}
/****************************************************
* Reformat the variable to match the mesh *
****************************************************/
void IO::reformatVariable( const IO::Mesh& mesh, IO::Variable& var )
* Reformat the variable to match the mesh *
****************************************************/
void IO::reformatVariable( const IO::Mesh &mesh, IO::Variable &var )
{
if ( mesh.className() == "DomainMesh" ) {
const IO::DomainMesh& mesh2 = dynamic_cast<const IO::DomainMesh&>( mesh );
const IO::DomainMesh &mesh2 = dynamic_cast<const IO::DomainMesh &>( mesh );
if ( var.type == VariableType::NodeVariable ) {
size_t N2 = var.data.length() / ((mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1));
ASSERT( (mesh2.nx+1)*(mesh2.ny+1)*(mesh2.nz+1)*N2 == var.data.length() );
var.data.reshape( { (size_t) mesh2.nx+1, (size_t) mesh2.ny+1, (size_t) mesh2.nz+1, N2 } );
size_t N2 =
var.data.length() / ( ( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) );
ASSERT(
( mesh2.nx + 1 ) * ( mesh2.ny + 1 ) * ( mesh2.nz + 1 ) * N2 == var.data.length() );
var.data.reshape(
{ (size_t) mesh2.nx + 1, (size_t) mesh2.ny + 1, (size_t) mesh2.nz + 1, N2 } );
} else if ( var.type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::VolumeVariable ) {
size_t N2 = var.data.length() / (mesh2.nx*mesh2.ny*mesh2.nz);
ASSERT( mesh2.nx*mesh2.ny*mesh2.nz*N2 == var.data.length() );
size_t N2 = var.data.length() / ( mesh2.nx * mesh2.ny * mesh2.nz );
ASSERT( mesh2.nx * mesh2.ny * mesh2.nz * N2 == var.data.length() );
var.data.reshape( { (size_t) mesh2.nx, (size_t) mesh2.ny, (size_t) mesh2.nz, N2 } );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
} else if ( mesh.className() == "PointList" ) {
const IO::PointList& mesh2 = dynamic_cast<const IO::PointList&>( mesh );
size_t N = mesh2.points.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
const IO::PointList &mesh2 = dynamic_cast<const IO::PointList &>( mesh );
size_t N = mesh2.points.size();
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else if ( mesh.className()=="TriMesh" || mesh.className() == "TriList" ) {
std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh*>(&mesh), []( void* ) {} );
} else if ( mesh.className() == "TriMesh" || mesh.className() == "TriList" ) {
std::shared_ptr<Mesh> mesh_ptr( const_cast<Mesh *>( &mesh ), []( void * ) {} );
std::shared_ptr<TriMesh> mesh2 = getTriMesh( mesh_ptr );
if ( var.type == VariableType::NodeVariable ) {
size_t N = mesh2->vertices->points.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
size_t N = mesh2->vertices->points.size();
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else if ( var.type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( var.type == VariableType::VolumeVariable ) {
size_t N = mesh2->A.size();
size_t N_var = var.data.length()/N;
ASSERT( N*N_var == var.data.length() );
size_t N = mesh2->A.size();
size_t N_var = var.data.length() / N;
ASSERT( N * N_var == var.data.length() );
var.data.reshape( { N, N_var } );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
} else {
ERROR("Unknown mesh type");
ERROR( "Unknown mesh type" );
}
}

59
IO/Reader.h
View File

@ -29,20 +29,59 @@ namespace IO {
//! Get the path to a file
std::string getPath( const std::string& filename );
std::string getPath( const std::string &filename );
//! List the timesteps in the given directors (dumps.LBPM)
std::vector<std::string> readTimesteps( const std::string& filename );
/*!
* @brief Get the maximum number of domains written
* @details This function reads the summary files to determine the maximum
* number of domains in the output.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @param[in] comm Optional comm to use to reduce IO load by
* reading on rank 0 and then communicating the result
*/
int maxDomains( const std::string &path, const std::string &format = "auto",
const Utilities::MPI &comm = MPI_COMM_SELF );
/*!
* @brief Read the timestep list
* @details This function reads the timestep list from the summary file.
* @param[in] path The path to use for reading
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility)
* new - New format, 1 file/process
* silo - Silo
* auto - Auto-determin the format
* @return append Append any existing data (default is false)
*/
std::vector<std::string> readTimesteps(
const std::string &path, const std::string &format = "auto" );
/*!
* @brief Read the data for the timestep
* @details This function reads the mesh and variable data provided for the given timestep.
* @param[in] path The path to use for reading
* @param[in] timestep The timestep iteration
* @param[in] domain The desired domain to read
*/
std::vector<IO::MeshDataStruct> readData(
const std::string &path, const std::string &timestep, int domain );
//! Read the list of mesh databases for the given timestep
std::vector<IO::MeshDatabase> getMeshList( const std::string& path, const std::string& timestep );
std::vector<IO::MeshDatabase> getMeshList( const std::string &path, const std::string &timestep );
//! Read the given mesh domain
std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain );
std::shared_ptr<IO::Mesh> getMesh( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain );
/*!
@ -55,8 +94,8 @@ std::shared_ptr<IO::Mesh> getMesh( const std::string& path, const std::string& t
* @param[in] variable The variable name to read
* @return Returns the variable data as a linear array
*/
std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::string& timestep,
const MeshDatabase& meshDatabase, int domain, const std::string& variable );
std::shared_ptr<IO::Variable> getVariable( const std::string &path, const std::string &timestep,
const MeshDatabase &meshDatabase, int domain, const std::string &variable );
/*!
@ -65,9 +104,9 @@ std::shared_ptr<IO::Variable> getVariable( const std::string& path, const std::s
* @param[in] mesh The underlying mesh
* @param[in/out] variable The variable name to read
*/
void reformatVariable( const IO::Mesh& mesh, IO::Variable& var );
void reformatVariable( const IO::Mesh &mesh, IO::Variable &var );
} // IO namespace
} // namespace IO
#endif

526
IO/Writer.cpp
View File

@ -14,30 +14,71 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/Writer.h"
#include "IO/MeshDatabase.h"
#include "IO/IOHelpers.h"
#include "IO/MeshDatabase.h"
#include "IO/silo.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include <sys/stat.h>
#include <algorithm>
#include <vector>
#include <set>
#include <memory>
#include <set>
#include <sys/stat.h>
#include <vector>
enum class Format { OLD, NEW, SILO, UNKNOWN };
enum class Format { OLD, NEW, SILO, UNKNOWN };
/****************************************************
* Initialize the writer *
****************************************************/
* Recursively create the subdirectory *
****************************************************/
static void recursiveMkdir( const std::string &path, mode_t mode )
{
// Iterate through the root directories until we create the desired path
for ( size_t pos = 0; pos < path.size(); ) {
// slide backwards in string until next slash found
pos++;
for ( ; pos < path.size(); pos++ ) {
if ( path[pos] == '/' || path[pos] == 92 )
break;
}
// Create the temporary path
auto path2 = path.substr( 0, pos );
// Check if the temporary path exists
struct stat status;
int result = stat( path2.data(), &status );
if ( result == 0 ) {
// if there is a part of the path that already exists make sure it is really a directory
if ( !S_ISDIR( status.st_mode ) ) {
ERROR(
"Error in recursiveMkdir...\n"
" Cannot create directories in path = " +
path +
"\n because some intermediate item in path exists and is NOT a directory" );
}
continue;
}
// Create the directory and test the result
result = mkdir( path2.data(), mode );
if ( result != 0 ) {
// Maybe another rank created the directory, check
int result = stat( path2.data(), &status );
if ( result != 0 && !S_ISDIR( status.st_mode ) )
ERROR( "Error in Utilities::recursiveMkdir...\n"
" Cannot create directory = " +
path2 );
}
}
}
/****************************************************
* Initialize the writer *
****************************************************/
static std::string global_IO_path;
static Format global_IO_format = Format::UNKNOWN;
void IO::initialize( const std::string& path, const std::string& format, bool append )
void IO::initialize( const std::string &path, const std::string &format, bool append )
{
if ( path.empty() )
global_IO_path = ".";
@ -50,161 +91,168 @@ void IO::initialize( const std::string& path, const std::string& format, bool ap
else if ( format == "silo" )
global_IO_format = Format::SILO;
else
ERROR("Unknown format");
int rank = comm_rank(MPI_COMM_WORLD);
if ( !append && rank==0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
ERROR( "Unknown format" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( !append && rank == 0 ) {
recursiveMkdir( path, S_IRWXU | S_IRGRP );
std::string filename;
if ( global_IO_format==Format::OLD || global_IO_format==Format::NEW )
if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW )
filename = global_IO_path + "/summary.LBM";
else if ( global_IO_format==Format::SILO )
else if ( global_IO_format == Format::SILO )
filename = global_IO_path + "/LBM.visit";
else
ERROR("Unknown format");
auto fid = fopen(filename.c_str(),"wb");
fclose(fid);
ERROR( "Unknown format" );
auto fid = fopen( filename.c_str(), "wb" );
fclose( fid );
}
}
// Write the mesh data in the original format
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat( const std::vector<IO::MeshDataStruct>& meshData, const std::string& path )
static std::vector<IO::MeshDatabase> writeMeshesOrigFormat(
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, int rank )
{
int rank = MPI_WORLD_RANK();
std::vector<IO::MeshDatabase> meshes_written;
for (size_t i=0; i<meshData.size(); i++) {
for ( size_t i = 0; i < meshData.size(); i++ ) {
char domainname[100], filename[100], fullpath[200];
sprintf(domainname,"%05i",rank);
sprintf(filename,"%s.%05i",meshData[i].meshName.c_str(),rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
FILE *fid = fopen(fullpath,"wb");
INSIST(fid!=NULL,std::string("Error opening file: ")+fullpath);
sprintf( domainname, "%05i", rank );
sprintf( filename, "%s.%05i", meshData[i].meshName.c_str(), rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
FILE *fid = fopen( fullpath, "wb" );
INSIST( fid != NULL, std::string( "Error opening file: " ) + fullpath );
std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
IO::MeshDatabase mesh_entry;
mesh_entry.name = meshData[i].meshName;
mesh_entry.type = meshType(*mesh);
mesh_entry.name = meshData[i].meshName;
mesh_entry.type = meshType( *mesh );
mesh_entry.meshClass = meshData[i].mesh->className();
mesh_entry.format = 1;
mesh_entry.format = IO::FileFormat::OLD;
IO::DatabaseEntry domain;
domain.name = domainname;
domain.file = filename;
domain.name = domainname;
domain.file = filename;
domain.offset = 0;
mesh_entry.domains.push_back(domain);
mesh_entry.domains.push_back( domain );
if ( !meshData[i].vars.empty() ) {
printf("Warning: variables are not supported with this format\n");
//for (size_t j=0; j<meshData[i].vars.size(); j++)
printf( "Warning: variables are not supported with this format (original)\n" );
// for (size_t j=0; j<meshData[i].vars.size(); j++)
// mesh_entry.variables.push_back( meshData[i].vars[j]->name );
}
const std::string meshClass = mesh->className();
if ( meshClass=="PointList" ) {
if ( meshClass == "PointList" ) {
// List of points
std::shared_ptr<IO::PointList> pointlist = std::dynamic_pointer_cast<IO::PointList>(mesh);
const std::vector<Point>& P = pointlist->points;
for (size_t i=0; i<P.size(); i++) {
std::shared_ptr<IO::PointList> pointlist =
std::dynamic_pointer_cast<IO::PointList>( mesh );
const std::vector<Point> &P = pointlist->points;
for ( size_t i = 0; i < P.size(); i++ ) {
double x[3];
x[0] = P[i].x; x[1] = P[i].y; x[2] = P[i].z;
fwrite(x,sizeof(double),3,fid);
x[0] = P[i].x;
x[1] = P[i].y;
x[2] = P[i].z;
fwrite( x, sizeof( double ), 3, fid );
}
} else if ( meshClass=="TriList" || meshClass=="TriMesh" ) {
} else if ( meshClass == "TriList" || meshClass == "TriMesh" ) {
// Triangle mesh
std::shared_ptr<IO::TriList> trilist = IO::getTriList(mesh);
const std::vector<Point>& A = trilist->A;
const std::vector<Point>& B = trilist->B;
const std::vector<Point>& C = trilist->C;
for (size_t i=0; i<A.size(); i++) {
std::shared_ptr<IO::TriList> trilist = IO::getTriList( mesh );
const std::vector<Point> &A = trilist->A;
const std::vector<Point> &B = trilist->B;
const std::vector<Point> &C = trilist->C;
for ( size_t i = 0; i < A.size(); i++ ) {
double tri[9];
tri[0] = A[i].x; tri[1] = A[i].y; tri[2] = A[i].z;
tri[3] = B[i].x; tri[4] = B[i].y; tri[5] = B[i].z;
tri[6] = C[i].x; tri[7] = C[i].y; tri[8] = C[i].z;
fwrite(tri,sizeof(double),9,fid);
tri[0] = A[i].x;
tri[1] = A[i].y;
tri[2] = A[i].z;
tri[3] = B[i].x;
tri[4] = B[i].y;
tri[5] = B[i].z;
tri[6] = C[i].x;
tri[7] = C[i].y;
tri[8] = C[i].z;
fwrite( tri, sizeof( double ), 9, fid );
}
} else if ( meshClass=="DomainMesh" ) {
} else if ( meshClass == "DomainMesh" ) {
// This format was never supported with the old format
} else {
ERROR("Unknown mesh");
ERROR( "Unknown mesh" );
}
fclose(fid);
fclose( fid );
std::sort( mesh_entry.variables.begin(), mesh_entry.variables.end() );
mesh_entry.variables.erase( std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ), mesh_entry.variables.end() );
meshes_written.push_back(mesh_entry);
mesh_entry.variables.erase(
std::unique( mesh_entry.variables.begin(), mesh_entry.variables.end() ),
mesh_entry.variables.end() );
meshes_written.push_back( mesh_entry );
}
return meshes_written;
}
// Create the database entry for the mesh data
static IO::MeshDatabase getDatabase( const std::string& filename, const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase getDatabase(
const std::string &filename, const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
int rank = MPI_WORLD_RANK();
char domainname[100];
sprintf(domainname,"%s_%05i",mesh.meshName.c_str(),rank);
sprintf( domainname, "%s_%05i", mesh.meshName.c_str(), rank );
// Create the MeshDatabase
IO::MeshDatabase database;
database.name = mesh.meshName;
database.type = meshType(*(mesh.mesh));
database.name = mesh.meshName;
database.type = meshType( *( mesh.mesh ) );
database.meshClass = mesh.mesh->className();
database.format = format;
database.format = format;
// Write the mesh
IO::DatabaseEntry domain;
domain.name = domainname;
domain.file = filename;
domain.name = domainname;
domain.file = filename;
domain.offset = -1;
database.domains.push_back(domain);
database.domains.push_back( domain );
// Write the variables
for (size_t i=0; i<mesh.vars.size(); i++) {
for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
// Add basic variable info
IO::VariableDatabase info;
info.name = mesh.vars[i]->name;
info.type = mesh.vars[i]->type;
info.dim = mesh.vars[i]->dim;
database.variables.push_back(info);
info.dim = mesh.vars[i]->dim;
database.variables.push_back( info );
// Add domain variable info
IO::DatabaseEntry variable;
variable.name = mesh.vars[i]->name;
variable.file = filename;
variable.name = mesh.vars[i]->name;
variable.file = filename;
variable.offset = -1;
std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
database.variable_data.insert(
std::pair<std::pair<std::string,std::string>,IO::DatabaseEntry>(key,variable) );
std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
database.variable_data.insert(
std::pair<std::pair<std::string, std::string>, IO::DatabaseEntry>( key, variable ) );
}
return database;
}
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase write_domain( FILE *fid, const std::string &filename,
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
const int level = 0;
int rank = MPI_WORLD_RANK();
// Create the MeshDatabase
IO::MeshDatabase database = getDatabase( filename, mesh, format );
IO::MeshDatabase database = getDatabase( filename, mesh, format, rank );
// Write the mesh
IO::DatabaseEntry& domain = database.domains[0];
domain.offset = ftell(fid);
std::pair<size_t,void*> data = mesh.mesh->pack(level);
fprintf(fid,"Mesh: %s-%05i: %lu\n",mesh.meshName.c_str(),rank,data.first);
fwrite(data.second,1,data.first,fid);
fprintf(fid,"\n");
delete [] (char*) data.second;
IO::DatabaseEntry &domain = database.domains[0];
domain.offset = ftell( fid );
std::pair<size_t, void *> data = mesh.mesh->pack( level );
fprintf( fid, "Mesh: %s-%05i: %lu\n", mesh.meshName.c_str(), rank, data.first );
fwrite( data.second, 1, data.first, fid );
fprintf( fid, "\n" );
delete[]( char * ) data.second;
// Write the variables
for (size_t i=0; i<mesh.vars.size(); i++) {
std::pair<std::string,std::string> key(domain.name,mesh.vars[i]->name);
IO::DatabaseEntry& variable = database.variable_data[key];
variable.offset = ftell(fid);
int dim = mesh.vars[i]->dim;
int type = static_cast<int>(mesh.vars[i]->type);
size_t N = mesh.vars[i]->data.length();
if ( type == static_cast<int>(IO::VariableType::NullVariable) ) {
ERROR("Variable type not set");
}
size_t N_mesh = mesh.mesh->numberPointsVar(mesh.vars[i]->type);
ASSERT(N==dim*N_mesh);
fprintf(fid,"Var: %s-%05i-%s: %i, %i, %lu, %lu, double\n",
database.name.c_str(), rank, variable.name.c_str(),
dim, type, N_mesh, N*sizeof(double) );
fwrite(mesh.vars[i]->data.data(),sizeof(double),N,fid);
fprintf(fid,"\n");
for ( size_t i = 0; i < mesh.vars.size(); i++ ) {
ASSERT( mesh.vars[i]->type != IO::VariableType::NullVariable );
std::pair<std::string, std::string> key( domain.name, mesh.vars[i]->name );
auto &variable = database.variable_data[key];
variable.offset = ftell( fid );
int dim = mesh.vars[i]->dim;
auto type = getString( mesh.vars[i]->type );
size_t N = mesh.vars[i]->data.length();
size_t N_mesh = mesh.mesh->numberPointsVar( mesh.vars[i]->type );
ASSERT( N == dim * N_mesh );
fprintf( fid, "Var: %s-%05i-%s: %i, %s, %lu, %lu, double\n", database.name.c_str(), rank,
variable.name.c_str(), dim, type.data(), N_mesh, N * sizeof( double ) );
fwrite( mesh.vars[i]->data.data(), sizeof( double ), N, fid );
fprintf( fid, "\n" );
}
return database;
}
@ -213,72 +261,74 @@ static IO::MeshDatabase write_domain( FILE *fid, const std::string& filename,
#ifdef USE_SILO
// Write a PointList mesh (and variables) to a file
template<class TYPE>
static void writeSiloPointMesh( DBfile *fid, const IO::PointList& mesh, const std::string& meshname )
static void writeSiloPointMesh(
DBfile *fid, const IO::PointList &mesh, const std::string &meshname )
{
const auto& points = mesh.getPoints();
std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.getPoints();
std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
IO::silo::writePointMesh<TYPE>( fid, meshname, 3, points.size(), coords );
}
static void writeSiloPointList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloPointList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::PointList& mesh = dynamic_cast<IO::PointList&>( *meshData.mesh );
const IO::PointList &mesh = dynamic_cast<IO::PointList &>( *meshData.mesh );
const std::string meshname = database.domains[0].name;
if ( meshData.precision == IO::DataType::Double ) {
writeSiloPointMesh<double>( fid, mesh, meshname );
} else if ( meshData.precision == IO::DataType::Float ) {
writeSiloPointMesh<float>( fid, mesh, meshname );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
const auto& points = mesh.getPoints();
std::vector<double> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.getPoints();
std::vector<double> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const double *coords[] = { x.data(), y.data(), z.data() };
silo::writePointMesh( fid, meshname, 3, points.size(), coords );
for (size_t i=0; i<meshData.vars.size(); i++) {
const IO::Variable& var = *meshData.vars[i];
IO::silo::writePointMesh( fid, meshname, 3, points.size(), coords );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writePointMeshVariable( fid, meshname, var.name, var.data );
IO::silo::writePointMeshVariable( fid, meshname, var.name, var.data );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writePointMeshVariable( fid, meshname, var.name, data2 );
IO::silo::writePointMeshVariable( fid, meshname, var.name, data2 );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
// Write a TriMesh mesh (and variables) to a file
template<class TYPE>
static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh& mesh, const std::string& meshname )
static void writeSiloTriMesh( DBfile *fid, const IO::TriMesh &mesh, const std::string &meshname )
{
const auto& points = mesh.vertices->getPoints();
std::vector<TYPE> x(points.size()), y(points.size()), z(points.size());
for (size_t i=0; i<x.size(); i++) {
const auto &points = mesh.vertices->getPoints();
std::vector<TYPE> x( points.size() ), y( points.size() ), z( points.size() );
for ( size_t i = 0; i < x.size(); i++ ) {
x[i] = points[i].x;
y[i] = points[i].y;
z[i] = points[i].z;
}
const TYPE *coords[] = { x.data(), y.data(), z.data() };
const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
const int *tri[] = { mesh.A.data(), mesh.B.data(), mesh.C.data() };
IO::silo::writeTriMesh<TYPE>( fid, meshname, 3, 2, points.size(), coords, mesh.A.size(), tri );
}
static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
const IO::TriMesh& mesh, IO::MeshDatabase database )
static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct &meshData,
const IO::TriMesh &mesh, IO::MeshDatabase database )
{
const std::string meshname = database.domains[0].name;
if ( meshData.precision == IO::DataType::Double ) {
@ -286,238 +336,240 @@ static void writeSiloTriMesh2( DBfile *fid, const IO::MeshDataStruct& meshData,
} else if ( meshData.precision == IO::DataType::Float ) {
writeSiloTriMesh<float>( fid, mesh, meshname );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
for (size_t i=0; i<meshData.vars.size(); i++) {
const IO::Variable& var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const IO::Variable &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, type );
IO::silo::writeTriMeshVariable( fid, 3, meshname, var.name, data2, var.type );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
static void writeSiloTriMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::TriMesh& mesh = dynamic_cast<IO::TriMesh&>( *meshData.mesh );
const IO::TriMesh &mesh = dynamic_cast<IO::TriMesh &>( *meshData.mesh );
writeSiloTriMesh2( fid, meshData, mesh, database );
}
static void writeSiloTriList( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloTriList(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
auto mesh = getTriMesh( meshData.mesh );
writeSiloTriMesh2( fid, meshData, *mesh, database );
}
// Write a DomainMesh mesh (and variables) to a file
static void writeSiloDomainMesh( DBfile *fid, const IO::MeshDataStruct& meshData, IO::MeshDatabase database )
static void writeSiloDomainMesh(
DBfile *fid, const IO::MeshDataStruct &meshData, IO::MeshDatabase database )
{
const IO::DomainMesh& mesh = dynamic_cast<IO::DomainMesh&>( *meshData.mesh );
const IO::DomainMesh &mesh = dynamic_cast<IO::DomainMesh &>( *meshData.mesh );
RankInfoStruct info( mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz );
std::array<double,6> range = { info.ix*mesh.Lx/info.nx, (info.ix+1)*mesh.Lx/info.nx,
info.jy*mesh.Ly/info.ny, (info.jy+1)*mesh.Ly/info.ny,
info.kz*mesh.Lz/info.nz, (info.kz+1)*mesh.Lz/info.nz };
std::array<int,3> N = { mesh.nx, mesh.ny, mesh.nz };
auto meshname = database.domains[0].name;
silo::writeUniformMesh<3>( fid, meshname, range, N );
silo::write<int>( fid, meshname+"_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
for (size_t i=0; i<meshData.vars.size(); i++) {
const auto& var = *meshData.vars[i];
auto type = static_cast<silo::VariableType>( var.type );
std::array<double, 6> range = { info.ix * mesh.Lx / info.nx,
( info.ix + 1 ) * mesh.Lx / info.nx, info.jy * mesh.Ly / info.ny,
( info.jy + 1 ) * mesh.Ly / info.ny, info.kz * mesh.Lz / info.nz,
( info.kz + 1 ) * mesh.Lz / info.nz };
std::array<int, 3> N = { mesh.nx, mesh.ny, mesh.nz };
auto meshname = database.domains[0].name;
IO::silo::writeUniformMesh<3>( fid, meshname, range, N );
IO::silo::write<int>(
fid, meshname + "_rankinfo", { mesh.rank, mesh.nprocx, mesh.nprocy, mesh.nprocz } );
for ( size_t i = 0; i < meshData.vars.size(); i++ ) {
const auto &var = *meshData.vars[i];
if ( var.precision == IO::DataType::Double ) {
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, var.data, var.type );
} else if ( var.precision == IO::DataType::Float ) {
Array<float> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else if ( var.precision == IO::DataType::Int ) {
Array<int> data2( var.data.size() );
data2.copy( var.data );
silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, type );
IO::silo::writeUniformMeshVariable<3>( fid, meshname, N, var.name, data2, var.type );
} else {
ERROR("Unsupported format");
ERROR( "Unsupported format" );
}
}
}
// Write a mesh (and variables) to a file
static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string& filename,
const IO::MeshDataStruct& mesh, int format )
static IO::MeshDatabase write_domain_silo( DBfile *fid, const std::string &filename,
const IO::MeshDataStruct &mesh, IO::FileFormat format, int rank )
{
// Create the MeshDatabase
auto database = getDatabase( filename, mesh, format );
if ( database.meshClass=="PointList" ) {
auto database = getDatabase( filename, mesh, format, rank );
if ( database.meshClass == "PointList" ) {
writeSiloPointList( fid, mesh, database );
} else if ( database.meshClass=="TriMesh" ) {
} else if ( database.meshClass == "TriMesh" ) {
writeSiloTriMesh( fid, mesh, database );
} else if ( database.meshClass=="TriList" ) {
} else if ( database.meshClass == "TriList" ) {
writeSiloTriList( fid, mesh, database );
} else if ( database.meshClass=="DomainMesh" ) {
} else if ( database.meshClass == "DomainMesh" ) {
writeSiloDomainMesh( fid, mesh, database );
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
return database;
}
// Write the summary file for silo
std::pair<int,int> getSiloMeshType( const std::string& meshClass )
std::pair<int, int> getSiloMeshType( const std::string &meshClass )
{
int meshType = 0;
int varType = 0;
if ( meshClass=="PointList" ) {
int varType = 0;
if ( meshClass == "PointList" ) {
meshType = DB_POINTMESH;
varType = DB_POINTVAR;
} else if ( meshClass=="TriMesh" ) {
} else if ( meshClass == "TriMesh" ) {
meshType = DB_UCDMESH;
varType = DB_UCDVAR;
} else if ( meshClass=="TriList" ) {
} else if ( meshClass == "TriList" ) {
meshType = DB_UCDMESH;
varType = DB_UCDVAR;
} else if ( meshClass=="DomainMesh" ) {
} else if ( meshClass == "DomainMesh" ) {
meshType = DB_QUAD_RECT;
varType = DB_QUADVAR;
} else {
ERROR("Unknown mesh class");
ERROR( "Unknown mesh class" );
}
return std::make_pair( meshType, varType );
}
void writeSiloSummary( const std::vector<IO::MeshDatabase>& meshes_written, const std::string& filename )
void writeSiloSummary(
const std::vector<IO::MeshDatabase> &meshes_written, const std::string &filename )
{
auto fid = silo::open( filename, silo::CREATE );
for ( const auto& data : meshes_written ) {
auto fid = IO::silo::open( filename, IO::silo::CREATE );
for ( const auto &data : meshes_written ) {
auto type = getSiloMeshType( data.meshClass );
std::vector<int> meshTypes( data.domains.size(), type.first );
std::vector<int> varTypes( data.domains.size(), type.second );
std::vector<std::string> meshNames;
for ( const auto& tmp : data.domains )
for ( const auto &tmp : data.domains )
meshNames.push_back( tmp.file + ":" + tmp.name );
silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
for (const auto& variable : data.variables ) {
IO::silo::writeMultiMesh( fid, data.name, meshNames, meshTypes );
for ( const auto &variable : data.variables ) {
std::vector<std::string> varnames;
for ( const auto& tmp : data.domains )
for ( const auto &tmp : data.domains )
varnames.push_back( tmp.file + ":" + variable.name );
silo::writeMultiVar( fid, variable.name, varnames, varTypes );
IO::silo::writeMultiVar( fid, variable.name, varnames, varTypes );
}
}
silo::close( fid );
IO::silo::close( fid );
}
#endif
// Write the mesh data in the new format
static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
static std::vector<IO::MeshDatabase> writeMeshesNewFormat(
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
int rank = MPI_WORLD_RANK();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i",rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
FILE *fid = fopen(fullpath,"wb");
for (size_t i=0; i<meshData.size(); i++) {
sprintf( filename, "%05i", rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
FILE *fid = fopen( fullpath, "wb" );
for ( size_t i = 0; i < meshData.size(); i++ ) {
std::shared_ptr<IO::Mesh> mesh = meshData[i].mesh;
meshes_written.push_back( write_domain(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain( fid, filename, meshData[i], format, rank ) );
}
fclose(fid);
fclose( fid );
return meshes_written;
}
// Write the mesh data to silo
static std::vector<IO::MeshDatabase> writeMeshesSilo(
const std::vector<IO::MeshDataStruct>& meshData, const std::string& path, int format )
static std::vector<IO::MeshDatabase> writeMeshesSilo(
const std::vector<IO::MeshDataStruct> &meshData, const std::string &path, IO::FileFormat format,
int rank )
{
#ifdef USE_SILO
int rank = MPI_WORLD_RANK();
std::vector<IO::MeshDatabase> meshes_written;
char filename[100], fullpath[200];
sprintf(filename,"%05i.silo",rank);
sprintf(fullpath,"%s/%s",path.c_str(),filename);
auto fid = silo::open( fullpath, silo::CREATE );
for (size_t i=0; i<meshData.size(); i++) {
sprintf( filename, "%05i.silo", rank );
sprintf( fullpath, "%s/%s", path.c_str(), filename );
auto fid = IO::silo::open( fullpath, IO::silo::CREATE );
for ( size_t i = 0; i < meshData.size(); i++ ) {
auto mesh = meshData[i].mesh;
meshes_written.push_back( write_domain_silo(fid,filename,meshData[i],format) );
meshes_written.push_back( write_domain_silo( fid, filename, meshData[i], format, rank ) );
}
silo::close( fid );
IO::silo::close( fid );
return meshes_written;
#else
ERROR("Application built without silo support");
NULL_USE( meshData );
NULL_USE( path );
NULL_USE( format );
NULL_USE( rank );
ERROR( "Application built without silo support" );
return std::vector<IO::MeshDatabase>();
#endif
}
}
/****************************************************
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
* Write the mesh data *
****************************************************/
void IO::writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm )
{
if ( global_IO_path.empty() )
IO::initialize( );
PROFILE_START("writeData");
int rank = comm_rank(comm);
IO::initialize();
PROFILE_START( "writeData" );
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
// Check the meshData before writing
for ( const auto& data : meshData ) {
if ( !data.check() )
ERROR("Error in meshData");
}
for ( const auto &data : meshData )
ASSERT( data.check() );
// Create the output directory
std::string path = global_IO_path + "/" + subdir;
if ( rank == 0 ) {
mkdir(path.c_str(),S_IRWXU|S_IRGRP);
}
MPI_Barrier(comm);
recursiveMkdir( path, S_IRWXU | S_IRGRP );
// Write the mesh files
std::vector<IO::MeshDatabase> meshes_written;
if ( global_IO_format == Format::OLD ) {
// Write the original triangle format
meshes_written = writeMeshesOrigFormat( meshData, path );
meshes_written = writeMeshesOrigFormat( meshData, path, rank );
} else if ( global_IO_format == Format::NEW ) {
// Write the new format (double precision)
meshes_written = writeMeshesNewFormat( meshData, path, 2 );
meshes_written = writeMeshesNewFormat( meshData, path, IO::FileFormat::NEW, rank );
} else if ( global_IO_format == Format::SILO ) {
// Write silo
meshes_written = writeMeshesSilo( meshData, path, 4 );
meshes_written = writeMeshesSilo( meshData, path, IO::FileFormat::SILO, rank );
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
// Gather a complete list of files on rank 0
meshes_written = gatherAll(meshes_written,comm);
meshes_written = gatherAll( meshes_written, comm );
// Write the summary files
if ( rank == 0 ) {
// Write the summary file for the current timestep
char filename[200];
sprintf(filename,"%s/LBM.summary",path.c_str());
write(meshes_written,filename);
// Write summary silo file if needed
#ifdef USE_SILO
sprintf( filename, "%s/LBM.summary", path.c_str() );
write( meshes_written, filename );
// Write summary silo file if needed
#ifdef USE_SILO
if ( global_IO_format == Format::SILO ) {
sprintf(filename,"%s/summary.silo",path.c_str());
writeSiloSummary(meshes_written,filename);
sprintf( filename, "%s/summary.silo", path.c_str() );
writeSiloSummary( meshes_written, filename );
}
#endif
#endif
// Add the timestep to the global summary file
if ( global_IO_format == Format::OLD || global_IO_format == Format::NEW ) {
auto filename = global_IO_path+"/summary.LBM";
FILE *fid = fopen(filename.c_str(),"ab");
fprintf(fid,"%s/\n",subdir.c_str());
fclose(fid);
auto filename = global_IO_path + "/summary.LBM";
FILE *fid = fopen( filename.c_str(), "ab" );
fprintf( fid, "%s/\n", subdir.c_str() );
fclose( fid );
} else if ( global_IO_format == Format::SILO ) {
auto filename = global_IO_path+"/LBM.visit";
FILE *fid = fopen(filename.c_str(),"ab");
fprintf(fid,"%s/summary.silo\n",subdir.c_str());
fclose(fid);
auto filename = global_IO_path + "/LBM.visit";
FILE *fid = fopen( filename.c_str(), "ab" );
fprintf( fid, "%s/summary.silo\n", subdir.c_str() );
fclose( fid );
} else {
ERROR("Unknown format");
ERROR( "Unknown format" );
}
}
PROFILE_STOP("writeData");
PROFILE_STOP( "writeData" );
}

25
IO/Writer.h
View File

@ -29,17 +29,18 @@ namespace IO {
/*!
* @brief Initialize the writer
* @details This function initializes the writer to the given path. All subsequent
* writes will occur in this directory. If this is not called, then it will default
* to the current path.
* @details This function initializes the writer to the given path.
* All subsequent writes will occur in this directory.
* If this is not called, then it will default to the current path.
* @param[in] path The path to use for writes
* @param[in] format The data format to use:
* old - Old mesh format (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process
* silo - Silo
* old - Old mesh format
* (provided for backward compatibility, cannot write variables)
* new - New format, 1 file/process silo - Silo
* @param[in] append Append any existing data (default is false)
*/
void initialize( const std::string& path="", const std::string& format="silo", bool append=false );
void initialize(
const std::string &path = "", const std::string &format = "silo", bool append = false );
/*!
@ -49,7 +50,8 @@ void initialize( const std::string& path="", const std::string& format="silo", b
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm );
void writeData( const std::string &subdir, const std::vector<IO::MeshDataStruct> &meshData,
const Utilities::MPI &comm );
/*!
@ -59,14 +61,15 @@ void writeData( const std::string& subdir, const std::vector<IO::MeshDataStruct>
* @param[in] meshData The data to write
* @param[in] comm The comm to use for writing (usually MPI_COMM_WORLD or a dup thereof)
*/
inline void writeData( int timestep, const std::vector<IO::MeshDataStruct>& meshData, MPI_Comm comm )
inline void writeData(
int timestep, const std::vector<IO::MeshDataStruct> &meshData, const Utilities::MPI &comm )
{
char subdir[100];
sprintf(subdir,"vis%03i",timestep);
sprintf( subdir, "vis%03i", timestep );
writeData( subdir, meshData, comm );
}
} // IO namespace
} // namespace IO
#endif

387
IO/netcdf.cpp
View File

@ -14,8 +14,8 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/netcdf.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
@ -27,14 +27,14 @@
#include <netcdf_par.h>
#define CHECK_NC_ERR( ERR ) \
do { \
if ( ERR != NC_NOERR ) { \
#define CHECK_NC_ERR( ERR ) \
do { \
if ( ERR != NC_NOERR ) { \
std::string msg = "Error calling netcdf routine: "; \
msg += nc_strerror( ERR ); \
ERROR( msg ); \
} \
} while (0)
msg += nc_strerror( ERR ); \
ERROR( msg ); \
} \
} while ( 0 )
namespace netcdf {
@ -65,43 +65,64 @@ static inline VariableType convertType( nc_type type )
else if ( type == NC_DOUBLE )
type2 = DOUBLE;
else
ERROR("Unknown type");
ERROR( "Unknown type" );
return type2;
}
// Get nc_type from the template
template<class T> inline nc_type getType();
template<> inline nc_type getType<char>() { return NC_CHAR; }
template<> inline nc_type getType<short>() { return NC_SHORT; }
template<> inline nc_type getType<int>() { return NC_INT; }
template<> inline nc_type getType<float>() { return NC_FLOAT; }
template<> inline nc_type getType<double>() { return NC_DOUBLE; }
template<class T>
inline nc_type getType();
template<>
inline nc_type getType<char>()
{
return NC_CHAR;
}
template<>
inline nc_type getType<short>()
{
return NC_SHORT;
}
template<>
inline nc_type getType<int>()
{
return NC_INT;
}
template<>
inline nc_type getType<float>()
{
return NC_FLOAT;
}
template<>
inline nc_type getType<double>()
{
return NC_DOUBLE;
}
// Function to reverse an array
template<class TYPE>
inline std::vector<TYPE> reverse( const std::vector<TYPE>& x )
inline std::vector<TYPE> reverse( const std::vector<TYPE> &x )
{
std::vector<TYPE> y(x.size());
for (size_t i=0; i<x.size(); i++)
y[i] = x[x.size()-i-1];
std::vector<TYPE> y( x.size() );
for ( size_t i = 0; i < x.size(); i++ )
y[i] = x[x.size() - i - 1];
return y;
}
// Function to reverse an array
template<class TYPE1, class TYPE2>
inline std::vector<TYPE2> convert( const std::vector<TYPE1>& x )
inline std::vector<TYPE2> convert( const std::vector<TYPE1> &x )
{
std::vector<TYPE2> y(x.size());
for (size_t i=0; i<x.size(); i++)
y[i] = static_cast<TYPE2>(x[i]);
std::vector<TYPE2> y( x.size() );
for ( size_t i = 0; i < x.size(); i++ )
y[i] = static_cast<TYPE2>( x[i] );
return y;
}
/****************************************************
* Convert the VariableType to a string *
****************************************************/
* Convert the VariableType to a string *
****************************************************/
std::string VariableTypeName( VariableType type )
{
if ( type == BYTE )
@ -129,12 +150,12 @@ std::string VariableTypeName( VariableType type )
/****************************************************
* Open/close a file *
****************************************************/
int open( const std::string& filename, FileMode mode, MPI_Comm comm )
* Open/close a file *
****************************************************/
int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm )
{
int fid = 0;
if ( comm == MPI_COMM_NULL ) {
if ( comm.isNull() ) {
if ( mode == READ ) {
int err = nc_open( filename.c_str(), NC_NOWRITE, &fid );
CHECK_NC_ERR( err );
@ -142,23 +163,26 @@ int open( const std::string& filename, FileMode mode, MPI_Comm comm )
int err = nc_open( filename.c_str(), NC_WRITE, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create( filename.c_str(), NC_SHARE|NC_64BIT_OFFSET, &fid );
int err = nc_create( filename.c_str(), NC_SHARE | NC_64BIT_OFFSET, &fid );
CHECK_NC_ERR( err );
} else {
ERROR("Unknown file mode");
ERROR( "Unknown file mode" );
}
} else {
if ( mode == READ ) {
int err = nc_open_par( filename.c_str(), NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
int err = nc_open_par(
filename.c_str(), NC_MPIPOSIX, comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == WRITE ) {
int err = nc_open_par( filename.c_str(), NC_WRITE|NC_MPIPOSIX, comm, MPI_INFO_NULL, &fid );
int err = nc_open_par( filename.c_str(), NC_WRITE | NC_MPIPOSIX, comm.getCommunicator(),
MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else if ( mode == CREATE ) {
int err = nc_create_par( filename.c_str(), NC_NETCDF4|NC_MPIIO, comm, MPI_INFO_NULL, &fid );
int err = nc_create_par( filename.c_str(), NC_NETCDF4 | NC_MPIIO,
comm.getCommunicator(), MPI_INFO_NULL, &fid );
CHECK_NC_ERR( err );
} else {
ERROR("Unknown file mode");
ERROR( "Unknown file mode" );
}
}
return fid;
@ -167,42 +191,42 @@ void close( int fid )
{
int err = nc_close( fid );
if ( err != NC_NOERR )
ERROR("Error closing file");
ERROR( "Error closing file" );
}
/****************************************************
* Query basic properties *
****************************************************/
* Query basic properties *
****************************************************/
static std::vector<size_t> getDimVar( int fid, int varid )
{
int ndim = 0;
int err = nc_inq_varndims( fid, varid, &ndim );
int err = nc_inq_varndims( fid, varid, &ndim );
CHECK_NC_ERR( err );
std::vector<size_t> dims(ndim,0);
int dimid[64] = {-1};
err = nc_inq_vardimid( fid, varid, dimid );
std::vector<size_t> dims( ndim, 0 );
int dimid[64] = { -1 };
err = nc_inq_vardimid( fid, varid, dimid );
CHECK_NC_ERR( err );
for (int i=0; i<ndim; i++) {
for ( int i = 0; i < ndim; i++ ) {
err = nc_inq_dimlen( fid, dimid[i], &dims[i] );
CHECK_NC_ERR( err );
}
return dims;
}
static int getVarID( int fid, const std::string& var )
static int getVarID( int fid, const std::string &var )
{
int id = -1;
int id = -1;
int err = nc_inq_varid( fid, var.c_str(), &id );
CHECK_NC_ERR( err );
return id;
}
std::vector<size_t> getVarDim( int fid, const std::string& var )
std::vector<size_t> getVarDim( int fid, const std::string &var )
{
return getDimVar( fid, getVarID( fid, var ) );
}
std::vector<size_t> getAttDim( int fid, const std::string& att )
std::vector<size_t> getAttDim( int fid, const std::string &att )
{
std::vector<size_t> dim(1,0);
std::vector<size_t> dim( 1, 0 );
int err = nc_inq_attlen( fid, NC_GLOBAL, att.c_str(), dim.data() );
CHECK_NC_ERR( err );
return dim;
@ -212,9 +236,9 @@ std::vector<std::string> getVarNames( int fid )
int nvar;
int err = nc_inq( fid, NULL, &nvar, NULL, NULL );
CHECK_NC_ERR( err );
std::vector<std::string> vars(nvar);
for (int i=0; i<nvar; i++) {
char name[NC_MAX_NAME+1];
std::vector<std::string> vars( nvar );
for ( int i = 0; i < nvar; i++ ) {
char name[NC_MAX_NAME + 1];
err = nc_inq_varname( fid, i, name );
CHECK_NC_ERR( err );
vars[i] = name;
@ -226,262 +250,269 @@ std::vector<std::string> getAttNames( int fid )
int natt;
int err = nc_inq( fid, NULL, NULL, &natt, NULL );
CHECK_NC_ERR( err );
std::vector<std::string> att(natt);
for (int i=0; i<natt; i++) {
char name[NC_MAX_NAME+1];
err = nc_inq_attname( fid, NC_GLOBAL, i, name );
std::vector<std::string> att( natt );
for ( int i = 0; i < natt; i++ ) {
char name[NC_MAX_NAME + 1];
err = nc_inq_attname( fid, NC_GLOBAL, i, name );
CHECK_NC_ERR( err );
att[i] = name;
}
return att;
}
VariableType getVarType( int fid, const std::string& var )
VariableType getVarType( int fid, const std::string &var )
{
int varid = -1;
int err = nc_inq_varid( fid, var.c_str(), &varid );
int err = nc_inq_varid( fid, var.c_str(), &varid );
CHECK_NC_ERR( err );
nc_type type=0;
err = nc_inq_vartype( fid, varid, &type );
nc_type type = 0;
err = nc_inq_vartype( fid, varid, &type );
CHECK_NC_ERR( err );
return convertType(type);
return convertType( type );
}
VariableType getAttType( int fid, const std::string& att )
VariableType getAttType( int fid, const std::string &att )
{
nc_type type=0;
int err = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type );
nc_type type = 0;
int err = nc_inq_atttype( fid, NC_GLOBAL, att.c_str(), &type );
CHECK_NC_ERR( err );
return convertType(type);
return convertType( type );
}
/****************************************************
* Read a variable *
****************************************************/
* Read a variable *
****************************************************/
template<>
Array<unsigned short> getVar<unsigned short>( int fid, const std::string& var )
Array<unsigned short> getVar<unsigned short>( int fid, const std::string &var )
{
PROFILE_START("getVar<unsigned short>");
Array<unsigned short> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_ushort( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<unsigned short>" );
Array<unsigned short> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_ushort( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<unsigned short>");
PROFILE_STOP( "getVar<unsigned short>" );
return x.reverseDim();
}
template<>
Array<short> getVar<short>( int fid, const std::string& var )
Array<short> getVar<short>( int fid, const std::string &var )
{
PROFILE_START("getVar<short>");
Array<short> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_short( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<short>" );
Array<short> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_short( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<short>");
PROFILE_STOP( "getVar<short>" );
return x.reverseDim();
}
template<>
Array<unsigned int> getVar<unsigned int>( int fid, const std::string& var )
Array<unsigned int> getVar<unsigned int>( int fid, const std::string &var )
{
PROFILE_START("getVar<unsigned int>");
Array<unsigned int> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_uint( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<unsigned int>" );
Array<unsigned int> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_uint( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<unsigned int>");
PROFILE_STOP( "getVar<unsigned int>" );
return x.reverseDim();
}
template<>
Array<int> getVar<int>( int fid, const std::string& var )
Array<int> getVar<int>( int fid, const std::string &var )
{
PROFILE_START("getVar<int>");
Array<int> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_int( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<int>" );
Array<int> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_int( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<int>");
PROFILE_STOP( "getVar<int>" );
return x.reverseDim();
}
template<>
Array<float> getVar<float>( int fid, const std::string& var )
Array<float> getVar<float>( int fid, const std::string &var )
{
PROFILE_START("getVar<float>");
Array<float> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_float( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<float>" );
Array<float> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_float( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<float>");
PROFILE_STOP( "getVar<float>" );
return x.reverseDim();
}
template<>
Array<double> getVar<double>( int fid, const std::string& var )
Array<double> getVar<double>( int fid, const std::string &var )
{
PROFILE_START("getVar<double>");
Array<double> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_double( fid, getVarID(fid,var), x.data() );
PROFILE_START( "getVar<double>" );
Array<double> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_double( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<double>");
PROFILE_STOP( "getVar<double>" );
return x.reverseDim();
}
template<>
Array<char> getVar<char>( int fid, const std::string& var )
{
PROFILE_START("getVar<char>");
Array<char> x( reverse(getVarDim(fid,var)) );
int err = nc_get_var_text( fid, getVarID(fid,var), x.data() );
Array<char> getVar<char>( int fid, const std::string &var )
{
PROFILE_START( "getVar<char>" );
Array<char> x( reverse( getVarDim( fid, var ) ) );
int err = nc_get_var_text( fid, getVarID( fid, var ), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<char>");
PROFILE_STOP( "getVar<char>" );
return x.reverseDim();
}
template<>
Array<std::string> getVar<std::string>( int fid, const std::string& var )
Array<std::string> getVar<std::string>( int fid, const std::string &var )
{
PROFILE_START("getVar<std::string>");
Array<char> tmp = getVar<char>( fid, var );
std::vector<size_t> dim = {tmp.size(0), tmp.size(1), tmp.size(2) };
PROFILE_START( "getVar<std::string>" );
Array<char> tmp = getVar<char>( fid, var );
std::vector<size_t> dim = { tmp.size( 0 ), tmp.size( 1 ), tmp.size( 2 ) };
if ( dim.size() == 1 )
dim[0] = 1;
else
dim.erase( dim.begin() );
Array<std::string> text(dim);
for (size_t i=0; i<text.length(); i++)
text(i) = &(tmp(0,i));
PROFILE_STOP("getVar<std::string>");
Array<std::string> text( dim );
for ( size_t i = 0; i < text.length(); i++ )
text( i ) = &( tmp( 0, i ) );
PROFILE_STOP( "getVar<std::string>" );
return text;
}
static inline void get_stride_args( const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride,
size_t *startp, size_t *countp, ptrdiff_t *stridep )
static inline void get_stride_args( const std::vector<int> &start, const std::vector<int> &count,
const std::vector<int> &stride, size_t *startp, size_t *countp, ptrdiff_t *stridep )
{
for (size_t i=0; i<start.size(); i++)
for ( size_t i = 0; i < start.size(); i++ )
startp[i] = start[i];
for (size_t i=0; i<count.size(); i++)
for ( size_t i = 0; i < count.size(); i++ )
countp[i] = count[i];
for (size_t i=0; i<stride.size(); i++)
for ( size_t i = 0; i < stride.size(); i++ )
stridep[i] = stride[i];
}
template<class TYPE>
int nc_get_vars_TYPE( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], TYPE *ptr );
int nc_get_vars_TYPE( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], TYPE *ptr );
template<>
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], short *ptr )
int nc_get_vars_TYPE<short>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], short *ptr )
{
return nc_get_vars_short( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], int *ptr )
int nc_get_vars_TYPE<int>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], int *ptr )
{
return nc_get_vars_int( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], float *ptr )
int nc_get_vars_TYPE<float>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], float *ptr )
{
return nc_get_vars_float( fid, varid, start, count, stride, ptr );
}
template<>
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[],
const size_t count[], const ptrdiff_t stride[], double *ptr )
int nc_get_vars_TYPE<double>( int fid, int varid, const size_t start[], const size_t count[],
const ptrdiff_t stride[], double *ptr )
{
return nc_get_vars_double( fid, varid, start, count, stride, ptr );
}
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride )
Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
const std::vector<int> &count, const std::vector<int> &stride )
{
PROFILE_START("getVar<> (strided)");
PROFILE_START( "getVar<> (strided)" );
std::vector<size_t> var_size = getVarDim( fid, var );
for (int d=0; d<(int)var_size.size(); d++) {
if ( start[d]<0 || start[d]+stride[d]*(count[d]-1)>(int)var_size[d] ) {
int rank = comm_rank(MPI_COMM_WORLD);
for ( int d = 0; d < (int) var_size.size(); d++ ) {
if ( start[d] < 0 || start[d] + stride[d] * ( count[d] - 1 ) > (int) var_size[d] ) {
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
char tmp[1000];
sprintf(tmp,"%i: Range exceeded array dimension:\n"
sprintf( tmp,
"%i: Range exceeded array dimension:\n"
" start[%i]=%i, count[%i]=%i, stride[%i]=%i, var_size[%i]=%i",
rank,d,start[d],d,count[d],d,stride[d],d,(int)var_size[d]);
ERROR(tmp);
rank, d, start[d], d, count[d], d, stride[d], d, (int) var_size[d] );
ERROR( tmp );
}
}
Array<TYPE> x( reverse(convert<int,size_t>(count)) );
Array<TYPE> x( reverse( convert<int, size_t>( count ) ) );
size_t startp[10], countp[10];
ptrdiff_t stridep[10];
get_stride_args( start, count, stride, startp, countp, stridep );
int err = nc_get_vars_TYPE<TYPE>( fid, getVarID(fid,var), startp, countp, stridep, x.data() );
int err =
nc_get_vars_TYPE<TYPE>( fid, getVarID( fid, var ), startp, countp, stridep, x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getVar<> (strided)");
PROFILE_STOP( "getVar<> (strided)" );
return x.reverseDim();
}
template Array<short> getVar<short>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<int> getVar<int>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<float> getVar<float>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<double> getVar<double>( int, const std::string&, const std::vector<int>&, const std::vector<int>&, const std::vector<int>& );
template Array<short> getVar<short>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<int> getVar<int>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<float> getVar<float>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
template Array<double> getVar<double>( int, const std::string &, const std::vector<int> &,
const std::vector<int> &, const std::vector<int> & );
/****************************************************
* Read an attribute *
****************************************************/
* Read an attribute *
****************************************************/
template<>
Array<double> getAtt<double>( int fid, const std::string& att )
Array<double> getAtt<double>( int fid, const std::string &att )
{
PROFILE_START("getAtt<double>");
Array<double> x( getAttDim(fid,att) );
PROFILE_START( "getAtt<double>" );
Array<double> x( getAttDim( fid, att ) );
int err = nc_get_att_double( fid, NC_GLOBAL, att.c_str(), x.data() );
CHECK_NC_ERR( err );
PROFILE_STOP("getAtt<double>");
PROFILE_STOP( "getAtt<double>" );
return x;
}
template<>
Array<std::string> getAtt<std::string>( int fid, const std::string& att )
Array<std::string> getAtt<std::string>( int fid, const std::string &att )
{
PROFILE_START("getAtt<std::string>");
char *tmp = new char[getAttDim(fid,att)[0]];
Array<std::string> x(1);
x(0) = tmp;
delete [] tmp;
PROFILE_STOP("getAtt<std::string>");
PROFILE_START( "getAtt<std::string>" );
char *tmp = new char[getAttDim( fid, att )[0]];
Array<std::string> x( 1 );
x( 0 ) = tmp;
delete[] tmp;
PROFILE_STOP( "getAtt<std::string>" );
return x;
}
/****************************************************
* Write an array to a file *
****************************************************/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims )
* Write an array to a file *
****************************************************/
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims )
{
std::vector<int> dimid(names.size(),0);
for (size_t i=0; i<names.size(); i++) {
int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i]);
std::vector<int> dimid( names.size(), 0 );
for ( size_t i = 0; i < names.size(); i++ ) {
int err = nc_def_dim( fid, names[i].c_str(), dims[i], &dimid[i] );
CHECK_NC_ERR( err );
}
return dimid;
}
template<class TYPE>
void write( int fid, const std::string& var, const std::vector<int>& dimids,
const Array<TYPE>& data, const RankInfoStruct& info )
void write( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<TYPE> &data, const RankInfoStruct &info )
{
// Define the variable
int varid = 0;
int err = nc_def_var( fid, var.c_str(), getType<TYPE>(), data.ndim(), dimids.data(), &varid );
int err = nc_def_var( fid, var.c_str(), getType<TYPE>(), data.ndim(), dimids.data(), &varid );
CHECK_NC_ERR( err );
// exit define mode
// exit define mode
err = nc_enddef( fid );
CHECK_NC_ERR( err );
// set the access method to use MPI/PnetCDF collective I/O
// set the access method to use MPI/PnetCDF collective I/O
err = nc_var_par_access( fid, varid, NC_INDEPENDENT );
CHECK_NC_ERR( err );
// parallel write: each process writes its subarray to the file
auto x = data.reverseDim();
std::vector<size_t> count = { data.size(0), data.size(1), data.size(2) };
std::vector<size_t> start = { info.ix*data.size(0), info.jy*data.size(1), info.kz*data.size(2) };
auto x = data.reverseDim();
std::vector<size_t> count = { data.size( 0 ), data.size( 1 ), data.size( 2 ) };
std::vector<size_t> start = { info.ix * data.size( 0 ), info.jy * data.size( 1 ),
info.kz * data.size( 2 ) };
nc_put_vara( fid, varid, start.data(), count.data(), x.data() );
}
template void write<short>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<short>& data, const RankInfoStruct& info );
template void write<int>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<int>& data, const RankInfoStruct& info );
template void write<float>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<float>& data, const RankInfoStruct& info );
template void write<double>( int fid, const std::string& var, const std::vector<int>& dimids, const Array<double>& data, const RankInfoStruct& info );
template void write<short>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<short> &data, const RankInfoStruct &info );
template void write<int>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<int> &data, const RankInfoStruct &info );
template void write<float>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<float> &data, const RankInfoStruct &info );
template void write<double>( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<double> &data, const RankInfoStruct &info );
}; // netcdf namespace
}; // namespace netcdf
#else
#endif

75
IO/netcdf.h
View File

@ -20,9 +20,8 @@
#include <vector>
#include "common/Array.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/MPI.h"
namespace netcdf {
@ -41,116 +40,118 @@ std::string VariableTypeName( VariableType type );
/*!
* @brief Open netcdf file
* @detailed This function opens a netcdf file
* @details This function opens a netcdf file
* @return This function returns a handle to the file
* @param filename File to open
* @param mode Open the file for reading or writing
* @param comm MPI communicator to use (MPI_COMM_WORLD: don't use parallel netcdf)
*/
int open( const std::string& filename, FileMode mode, MPI_Comm comm=MPI_COMM_NULL );
*/
int open( const std::string &filename, FileMode mode, const Utilities::MPI &comm = MPI_COMM_NULL );
/*!
* @brief Close netcdf file
* @detailed This function closes a netcdf file
* @details This function closes a netcdf file
* @param fid Handle to the open file
*/
*/
void close( int fid );
/*!
* @brief Read the variable names
* @detailed This function reads a list of the variable names in the file
* @details This function reads a list of the variable names in the file
* @param fid Handle to the open file
*/
*/
std::vector<std::string> getVarNames( int fid );
/*!
* @brief Read the attribute names
* @detailed This function reads a list of the attribute names in the file
* @details This function reads a list of the attribute names in the file
* @param fid Handle to the open file
*/
*/
std::vector<std::string> getAttNames( int fid );
/*!
* @brief Return the variable type
* @detailed This function returns the type for a variable
* @details This function returns the type for a variable
* @param fid Handle to the open file
* @param var Variable to read
*/
VariableType getVarType( int fid, const std::string& var );
*/
VariableType getVarType( int fid, const std::string &var );
/*!
* @brief Return the attribute type
* @detailed This function returns the type for an attribute
* @details This function returns the type for an attribute
* @param fid Handle to the open file
* @param att Attribute to read
*/
VariableType getAttType( int fid, const std::string& att );
*/
VariableType getAttType( int fid, const std::string &att );
/*!
* @brief Return the variable dimensions
* @detailed This function returns the die for a variable
* @details This function returns the die for a variable
* @param fid Handle to the open file
* @param var Variable to read
*/
std::vector<size_t> getVarDim( int fid, const std::string& var );
*/
std::vector<size_t> getVarDim( int fid, const std::string &var );
/*!
* @brief Read a variable
* @detailed This function reads a variable with the given name from the file
* @details This function reads a variable with the given name from the file
* @param fid Handle to the open file
* @param var Variable to read
*/
*/
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var );
Array<TYPE> getVar( int fid, const std::string &var );
/*!
* @brief Read a strided variable
* @detailed This function reads a strided variable with the given name from the file
* @details This function reads a strided variable with the given name from the file
* @param fid Handle to the open file
* @param var Variable to read
* @param start Starting corner for the read
* @param count Number of elements to read
* @param stride Stride size for the read
*/
*/
template<class TYPE>
Array<TYPE> getVar( int fid, const std::string& var, const std::vector<int>& start,
const std::vector<int>& count, const std::vector<int>& stride );
Array<TYPE> getVar( int fid, const std::string &var, const std::vector<int> &start,
const std::vector<int> &count, const std::vector<int> &stride );
/*!
* @brief Read an attribute
* @detailed This function reads an attribute with the given name from the file
* @details This function reads an attribute with the given name from the file
* @param fid Handle to the open file
* @param att Attribute to read
*/
*/
template<class TYPE>
Array<TYPE> getAtt( int fid, const std::string& att );
Array<TYPE> getAtt( int fid, const std::string &att );
/*!
* @brief Write the dimensions
* @detailed This function writes the grid dimensions to netcdf.
* @details This function writes the grid dimensions to netcdf.
* @param fid Handle to the open file
*/
std::vector<int> defDim( int fid, const std::vector<std::string>& names, const std::vector<int>& dims );
*/
std::vector<int> defDim(
int fid, const std::vector<std::string> &names, const std::vector<int> &dims );
/*!
* @brief Write a variable
* @detailed This function writes a variable to netcdf.
* @details This function writes a variable to netcdf.
* @param fid Handle to the open file
*/
*/
template<class TYPE>
void write( int fid, const std::string& var, const std::vector<int>& dimids, const Array<TYPE>& data, const RankInfoStruct& rank_info );
void write( int fid, const std::string &var, const std::vector<int> &dimids,
const Array<TYPE> &data, const RankInfoStruct &rank_info );
}; // netcdf namespace
}; // namespace netcdf
#endif

91
IO/silo.cpp
View File

@ -14,8 +14,8 @@
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "IO/silo.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
@ -25,14 +25,13 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
* Open/close a file *
****************************************************/
DBfile* open( const std::string& filename, FileMode mode )
* Open/close a file *
****************************************************/
DBfile *open( const std::string &filename, FileMode mode )
{
DBfile *fid = nullptr;
if ( mode == CREATE ) {
@ -44,82 +43,78 @@ DBfile* open( const std::string& filename, FileMode mode )
}
return fid;
}
void close( DBfile* fid )
{
DBClose( fid );
}
void close( DBfile *fid ) { DBClose( fid ); }
/****************************************************
* Helper functions *
****************************************************/
VariableDataType varDataType( DBfile *fid, const std::string& name )
* Helper functions *
****************************************************/
DataType varDataType( DBfile *fid, const std::string &name )
{
auto type = DBGetVarType( fid, name.c_str() );
VariableDataType type2 = VariableDataType::UNKNOWN;
auto type = DBGetVarType( fid, name.c_str() );
DataType type2 = DataType::Null;
if ( type == DB_DOUBLE )
type2 = VariableDataType::DOUBLE;
type2 = DataType::Double;
else if ( type == DB_FLOAT )
type2 = VariableDataType::FLOAT;
type2 = DataType::Float;
else if ( type == DB_INT )
type2 = VariableDataType::INT;
type2 = DataType::Int;
return type2;
}
/****************************************************
* Write/read a uniform mesh to silo *
****************************************************/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N )
* Write/read a uniform mesh to silo *
****************************************************/
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N )
{
DBquadmesh* mesh = DBGetQuadmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
range.resize(2*ndim);
N.resize(ndim);
for (int d=0; d<ndim; d++) {
N[d] = mesh->dims[d]-1;
range[2*d+0] = mesh->min_extents[d];
range[2*d+1] = mesh->max_extents[d];
DBquadmesh *mesh = DBGetQuadmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
range.resize( 2 * ndim );
N.resize( ndim );
for ( int d = 0; d < ndim; d++ ) {
N[d] = mesh->dims[d] - 1;
range[2 * d + 0] = mesh->min_extents[d];
range[2 * d + 1] = mesh->max_extents[d];
}
DBFreeQuadmesh( mesh );
}
/****************************************************
* Write a multimesh *
****************************************************/
void writeMultiMesh( DBfile* fid, const std::string& meshname,
const std::vector<std::string>& meshNames,
const std::vector<int>& meshTypes )
* Write a multimesh *
****************************************************/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string> &meshNames, const std::vector<int> &meshTypes )
{
std::vector<char*> meshnames(meshNames.size());
std::vector<char *> meshnames( meshNames.size() );
for ( size_t i = 0; i < meshNames.size(); ++i )
meshnames[i] = (char *) meshNames[i].c_str();
std::string tree_name = meshname + "_tree";
DBoptlist *optList = DBMakeOptlist( 1 );
DBAddOption( optList, DBOPT_MRGTREE_NAME, (char *) tree_name.c_str() );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(), (int*) meshTypes.data(), nullptr );
DBPutMultimesh( fid, meshname.c_str(), meshNames.size(), meshnames.data(),
(int *) meshTypes.data(), nullptr );
DBFreeOptlist( optList );
}
/****************************************************
* Write a multivariable *
****************************************************/
void writeMultiVar( DBfile* fid, const std::string& varname,
const std::vector<std::string>& varNames,
const std::vector<int>& varTypes )
* Write a multivariable *
****************************************************/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &varNames, const std::vector<int> &varTypes )
{
std::vector<char*> varnames(varNames.size(),nullptr);
for (size_t j=0; j<varNames.size(); j++)
varnames[j] = const_cast<char*>(varNames[j].c_str());
DBPutMultivar( fid, varname.c_str(), varNames.size(), varnames.data(), (int*) varTypes.data(), nullptr );
std::vector<char *> varnames( varNames.size(), nullptr );
for ( size_t j = 0; j < varNames.size(); j++ )
varnames[j] = const_cast<char *>( varNames[j].c_str() );
DBPutMultivar(
fid, varname.c_str(), varNames.size(), varnames.data(), (int *) varTypes.data(), nullptr );
}
}; // silo namespace
}; // namespace IO::silo
#else

159
IO/silo.h
View File

@ -16,185 +16,182 @@
#ifndef SILO_INTERFACE
#define SILO_INTERFACE
#include <array>
#include <string>
#include <vector>
#include <array>
#include "IO/Mesh.h"
#include "common/Array.h"
#include "common/MPI_Helpers.h"
#include "common/Communication.h"
#include "common/MPI.h"
#ifdef USE_SILO
#include <silo.h>
#include <silo.h>
#else
typedef int DBfile;
typedef int DBfile;
#endif
namespace silo {
namespace IO::silo {
enum FileMode { READ, WRITE, CREATE };
enum class VariableType : int { NodeVariable=1, EdgeVariable=2, SurfaceVariable=2, VolumeVariable=3, NullVariable=0 };
enum class VariableDataType { DOUBLE, FLOAT, INT, UNKNOWN };
/*!
* @brief Open silo file
* @detailed This function opens a silo file
* @details This function opens a silo file
* @param[in] filename File to open
* @param[in] mode Open the file for reading or writing
* @return This function returns a handle to the file
*/
DBfile* open( const std::string& filename, FileMode mode );
*/
DBfile *open( const std::string &filename, FileMode mode );
/*!
* @brief Close silo file
* @detailed This function closes a silo file
* @details This function closes a silo file
* @param[in] fid Handle to the open file
*/
void close( DBfile* fid );
*/
void close( DBfile *fid );
/*!
* @brief Get the variable type
* @detailed This function returns the type of variable data
* @details This function returns the type of variable data
* @param[in] fid Handle to the open file
* @param[in] name Name of variable
*/
VariableDataType varDataType( DBfile *dbfile, const std::string& name );
*/
DataType varDataType( DBfile *dbfile, const std::string &name );
/*!
* @brief Write data to silo
* @detailed This function writes an arbitrary array to silo
* @details This function writes an arbitrary array to silo
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @param[in] data Data to write
*/
*/
template<class TYPE>
void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data );
void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data );
/*!
* @brief Write data to silo
* @detailed This function writes an arbitrary array to silo
* @details This function writes an arbitrary array to silo
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Data read
*/
*/
template<class TYPE>
std::vector<TYPE> read( DBfile* fid, const std::string& varname );
std::vector<TYPE> read( DBfile *fid, const std::string &varname );
/*!
* @brief Write a uniform grid
* @detailed This function writes a uniform grid to silo as a Quadmesh
* @details This function writes a uniform grid to silo as a Quadmesh
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
* @param[in] N Number of cells in each direction
*/
*/
template<int NDIM>
void writeUniformMesh( DBfile* fid, const std::string& meshname,
const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N );
void writeUniformMesh( DBfile *fid, const std::string &meshname,
const std::array<double, 2 * NDIM> &range, const std::array<int, NDIM> &N );
/*!
* @brief Read a uniform grid
* @detailed This function reads a uniform grid from silo
* @details This function reads a uniform grid from silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[out] range Range of mesh { xmin, xmax, ymin, ymax, zmin, zmax }
* @param[out] N Number of cells in each direction
*/
void readUniformMesh( DBfile* fid, const std::string& meshname,
std::vector<double>& range, std::vector<int>& N );
*/
void readUniformMesh(
DBfile *fid, const std::string &meshname, std::vector<double> &range, std::vector<int> &N );
/*!
* @brief Write a uniform grid variable
* @detailed This function writes a uniform grid variable to silo as a Quadmesh
* @details This function writes a uniform grid variable to silo as a Quadmesh
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] N Number of cells in each direction
* @param[in] varname Variable name
* @param[in] data Variable data
* @param[in] type Variable type
*/
template< int NDIM, class TYPE >
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type );
*/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type );
/*!
* @brief Read a uniform mesh grid variable
* @detailed This function read a uniform mesh variable to silo
* @details This function read a uniform mesh variable to silo
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname );
/*!
* @brief Write a pointmesh
* @detailed This function writes a pointmesh to silo
* @details This function writes a pointmesh to silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] ndim Number of dimensions
* @param[in] N Number of points
* @param[in] coords Coordinates of the points
*/
*/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] );
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] );
/*!
* @brief Read a pointmesh
* @detailed This function reads a pointmesh from silo
* @details This function reads a pointmesh from silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @return Returns the coordinates as a N x ndim array
*/
* @return Returns the coordinates as a N x ndim array
*/
template<class TYPE>
Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname );
Array<TYPE> readPointMesh( DBfile *fid, const std::string &meshname );
/*!
* @brief Write a pointmesh grid variable
* @detailed This function writes a pointmesh variable to silo
* @details This function writes a pointmesh variable to silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] varname Variable name
* @param[in] data Variable data
*/
*/
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data );
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data );
/*!
* @brief Read a pointmesh grid variable
* @detailed This function reads a pointmesh variable from silo
* @details This function reads a pointmesh variable from silo
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readPointMeshVariable( DBfile *fid, const std::string &varname );
/*!
* @brief Write a triangle mesh
* @detailed This function writes a triangle (or simplex) based mesh to silo
* @details This function writes a triangle (or simplex) based mesh to silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] ndim Number of dimensions for the coordinates
@ -203,65 +200,65 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname );
* @param[in] coords Coordinates of the points
* @param[in] N_tri Number of triangles
* @param[in] tri Coordinates of the points
*/
*/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshname,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] );
void writeTriMesh( DBfile *fid, const std::string &meshname, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] );
/*!
* @brief Read a triangle mesh
* @detailed This function reads a triangle (or simplex) based mesh to silo
* @details This function reads a triangle (or simplex) based mesh to silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] coords Coordinates of the points
* @param[in] tri Coordinates of the points
*/
*/
template<class TYPE>
void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri );
void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri );
/*!
* @brief Write a triangle mesh grid variable
* @detailed This function writes a triangle mesh variable to silo
* @details This function writes a triangle mesh variable to silo
* @param[in] fid Handle to the open file
* @param[in] ndim Number of dimensions
* @param[in] meshname Mesh name
* @param[in] varname Variable name
* @param[in] data Variable data
* @param[in] type Variable type
*/
*/
template<class TYPE>
void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data, VariableType type );
void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname,
const std::string &varname, const Array<TYPE> &data, VariableType type );
/*!
* @brief Read a triangle mesh grid variable
* @detailed This function read a triangle mesh variable to silo
* @details This function read a triangle mesh variable to silo
* @param[in] fid Handle to the open file
* @param[in] varname Variable name
* @return Variable data
*/
*/
template<class TYPE>
Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname );
Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname );
/*!
* @brief Write a multimesh
* @detailed This function writes a multimesh to silo
* @details This function writes a multimesh to silo
* @param[in] fid Handle to the open file
* @param[in] meshname Mesh name
* @param[in] subMeshNames Names of the sub meshes in the form "filename:meshname"
* @param[in] subMeshTypes Type of each submesh
*/
void writeMultiMesh( DBfile* fid, const std::string& meshname,
const std::vector<std::string>& subMeshNames,
const std::vector<int>& subMeshTypes );
*/
void writeMultiMesh( DBfile *fid, const std::string &meshname,
const std::vector<std::string> &subMeshNames, const std::vector<int> &subMeshTypes );
/*!
* @brief Write a multivariable
* @detailed This function writes a multivariable to silo
* @details This function writes a multivariable to silo
* @return This function returns a handle to the file
* @param[in] fid Handle to the open file
* @param[in] varname Mesh name
@ -269,14 +266,12 @@ void writeMultiMesh( DBfile* fid, const std::string& meshname,
* @param[in] subVarTypes Type of each submesh
* @param[in] ndim Dimension of variable (used to determine suffix)
* @param[in] nvar Number of subvariables (used to determine suffix)
*/
void writeMultiVar( DBfile* fid, const std::string& varname,
const std::vector<std::string>& subVarNames,
const std::vector<int>& subVarTypes );
*/
void writeMultiVar( DBfile *fid, const std::string &varname,
const std::vector<std::string> &subVarNames, const std::vector<int> &subVarTypes );
}; // silo namespace
}; // namespace IO::silo
#endif
#include "IO/silo.hpp"

377
IO/silo.hpp
View File

@ -32,8 +32,8 @@
#define SILO_INTERFACE_HPP
#include "IO/silo.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "ProfilerApp.h"
@ -43,52 +43,77 @@
#include <silo.h>
namespace silo {
namespace IO::silo {
/****************************************************
* Helper functions *
****************************************************/
template<class TYPE> static constexpr int getType();
template<> constexpr int getType<double>() { return DB_DOUBLE; }
template<> constexpr int getType<float>() { return DB_FLOAT; }
template<> constexpr int getType<int>() { return DB_INT; }
* Helper functions *
****************************************************/
template<class TYPE>
inline void copyData( Array<TYPE>& data, int type, const void *src )
static constexpr int getType();
template<>
constexpr int getType<double>()
{
return DB_DOUBLE;
}
template<>
constexpr int getType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getType<int>()
{
return DB_INT;
}
template<class TYPE>
inline void copyData( Array<TYPE> &data, int type, const void *src )
{
if ( type == getType<TYPE>() )
memcpy( data.data(), src, data.length()*sizeof(TYPE) );
memcpy( data.data(), src, data.length() * sizeof( TYPE ) );
else if ( type == DB_DOUBLE )
data.copy( static_cast<const double*>(src) );
data.copy( static_cast<const double *>( src ) );
else if ( type == DB_FLOAT )
data.copy( static_cast<const float*>(src) );
data.copy( static_cast<const float *>( src ) );
else if ( type == DB_INT )
data.copy( static_cast<const int*>(src) );
data.copy( static_cast<const int *>( src ) );
else
ERROR("Unknown type");
ERROR( "Unknown type" );
}
/****************************************************
* Write/read an arbitrary vector *
****************************************************/
template<class TYPE> constexpr int getSiloType();
template<> constexpr int getSiloType<int>() { return DB_INT; }
template<> constexpr int getSiloType<float>() { return DB_FLOAT; }
template<> constexpr int getSiloType<double>() { return DB_DOUBLE; }
* Write/read an arbitrary vector *
****************************************************/
template<class TYPE>
void write( DBfile* fid, const std::string& varname, const std::vector<TYPE>& data )
constexpr int getSiloType();
template<>
constexpr int getSiloType<int>()
{
return DB_INT;
}
template<>
constexpr int getSiloType<float>()
{
return DB_FLOAT;
}
template<>
constexpr int getSiloType<double>()
{
return DB_DOUBLE;
}
template<class TYPE>
void write( DBfile *fid, const std::string &varname, const std::vector<TYPE> &data )
{
int dims = data.size();
int err = DBWrite( fid, varname.c_str(), (void*) data.data(), &dims, 1, getSiloType<TYPE>() );
int err = DBWrite( fid, varname.c_str(), (void *) data.data(), &dims, 1, getSiloType<TYPE>() );
ASSERT( err == 0 );
}
template<class TYPE>
std::vector<TYPE> read( DBfile* fid, const std::string& varname )
std::vector<TYPE> read( DBfile *fid, const std::string &varname )
{
int N = DBGetVarLength( fid, varname.c_str() );
std::vector<TYPE> data(N);
std::vector<TYPE> data( N );
int err = DBReadVar( fid, varname.c_str(), data.data() );
ASSERT( err == 0 );
return data;
@ -96,31 +121,31 @@ std::vector<TYPE> read( DBfile* fid, const std::string& varname )
/****************************************************
* Helper function to get variable suffixes *
****************************************************/
* Helper function to get variable suffixes *
****************************************************/
inline std::vector<std::string> getVarSuffix( int ndim, int nvars )
{
std::vector<std::string> suffix(nvars);
std::vector<std::string> suffix( nvars );
if ( nvars == 1 ) {
suffix[0] = "";
} else if ( nvars == ndim ) {
if ( ndim==2 ) {
if ( ndim == 2 ) {
suffix[0] = "_x";
suffix[1] = "_y";
} else if ( ndim==3 ) {
} else if ( ndim == 3 ) {
suffix[0] = "_x";
suffix[1] = "_y";
suffix[2] = "_z";
} else {
ERROR("Not finished");
ERROR( "Not finished" );
}
} else if ( nvars == ndim*ndim ) {
if ( ndim==2 ) {
} else if ( nvars == ndim * ndim ) {
if ( ndim == 2 ) {
suffix[0] = "_xx";
suffix[1] = "_xy";
suffix[2] = "_yx";
suffix[3] = "_yy";
} else if ( ndim==3 ) {
} else if ( ndim == 3 ) {
suffix[0] = "_xx";
suffix[1] = "_xy";
suffix[2] = "_xz";
@ -131,122 +156,127 @@ inline std::vector<std::string> getVarSuffix( int ndim, int nvars )
suffix[7] = "_zy";
suffix[8] = "_zz";
} else {
ERROR("Not finished");
ERROR( "Not finished" );
}
} else {
for (int i=0; i<nvars; i++)
suffix[i] = "_" + std::to_string(i+1);
for ( int i = 0; i < nvars; i++ )
suffix[i] = "_" + std::to_string( i + 1 );
}
return suffix;
}
/****************************************************
* Write/read a uniform mesh to silo *
****************************************************/
* Write/read a uniform mesh to silo *
****************************************************/
template<int NDIM>
void writeUniformMesh( DBfile* fid, const std::string& meshname,
const std::array<double,2*NDIM>& range, const std::array<int,NDIM>& N )
void writeUniformMesh( DBfile *fid, const std::string &meshname,
const std::array<double, 2 * NDIM> &range, const std::array<int, NDIM> &N )
{
PROFILE_START("writeUniformMesh",2);
PROFILE_START( "writeUniformMesh", 2 );
int dims[NDIM];
for (size_t d=0; d<N.size(); d++)
dims[d] = N[d]+1;
for ( size_t d = 0; d < N.size(); d++ )
dims[d] = N[d] + 1;
float *x = nullptr;
if ( NDIM >= 1 ) {
x = new float[dims[0]];
for (int i=0; i<N[0]; i++)
x[i] = range[0] + i*(range[1]-range[0])/N[0];
for ( int i = 0; i < N[0]; i++ )
x[i] = range[0] + i * ( range[1] - range[0] ) / N[0];
x[N[0]] = range[1];
}
float *y = nullptr;
if ( NDIM >= 2 ) {
y = new float[dims[1]];
for (int i=0; i<N[1]; i++)
y[i] = range[2] + i*(range[3]-range[2])/N[1];
for ( int i = 0; i < N[1]; i++ )
y[i] = range[2] + i * ( range[3] - range[2] ) / N[1];
y[N[1]] = range[3];
}
float *z = nullptr;
if ( NDIM >= 3 ) {
z = new float[dims[2]];
for (int i=0; i<N[2]; i++)
z[i] = range[4] + i*(range[5]-range[4])/N[2];
for ( int i = 0; i < N[2]; i++ )
z[i] = range[4] + i * ( range[5] - range[4] ) / N[2];
z[N[2]] = range[5];
}
float *coords[] = { x, y, z };
int err = DBPutQuadmesh( fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
int err = DBPutQuadmesh(
fid, meshname.c_str(), nullptr, coords, dims, NDIM, DB_FLOAT, DB_COLLINEAR, nullptr );
delete[] x;
delete[] y;
delete[] z;
ASSERT( err == 0 );
PROFILE_STOP("writeUniformMesh",2);
PROFILE_STOP( "writeUniformMesh", 2 );
}
/****************************************************
* Write a vector/tensor quad variable *
****************************************************/
template<int NDIM,class TYPE>
void writeUniformMeshVariable( DBfile* fid, const std::string& meshname, const std::array<int,NDIM>& N,
const std::string& varname, const Array<TYPE>& data, VariableType type )
* Write a vector/tensor quad variable *
****************************************************/
template<int NDIM, class TYPE>
void writeUniformMeshVariable( DBfile *fid, const std::string &meshname,
const std::array<int, NDIM> &N, const std::string &varname, const Array<TYPE> &data,
VariableType type )
{
PROFILE_START("writeUniformMeshVariable",2);
int nvars=1, dims[NDIM]={1};
PROFILE_START( "writeUniformMeshVariable", 2 );
int nvars = 1, dims[NDIM] = { 1 };
const TYPE *vars[NDIM] = { nullptr };
int vartype = 0;
int vartype = 0;
if ( type == VariableType::NodeVariable ) {
ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
for (int d=0; d<NDIM; d++)
ASSERT(N[d]+1==(int)data.size(d));
vartype = DB_NODECENT;
nvars = data.size(NDIM);
size_t N = data.length()/nvars;
for (int d=0; d<NDIM; d++)
dims[d] = data.size(d);
for (int i=0; i<nvars; i++)
vars[i] = &data(i*N);
ASSERT( data.ndim() == NDIM || data.ndim() == NDIM + 1 );
for ( int d = 0; d < NDIM; d++ )
ASSERT( N[d] + 1 == (int) data.size( d ) );
vartype = DB_NODECENT;
nvars = data.size( NDIM );
size_t N = data.length() / nvars;
for ( int d = 0; d < NDIM; d++ )
dims[d] = data.size( d );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( i * N );
} else if ( type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::VolumeVariable ) {
ASSERT( data.ndim()==NDIM || data.ndim()==NDIM+1 );
for (int d=0; d<NDIM; d++)
ASSERT(N[d]==(int)data.size(d));
vartype = DB_ZONECENT;
nvars = data.size(NDIM);
size_t N = data.length()/nvars;
for (int d=0; d<NDIM; d++)
dims[d] = data.size(d);
for (int i=0; i<nvars; i++)
vars[i] = &data(i*N);
ASSERT( data.ndim() == NDIM || data.ndim() == NDIM + 1 );
for ( int d = 0; d < NDIM; d++ )
ASSERT( N[d] == (int) data.size( d ) );
vartype = DB_ZONECENT;
nvars = data.size( NDIM );
size_t N = data.length() / nvars;
for ( int d = 0; d < NDIM; d++ )
dims[d] = data.size( d );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( i * N );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
auto suffix = getVarSuffix( NDIM, nvars );
std::vector<std::string> var_names(nvars);
for (int i=0; i<nvars; i++)
std::vector<std::string> var_names( nvars );
for ( int i = 0; i < nvars; i++ )
var_names[i] = varname + suffix[i];
std::vector<char*> varnames(nvars,nullptr);
for (int i=0; i<nvars; i++)
varnames[i] = const_cast<char*>(var_names[i].c_str());
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
int err = DBPutQuadvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
dims, NDIM, nullptr, 0, getType<TYPE>(), vartype, nullptr );
ASSERT( err == 0 );
PROFILE_STOP("writeUniformMeshVariable",2);
PROFILE_STOP( "writeUniformMeshVariable", 2 );
}
template<class TYPE>
Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
template<class TYPE>
Array<TYPE> readUniformMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetQuadvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeQuadvar( var );
std::vector<size_t> dims( var->ndims+1, var->nvals );
for (int d=0; d<var->ndims; d++)
std::vector<size_t> dims( var->ndims + 1, var->nvals );
for ( int d = 0; d < var->ndims; d++ )
dims[d] = var->dims[d];
data.reshape( dims );
return data;
@ -254,54 +284,55 @@ Array<TYPE> readUniformMeshVariable( DBfile* fid, const std::string& varname )
/****************************************************
* Read/write a point mesh/variable to silo *
****************************************************/
* Read/write a point mesh/variable to silo *
****************************************************/
template<class TYPE>
void writePointMesh( DBfile* fid, const std::string& meshname,
int ndim, int N, const TYPE *coords[] )
void writePointMesh(
DBfile *fid, const std::string &meshname, int ndim, int N, const TYPE *coords[] )
{
int err = DBPutPointmesh( fid, meshname.c_str(), ndim, coords, N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
}
template<class TYPE>
Array<TYPE> readPointMesh( DBfile* fid, const std::string& meshname )
template<class TYPE>
Array<TYPE> readPointMesh( DBfile *fid, const std::string &meshname )
{
auto mesh = DBGetPointmesh( fid, meshname.c_str() );
int N = mesh->nels;
int ndim = mesh->ndims;
Array<TYPE> coords(N,ndim);
int N = mesh->nels;
int ndim = mesh->ndims;
Array<TYPE> coords( N, ndim );
int type = mesh->datatype;
for (int d=0; d<ndim; d++) {
for ( int d = 0; d < ndim; d++ ) {
Array<TYPE> data2( N );
copyData<TYPE>( data2, type, mesh->coords[d] );
memcpy( &coords(0,d), data2.data(), N*sizeof(TYPE) );
memcpy( &coords( 0, d ), data2.data(), N * sizeof( TYPE ) );
}
DBFreePointmesh( mesh );
return coords;
}
template<class TYPE>
void writePointMeshVariable( DBfile* fid, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data )
void writePointMeshVariable(
DBfile *fid, const std::string &meshname, const std::string &varname, const Array<TYPE> &data )
{
int N = data.size(0);
int nvars = data.size(1);
std::vector<const TYPE*> vars(nvars);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
int err = DBPutPointvar( fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
int N = data.size( 0 );
int nvars = data.size( 1 );
std::vector<const TYPE *> vars( nvars );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
int err = DBPutPointvar(
fid, varname.c_str(), meshname.c_str(), nvars, vars.data(), N, getType<TYPE>(), nullptr );
ASSERT( err == 0 );
}
template<class TYPE>
Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
template<class TYPE>
Array<TYPE> readPointMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetPointvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeMeshvar( var );
return data;
@ -309,110 +340,110 @@ Array<TYPE> readPointMeshVariable( DBfile* fid, const std::string& varname )
/****************************************************
* Read/write a triangle mesh *
****************************************************/
* Read/write a triangle mesh *
****************************************************/
template<class TYPE>
void writeTriMesh( DBfile* fid, const std::string& meshName,
int ndim, int ndim_tri, int N, const TYPE *coords[], int N_tri, const int *tri[] )
void writeTriMesh( DBfile *fid, const std::string &meshName, int ndim, int ndim_tri, int N,
const TYPE *coords[], int N_tri, const int *tri[] )
{
auto zoneName = meshName + "_zones";
std::vector<int> nodelist( (ndim_tri+1)*N_tri );
for (int i=0, j=0; i<N_tri; i++) {
for (int d=0; d<ndim_tri+1; d++, j++)
std::vector<int> nodelist( ( ndim_tri + 1 ) * N_tri );
for ( int i = 0, j = 0; i < N_tri; i++ ) {
for ( int d = 0; d < ndim_tri + 1; d++, j++ )
nodelist[j] = tri[d][i];
}
int shapetype = 0;
if ( ndim_tri==1 )
if ( ndim_tri == 1 )
shapetype = DB_ZONETYPE_BEAM;
else if ( ndim_tri==2 )
else if ( ndim_tri == 2 )
shapetype = DB_ZONETYPE_TRIANGLE;
else if ( ndim_tri==3 )
else if ( ndim_tri == 3 )
shapetype = DB_ZONETYPE_PYRAMID;
else
ERROR("Unknown shapetype");
int shapesize = ndim_tri+1;
int shapecnt = N_tri;
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(),
nodelist.size(), 0, 0, 0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N,
nodelist.size(), zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
ERROR( "Unknown shapetype" );
int shapesize = ndim_tri + 1;
int shapecnt = N_tri;
DBPutZonelist2( fid, zoneName.c_str(), N_tri, ndim_tri, nodelist.data(), nodelist.size(), 0, 0,
0, &shapetype, &shapesize, &shapecnt, 1, nullptr );
DBPutUcdmesh( fid, meshName.c_str(), ndim, nullptr, coords, N, nodelist.size(),
zoneName.c_str(), nullptr, getType<TYPE>(), nullptr );
}
template<class TYPE>
void readTriMesh( DBfile* fid, const std::string& meshname, Array<TYPE>& coords, Array<int>& tri )
void readTriMesh( DBfile *fid, const std::string &meshname, Array<TYPE> &coords, Array<int> &tri )
{
auto mesh = DBGetUcdmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
auto mesh = DBGetUcdmesh( fid, meshname.c_str() );
int ndim = mesh->ndims;
int N_nodes = mesh->nnodes;
coords.resize(N_nodes,ndim);
coords.resize( N_nodes, ndim );
int mesh_type = mesh->datatype;
for (int d=0; d<ndim; d++) {
for ( int d = 0; d < ndim; d++ ) {
Array<TYPE> data2( N_nodes );
copyData<TYPE>( data2, mesh_type, mesh->coords[d] );
memcpy( &coords(0,d), data2.data(), N_nodes*sizeof(TYPE) );
memcpy( &coords( 0, d ), data2.data(), N_nodes * sizeof( TYPE ) );
}
auto zones = mesh->zones;
auto zones = mesh->zones;
int N_zones = zones->nzones;
ASSERT( zones->nshapes==1 );
ASSERT( zones->nshapes == 1 );
int shapesize = zones->shapesize[0];
tri.resize(N_zones,shapesize);
for (int i=0; i<N_zones; i++) {
for (int j=0; j<shapesize; j++)
tri(i,j) = zones->nodelist[i*shapesize+j];
tri.resize( N_zones, shapesize );
for ( int i = 0; i < N_zones; i++ ) {
for ( int j = 0; j < shapesize; j++ )
tri( i, j ) = zones->nodelist[i * shapesize + j];
}
DBFreeUcdmesh( mesh );
}
template<class TYPE>
void writeTriMeshVariable( DBfile* fid, int ndim, const std::string& meshname,
const std::string& varname, const Array<TYPE>& data, VariableType type )
void writeTriMeshVariable( DBfile *fid, int ndim, const std::string &meshname,
const std::string &varname, const Array<TYPE> &data, VariableType type )
{
int nvars = 0;
int vartype = 0;
int nvars = 0;
int vartype = 0;
const TYPE *vars[10] = { nullptr };
if ( type == VariableType::NodeVariable ) {
vartype = DB_NODECENT;
nvars = data.size(1);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
nvars = data.size( 1 );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
} else if ( type == VariableType::EdgeVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::SurfaceVariable ) {
ERROR("Not finished");
ERROR( "Not finished" );
} else if ( type == VariableType::VolumeVariable ) {
vartype = DB_ZONECENT;
nvars = data.size(1);
for (int i=0; i<nvars; i++)
vars[i] = &data(0,i);
nvars = data.size( 1 );
for ( int i = 0; i < nvars; i++ )
vars[i] = &data( 0, i );
} else {
ERROR("Invalid variable type");
ERROR( "Invalid variable type" );
}
auto suffix = getVarSuffix( ndim, nvars );
std::vector<std::string> var_names(nvars);
for (int i=0; i<nvars; i++)
std::vector<std::string> var_names( nvars );
for ( int i = 0; i < nvars; i++ )
var_names[i] = varname + suffix[i];
std::vector<char*> varnames(nvars,nullptr);
for (int i=0; i<nvars; i++)
varnames[i] = const_cast<char*>(var_names[i].c_str());
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars,
varnames.data(), vars, data.size(0), nullptr, 0, getType<TYPE>(), vartype, nullptr );
std::vector<char *> varnames( nvars, nullptr );
for ( int i = 0; i < nvars; i++ )
varnames[i] = const_cast<char *>( var_names[i].c_str() );
DBPutUcdvar( fid, varname.c_str(), meshname.c_str(), nvars, varnames.data(), vars,
data.size( 0 ), nullptr, 0, getType<TYPE>(), vartype, nullptr );
}
template<class TYPE>
Array<TYPE> readTriMeshVariable( DBfile* fid, const std::string& varname )
Array<TYPE> readTriMeshVariable( DBfile *fid, const std::string &varname )
{
auto var = DBGetUcdvar( fid, varname.c_str() );
ASSERT( var != nullptr );
Array<TYPE> data( var->nels, var->nvals );
int type = var->datatype;
for (int i=0; i<var->nvals; i++) {
for ( int i = 0; i < var->nvals; i++ ) {
Array<TYPE> data2( var->nels );
copyData<TYPE>( data2, type, var->vals[i] );
memcpy( &data(0,i), data2.data(), var->nels*sizeof(TYPE) );
memcpy( &data( 0, i ), data2.data(), var->nels * sizeof( TYPE ) );
}
DBFreeUcdvar( var );
return data;
}
}; // silo namespace
}; // namespace IO::silo
#endif

1
README.titan
View File

@ -26,7 +26,6 @@ cmake \
-D CMAKE_CXX_COMPILER:PATH=CC \
-D CFLAGS="-DCBUB" \
-D CXXFLAGS="-DCBUB" \
-D MPI_COMPILER:BOOL=TRUE \
-D MPIEXEC=aprun \
-D USE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=TRUE \
-D CMAKE_BUILD_TYPE:STRING=Debug \

3
StackTrace/ErrorHandlers.h
View File

@ -3,11 +3,10 @@
#include "StackTrace/StackTrace.h"
#include "common/MPI.h"
#include <functional>
#include "mpi.h"
namespace StackTrace
{

234
analysis/ElectroChemistry.cpp Normal file
View File

@ -0,0 +1,234 @@
#include "analysis/ElectroChemistry.h"
ElectroChemistryAnalyzer::ElectroChemistryAnalyzer(std::shared_ptr <Domain> dm):
Dm(dm)
{
Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0);
ElectricalPotential.resize(Nx,Ny,Nz); ElectricalPotential.fill(0);
Pressure.resize(Nx,Ny,Nz); Pressure.fill(0);
Rho.resize(Nx,Ny,Nz); Rho.fill(0);
Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field
Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0);
Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0);
SDs.resize(Nx,Ny,Nz); SDs.fill(0);
if (Dm->rank()==0){
bool WriteHeader=false;
TIMELOG = fopen("electrokinetic.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("electrokinetic.csv","a+");
if (WriteHeader)
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"TBD TBD\n");
}
}
}
ElectroChemistryAnalyzer::~ElectroChemistryAnalyzer(){
if (Dm->rank()==0){
fclose(TIMELOG);
}
}
void ElectroChemistryAnalyzer::SetParams(){
}
void ElectroChemistryAnalyzer::Basic(ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep){
int i,j,k;
double Vin=0.0;
double Vout=0.0;
Poisson.getElectricPotential(ElectricalPotential);
/* local sub-domain averages */
double rho_avg_local[Ion.number_ion_species];
double rho_mu_avg_local[Ion.number_ion_species];
double rho_mu_fluctuation_local[Ion.number_ion_species];
double rho_psi_avg_local[Ion.number_ion_species];
double rho_psi_fluctuation_local[Ion.number_ion_species];
/* global averages */
double rho_avg_global[Ion.number_ion_species];
double rho_mu_avg_global[Ion.number_ion_species];
double rho_mu_fluctuation_global[Ion.number_ion_species];
double rho_psi_avg_global[Ion.number_ion_species];
double rho_psi_fluctuation_global[Ion.number_ion_species];
for (int ion=0; ion<Ion.number_ion_species; ion++){
rho_avg_local[ion] = 0.0;
rho_mu_avg_local[ion] = 0.0;
rho_psi_avg_local[ion] = 0.0;
Ion.getIonConcentration(Rho,ion);
/* Compute averages for each ion */
for (k=1; k<Nz; k++){
for (j=1; j<Ny; j++){
for (i=1; i<Nx; i++){
rho_avg_local[ion] += Rho(i,j,k);
rho_mu_avg_local[ion] += Rho(i,j,k)*Rho(i,j,k);
rho_psi_avg_local[ion] += Rho(i,j,k)*ElectricalPotential(i,j,k);
}
}
}
rho_avg_global[ion]=Dm->Comm.sumReduce( rho_avg_local[ion]) / Volume;
rho_mu_avg_global[ion]=Dm->Comm.sumReduce( rho_mu_avg_local[ion]) / Volume;
rho_psi_avg_global[ion]=Dm->Comm.sumReduce( rho_psi_avg_local[ion]) / Volume;
if (rho_avg_global[ion] > 0.0){
rho_mu_avg_global[ion] /= rho_avg_global[ion];
rho_psi_avg_global[ion] /= rho_avg_global[ion];
}
}
for (int ion=0; ion<Ion.number_ion_species; ion++){
rho_mu_fluctuation_local[ion] = 0.0;
rho_psi_fluctuation_local[ion] = 0.0;
/* Compute averages for each ion */
for (k=1; k<Nz; k++){
for (j=1; j<Ny; j++){
for (i=1; i<Nx; i++){
rho_mu_fluctuation_local[ion] += (Rho(i,j,k)*Rho(i,j,k) - rho_mu_avg_global[ion]);
rho_psi_fluctuation_local[ion] += (Rho(i,j,k)*ElectricalPotential(i,j,k) - rho_psi_avg_global[ion]);
}
}
}
rho_mu_fluctuation_global[ion]=Dm->Comm.sumReduce( rho_mu_fluctuation_local[ion]);
rho_psi_fluctuation_global[ion]=Dm->Comm.sumReduce( rho_psi_fluctuation_local[ion]);
}
if (Dm->rank()==0){
fprintf(TIMELOG,"%i ",timestep);
for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
}
fprintf(TIMELOG,"%.8g %.8g\n",Vin,Vout);
fflush(TIMELOG);
}
/* else{
fprintf(TIMELOG,"%i ",timestep);
for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
}
fflush(TIMELOG);
} */
}
void ElectroChemistryAnalyzer::WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep){
auto vis_db = input_db->getDatabase( "Visualization" );
char VisName[40];
std::vector<IO::MeshDataStruct> visData;
fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
IO::initialize("","silo","false");
// Create the MeshDataStruct
visData.resize(1);
visData[0].meshName = "domain";
visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
auto ElectricPotential = std::make_shared<IO::Variable>();
std::vector<shared_ptr<IO::Variable>> IonConcentration;
for (int ion=0; ion<Ion.number_ion_species; ion++){
IonConcentration.push_back(std::make_shared<IO::Variable>());
}
auto VxVar = std::make_shared<IO::Variable>();
auto VyVar = std::make_shared<IO::Variable>();
auto VzVar = std::make_shared<IO::Variable>();
if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
ElectricPotential->name = "ElectricPotential";
ElectricPotential->type = IO::VariableType::VolumeVariable;
ElectricPotential->dim = 1;
ElectricPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(ElectricPotential);
}
if (vis_db->getWithDefault<bool>( "save_concentration", true )){
for (int ion=0; ion<Ion.number_ion_species; ion++){
sprintf(VisName,"IonConcentration_%i",ion+1);
IonConcentration[ion]->name = VisName;
IonConcentration[ion]->type = IO::VariableType::VolumeVariable;
IonConcentration[ion]->dim = 1;
IonConcentration[ion]->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(IonConcentration[ion]);
}
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VxVar);
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VyVar);
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VzVar);
}
if (vis_db->getWithDefault<bool>( "save_electric_potential", true )){
ASSERT(visData[0].vars[0]->name=="ElectricPotential");
Poisson.getElectricPotential(ElectricalPotential);
Array<double>& ElectricPotentialData = visData[0].vars[0]->data;
fillData.copy(ElectricalPotential,ElectricPotentialData);
}
if (vis_db->getWithDefault<bool>( "save_concentration", true )){
for (int ion=0; ion<Ion.number_ion_species; ion++){
sprintf(VisName,"IonConcentration_%i",ion+1);
IonConcentration[ion]->name = VisName;
ASSERT(visData[0].vars[1+ion]->name==VisName);
Array<double>& IonConcentrationData = visData[0].vars[1+ion]->data;
Ion.getIonConcentration(Rho,ion);
fillData.copy(Rho,IonConcentrationData);
}
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
ASSERT(visData[0].vars[1+Ion.number_ion_species+0]->name=="Velocity_x");
ASSERT(visData[0].vars[1+Ion.number_ion_species+1]->name=="Velocity_y");
ASSERT(visData[0].vars[1+Ion.number_ion_species+2]->name=="Velocity_z");
Stokes.getVelocity(Vel_x,Vel_y,Vel_z);
Array<double>& VelxData = visData[0].vars[1+Ion.number_ion_species+0]->data;
Array<double>& VelyData = visData[0].vars[1+Ion.number_ion_species+1]->data;
Array<double>& VelzData = visData[0].vars[1+Ion.number_ion_species+2]->data;
fillData.copy(Vel_x,VelxData);
fillData.copy(Vel_y,VelyData);
fillData.copy(Vel_z,VelzData);
}
if (vis_db->getWithDefault<bool>( "write_silo", true ))
IO::writeData( timestep, visData, Dm->Comm );
/* if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
char CurrentIDFilename[40];
sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
Averages.AggregateLabels(CurrentIDFilename);
}
*/
}

55
analysis/ElectroChemistry.h Normal file
View File

@ -0,0 +1,55 @@
/*
* averaging tools for electrochemistry
*/
#ifndef ElectroChem_INC
#define ElectroChem_INC
#include <vector>
#include "common/Domain.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
#include "models/IonModel.h"
#include "models/PoissonSolver.h"
#include "models/StokesModel.h"
class ElectroChemistryAnalyzer{
public:
std::shared_ptr <Domain> Dm;
double Volume;
// input variables
double rho_n, rho_w;
double nu_n, nu_w;
double gamma_wn, beta;
double Fx, Fy, Fz;
//...........................................................................
int Nx,Ny,Nz;
DoubleArray Rho; // density field
DoubleArray ChemicalPotential; // density field
DoubleArray ElectricalPotential; // density field
DoubleArray Pressure; // pressure field
DoubleArray Vel_x; // velocity field
DoubleArray Vel_y;
DoubleArray Vel_z;
DoubleArray SDs;
ElectroChemistryAnalyzer(std::shared_ptr <Domain> Dm);
~ElectroChemistryAnalyzer();
void SetParams();
void Basic( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, int timestep);
void WriteVis( ScaLBL_IonModel &Ion, ScaLBL_Poisson &Poisson, ScaLBL_StokesModel &Stokes, std::shared_ptr<Database> input_db, int timestep);
private:
FILE *TIMELOG;
};
#endif

181
analysis/FreeEnergy.cpp Normal file
View File

@ -0,0 +1,181 @@
#include "analysis/FreeEnergy.h"
FreeEnergyAnalyzer::FreeEnergyAnalyzer(std::shared_ptr <Domain> dm):
Dm(dm)
{
Nx=dm->Nx; Ny=dm->Ny; Nz=dm->Nz;
Volume=(Nx-2)*(Ny-2)*(Nz-2)*Dm->nprocx()*Dm->nprocy()*Dm->nprocz()*1.0;
ChemicalPotential.resize(Nx,Ny,Nz); ChemicalPotential.fill(0);
Phi.resize(Nx,Ny,Nz); Phi.fill(0);
Pressure.resize(Nx,Ny,Nz); Pressure.fill(0);
Rho.resize(Nx,Ny,Nz); Rho.fill(0);
Vel_x.resize(Nx,Ny,Nz); Vel_x.fill(0); // Gradient of the phase indicator field
Vel_y.resize(Nx,Ny,Nz); Vel_y.fill(0);
Vel_z.resize(Nx,Ny,Nz); Vel_z.fill(0);
SDs.resize(Nx,Ny,Nz); SDs.fill(0);
if (Dm->rank()==0){
bool WriteHeader=false;
TIMELOG = fopen("free.csv","r");
if (TIMELOG != NULL)
fclose(TIMELOG);
else
WriteHeader=true;
TIMELOG = fopen("free.csv","a+");
if (WriteHeader)
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"timestep\n");
}
}
}
FreeEnergyAnalyzer::~FreeEnergyAnalyzer(){
if (Dm->rank()==0){
fclose(TIMELOG);
}
}
void FreeEnergyAnalyzer::SetParams(){
}
void FreeEnergyAnalyzer::Basic(ScaLBL_FreeLeeModel &LeeModel, int timestep){
int i,j,k;
if (Dm->rank()==0){
fprintf(TIMELOG,"%i ",timestep);
/*for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_global[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_global[ion]);
}
*/
fprintf(TIMELOG,"\n");
fflush(TIMELOG);
}
/* else{
fprintf(TIMELOG,"%i ",timestep);
for (int ion=0; ion<Ion.number_ion_species; ion++){
fprintf(TIMELOG,"%.8g ",rho_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_avg_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_mu_fluctuation_local[ion]);
fprintf(TIMELOG,"%.8g ",rho_psi_fluctuation_local[ion]);
}
fflush(TIMELOG);
} */
}
void FreeEnergyAnalyzer::WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep){
auto vis_db = input_db->getDatabase( "Visualization" );
char VisName[40];
std::vector<IO::MeshDataStruct> visData;
fillHalo<double> fillData(Dm->Comm,Dm->rank_info,{Dm->Nx-2,Dm->Ny-2,Dm->Nz-2},{1,1,1},0,1);
IO::initialize("","silo","false");
// Create the MeshDataStruct
visData.resize(1);
visData[0].meshName = "domain";
visData[0].mesh = std::make_shared<IO::DomainMesh>( Dm->rank_info,Dm->Nx-2,Dm->Ny-2,Dm->Nz-2,Dm->Lx,Dm->Ly,Dm->Lz );
auto VisPhase = std::make_shared<IO::Variable>();
auto VisPressure = std::make_shared<IO::Variable>();
auto VisChemicalPotential = std::make_shared<IO::Variable>();
auto VxVar = std::make_shared<IO::Variable>();
auto VyVar = std::make_shared<IO::Variable>();
auto VzVar = std::make_shared<IO::Variable>();
if (vis_db->getWithDefault<bool>( "save_phase_field", true )){
VisPhase->name = "Phase";
VisPhase->type = IO::VariableType::VolumeVariable;
VisPhase->dim = 1;
VisPhase->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPhase);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
VisPressure->name = "Pressure";
VisPressure->type = IO::VariableType::VolumeVariable;
VisPressure->dim = 1;
VisPressure->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisPressure);
VisChemicalPotential->name = "ChemicalPotential";
VisChemicalPotential->type = IO::VariableType::VolumeVariable;
VisChemicalPotential->dim = 1;
VisChemicalPotential->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VisChemicalPotential);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
VxVar->name = "Velocity_x";
VxVar->type = IO::VariableType::VolumeVariable;
VxVar->dim = 1;
VxVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VxVar);
VyVar->name = "Velocity_y";
VyVar->type = IO::VariableType::VolumeVariable;
VyVar->dim = 1;
VyVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VyVar);
VzVar->name = "Velocity_z";
VzVar->type = IO::VariableType::VolumeVariable;
VzVar->dim = 1;
VzVar->data.resize(Dm->Nx-2,Dm->Ny-2,Dm->Nz-2);
visData[0].vars.push_back(VzVar);
}
if (vis_db->getWithDefault<bool>( "save_phase", true )){
ASSERT(visData[0].vars[0]->name=="Phase");
LeeModel.getPhase(Phi);
Array<double>& PhaseData = visData[0].vars[0]->data;
fillData.copy(Phi,PhaseData);
}
if (vis_db->getWithDefault<bool>( "save_potential", true )){
ASSERT(visData[0].vars[1]->name=="Pressure");
LeeModel.getPotential(Pressure, ChemicalPotential);
Array<double>& PressureData = visData[0].vars[1]->data;
fillData.copy(Pressure,PressureData);
ASSERT(visData[0].vars[2]->name=="ChemicalPotential");
Array<double>& ChemicalPotentialData = visData[0].vars[2]->data;
fillData.copy(ChemicalPotential,ChemicalPotentialData);
}
if (vis_db->getWithDefault<bool>( "save_velocity", false )){
ASSERT(visData[0].vars[3]->name=="Velocity_x");
ASSERT(visData[0].vars[4]->name=="Velocity_y");
ASSERT(visData[0].vars[5]->name=="Velocity_z");
LeeModel.getVelocity(Vel_x,Vel_y,Vel_z);
Array<double>& VelxData = visData[0].vars[3]->data;
Array<double>& VelyData = visData[0].vars[4]->data;
Array<double>& VelzData = visData[0].vars[5]->data;
fillData.copy(Vel_x,VelxData);
fillData.copy(Vel_y,VelyData);
fillData.copy(Vel_z,VelzData);
}
if (vis_db->getWithDefault<bool>( "write_silo", true ))
IO::writeData( timestep, visData, Dm->Comm );
/* if (vis_db->getWithDefault<bool>( "save_8bit_raw", true )){
char CurrentIDFilename[40];
sprintf(CurrentIDFilename,"id_t%d.raw",timestep);
Averages.AggregateLabels(CurrentIDFilename);
}
*/
}

54
analysis/FreeEnergy.h Normal file
View File

@ -0,0 +1,54 @@
/*
* averaging tools for electrochemistry
*/
#ifndef FreeEnergyAnalyzer_INC
#define FreeEnergyAnalyzer_INC
#include <vector>
#include "common/Domain.h"
#include "common/Utilities.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "analysis/SubPhase.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
#include "models/FreeLeeModel.h"
class FreeEnergyAnalyzer{
public:
std::shared_ptr <Domain> Dm;
double Volume;
// input variables
double rho_n, rho_w;
double nu_n, nu_w;
double gamma_wn, beta;
double Fx, Fy, Fz;
//...........................................................................
int Nx,Ny,Nz;
DoubleArray Rho;
DoubleArray Phi;
DoubleArray ChemicalPotential;
DoubleArray Pressure;
DoubleArray Vel_x;
DoubleArray Vel_y;
DoubleArray Vel_z;
DoubleArray SDs;
FreeEnergyAnalyzer(std::shared_ptr <Domain> Dm);
~FreeEnergyAnalyzer();
void SetParams();
void Basic( ScaLBL_FreeLeeModel &LeeModel, int timestep);
void WriteVis( ScaLBL_FreeLeeModel &LeeModel, std::shared_ptr<Database> input_db, int timestep);
private:
FILE *TIMELOG;
};
#endif

26
analysis/GreyPhase.cpp
View File

@ -112,27 +112,27 @@ void GreyPhaseAnalysis::Basic(){
}
}
}
Oil.M=sumReduce( Dm->Comm, Oil_local.M);
Oil.Px=sumReduce( Dm->Comm, Oil_local.Px);
Oil.Py=sumReduce( Dm->Comm, Oil_local.Py);
Oil.Pz=sumReduce( Dm->Comm, Oil_local.Pz);
Water.M=sumReduce( Dm->Comm, Water_local.M);
Water.Px=sumReduce( Dm->Comm, Water_local.Px);
Water.Py=sumReduce( Dm->Comm, Water_local.Py);
Water.Pz=sumReduce( Dm->Comm, Water_local.Pz);
Oil.M=Dm->Comm.sumReduce( Oil_local.M);
Oil.Px=Dm->Comm.sumReduce( Oil_local.Px);
Oil.Py=Dm->Comm.sumReduce( Oil_local.Py);
Oil.Pz=Dm->Comm.sumReduce( Oil_local.Pz);
Water.M=Dm->Comm.sumReduce( Water_local.M);
Water.Px=Dm->Comm.sumReduce( Water_local.Px);
Water.Py=Dm->Comm.sumReduce( Water_local.Py);
Water.Pz=Dm->Comm.sumReduce( Water_local.Pz);
//Oil.p /= Oil.M;
//Water.p /= Water.M;
count_w=sumReduce( Dm->Comm, count_w);
count_n=sumReduce( Dm->Comm, count_n);
count_w=Dm->Comm.sumReduce( count_w);
count_n=Dm->Comm.sumReduce( count_n);
if (count_w > 0.0)
Water.p=sumReduce( Dm->Comm, Water_local.p) / count_w;
Water.p=Dm->Comm.sumReduce( Water_local.p) / count_w;
else
Water.p = 0.0;
if (count_n > 0.0)
Oil.p=sumReduce( Dm->Comm, Oil_local.p) / count_n;
Oil.p=Dm->Comm.sumReduce( Oil_local.p) / count_n;
else
Oil.p = 0.0;

2
analysis/GreyPhase.h
View File

@ -10,7 +10,7 @@
#include "common/Communication.h"
#include "analysis/analysis.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"

21
analysis/Minkowski.cpp
View File

@ -20,7 +20,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -139,13 +139,13 @@ void Minkowski::ComputeScalar(const DoubleArray& Field, const double isovalue)
// convert X for 2D manifold to 3D object
Xi *= 0.5;
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// Phase averages
MPI_Allreduce(&Vi,&Vi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Xi,&Xi_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Ai,&Ai_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Ji,&Ji_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Barrier(Dm->Comm);
Vi_global = Dm->Comm.sumReduce( Vi );
Xi_global = Dm->Comm.sumReduce( Xi );
Ai_global = Dm->Comm.sumReduce( Ai );
Ji_global = Dm->Comm.sumReduce( Ji );
Dm->Comm.barrier();
PROFILE_STOP("ComputeScalar");
}
@ -236,7 +236,7 @@ int Minkowski::MeasureConnectedPathway(){
double vF=0.0;
n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
// int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){
@ -277,10 +277,11 @@ int Minkowski::MeasureConnectedPathway(double factor, const DoubleArray &Phi){
}
// Extract only the connected part of NWP
double vF=0.0;
double vF=0.0;
n_connected_components = ComputeGlobalBlobIDs(Nx-2,Ny-2,Nz-2,Dm->rank_info,distance,distance,vF,vF,label,Dm->Comm);
// int n_connected_components = ComputeGlobalPhaseComponent(Nx-2,Ny-2,Nz-2,Dm->rank_info,const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, Dm->Comm )
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=0; k<Nz; k++){
for (int j=0; j<Ny; j++){

2
analysis/Minkowski.h
View File

@ -29,7 +29,7 @@
#include "analysis/filters.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"

213
analysis/SubPhase.cpp
View File

@ -40,7 +40,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -50,7 +50,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
fprintf(SUBPHASE,"Vwc Awc Hwc Xwc "); // wc region
fprintf(SUBPHASE,"Vwd Awd Hwd Xwd Nwd "); // wd region
fprintf(SUBPHASE,"Vnc Anc Hnc Xnc "); // nc region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd region
fprintf(SUBPHASE,"Vnd And Hnd Xnd Nnd "); // nd regionin
fprintf(SUBPHASE,"Vi Ai Hi Xi "); // interface region
fprintf(SUBPHASE,"Vic Aic Hic Xic Nic\n"); // interface region
@ -65,7 +65,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
sprintf(LocalRankFilename,"%s%s","subphase.csv.",LocalRankString);
SUBPHASE = fopen(LocalRankFilename,"a+");
//fprintf(SUBPHASE,"--------------------------------------------------------------------------------------\n");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn ");
fprintf(SUBPHASE,"time rn rw nun nuw Fx Fy Fz iftwn wet ");
fprintf(SUBPHASE,"pwc pwd pnc pnd "); // pressures
fprintf(SUBPHASE,"Mwc Mwd Mwi Mnc Mnd Mni "); // mass
fprintf(SUBPHASE,"Pwc_x Pwd_x Pwi_x Pnc_x Pnd_x Pni_x "); // momentum
@ -93,7 +93,7 @@ SubPhase::SubPhase(std::shared_ptr <Domain> dm):
{
// If timelog is empty, write a short header to list the averages
//fprintf(TIMELOG,"--------------------------------------------------------------------------------------\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn\n");
fprintf(TIMELOG,"sw krw krn vw vn pw pn wet\n");
}
}
}
@ -109,7 +109,7 @@ SubPhase::~SubPhase()
void SubPhase::Write(int timestep)
{
if (Dm->rank()==0){
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction_global);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",gwc.p, gwd.p, gnc.p, gnd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.M, gwd.M, giwn.Mw, gnc.M, gnd.M, giwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",gwc.Px, gwd.Px, giwn.Pwx, gnc.Px, gnd.Px, giwn.Pnx);
@ -125,7 +125,7 @@ void SubPhase::Write(int timestep)
fflush(SUBPHASE);
}
else{
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn);
fprintf(SUBPHASE,"%i %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g ",timestep,rho_n,rho_w,nu_n,nu_w,Fx,Fy,Fz,gamma_wn,total_wetting_interaction);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g ",wc.p, wd.p, nc.p, nd.p);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.M, wd.M, iwn.Mw, nc.M, nd.M, iwn.Mn);
fprintf(SUBPHASE,"%.8g %.8g %.8g %.8g %.8g %.8g ",wc.Px, wd.Px, iwn.Pwx, nc.Px, nd.Px, iwn.Pnx);
@ -172,6 +172,21 @@ void SubPhase::Basic(){
double count_w = 0.0;
double count_n = 0.0;
/* compute the laplacian */
Dm->CommunicateMeshHalo(Phi);
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
// Compute all of the derivatives using finite differences
double fx = 0.5*(Phi(i+1,j,k) - Phi(i-1,j,k));
double fy = 0.5*(Phi(i,j+1,k) - Phi(i,j-1,k));
double fz = 0.5*(Phi(i,j,k+1) - Phi(i,j,k-1));
DelPhi(i,j,k) = sqrt(fx*fx+fy*fy+fz*fz);
}
}
}
Dm->CommunicateMeshHalo(DelPhi);
for (k=0; k<Nz; k++){
for (j=0; j<Ny; j++){
for (i=0; i<Nx; i++){
@ -184,6 +199,11 @@ void SubPhase::Basic(){
double phi = (nA-nB)/(nA+nB);
Phi(n) = phi;
}
if (Phi(n) != Phi(n)){
// check for NaN
Phi(n) = 0.0;
//printf("Nan at %i %i %i \n",i,j,k);
}
}
}
}
@ -229,25 +249,50 @@ void SubPhase::Basic(){
}
}
}
gwb.V=sumReduce( Dm->Comm, wb.V);
gnb.V=sumReduce( Dm->Comm, nb.V);
gwb.M=sumReduce( Dm->Comm, wb.M);
gnb.M=sumReduce( Dm->Comm, nb.M);
gwb.Px=sumReduce( Dm->Comm, wb.Px);
gwb.Py=sumReduce( Dm->Comm, wb.Py);
gwb.Pz=sumReduce( Dm->Comm, wb.Pz);
gnb.Px=sumReduce( Dm->Comm, nb.Px);
gnb.Py=sumReduce( Dm->Comm, nb.Py);
gnb.Pz=sumReduce( Dm->Comm, nb.Pz);
count_w=sumReduce( Dm->Comm, count_w);
count_n=sumReduce( Dm->Comm, count_n);
total_wetting_interaction = count_wetting_interaction = 0.0;
total_wetting_interaction_global = count_wetting_interaction_global=0.0;
for (k=kmin; k<kmax; k++){
for (j=jmin; j<Ny-1; j++){
for (i=imin; i<Nx-1; i++){
n = k*Nx*Ny + j*Nx + i;
// compute contribution of wetting terms (within two voxels of solid)
if ( Dm->id[n] > 0 && SDs(i,j,k) < 2.0 ){
count_wetting_interaction += 1.0;
total_wetting_interaction += DelPhi(i,j,k);
}
}
}
}
//printf("wetting interaction = %f, count = %f\n",total_wetting_interaction,count_wetting_interaction);
total_wetting_interaction_global=Dm->Comm.sumReduce( total_wetting_interaction);
count_wetting_interaction_global=Dm->Comm.sumReduce( count_wetting_interaction);
/* normalize wetting interactions <-- Don't do this if normalizing laplacian (use solid surface area)
if (count_wetting_interaction > 0.0)
total_wetting_interaction /= count_wetting_interaction;
if (count_wetting_interaction_global > 0.0)
total_wetting_interaction_global /= count_wetting_interaction_global;
*/
gwb.V=Dm->Comm.sumReduce( wb.V);
gnb.V=Dm->Comm.sumReduce( nb.V);
gwb.M=Dm->Comm.sumReduce( wb.M);
gnb.M=Dm->Comm.sumReduce( nb.M);
gwb.Px=Dm->Comm.sumReduce( wb.Px);
gwb.Py=Dm->Comm.sumReduce( wb.Py);
gwb.Pz=Dm->Comm.sumReduce( wb.Pz);
gnb.Px=Dm->Comm.sumReduce( nb.Px);
gnb.Py=Dm->Comm.sumReduce( nb.Py);
gnb.Pz=Dm->Comm.sumReduce( nb.Pz);
count_w=Dm->Comm.sumReduce( count_w);
count_n=Dm->Comm.sumReduce( count_n);
if (count_w > 0.0)
gwb.p=sumReduce( Dm->Comm, wb.p) / count_w;
gwb.p=Dm->Comm.sumReduce( wb.p) / count_w;
else
gwb.p = 0.0;
if (count_n > 0.0)
gnb.p=sumReduce( Dm->Comm, nb.p) / count_n;
gnb.p=Dm->Comm.sumReduce( nb.p) / count_n;
else
gnb.p = 0.0;
@ -303,7 +348,7 @@ void SubPhase::Basic(){
double krn = h*h*nu_n*not_water_flow_rate / force_mag ;
double krw = h*h*nu_w*water_flow_rate / force_mag;
//printf(" water saturation = %f, fractional flow =%f \n",saturation,fractional_flow);
fprintf(TIMELOG,"%.5g %.5g %.5g %.5g %.5g %.5g %.5g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p);
fprintf(TIMELOG,"%.8g %.8g %.8g %.8g %.8g %.8g %.8g %.8g\n",saturation,krw,krn,h*water_flow_rate,h*not_water_flow_rate, gwb.p, gnb.p, total_wetting_interaction_global);
fflush(TIMELOG);
}
if (err==true){
@ -445,14 +490,14 @@ void SubPhase::Full(){
nd.X -= nc.X;
// compute global entities
gnc.V=sumReduce( Dm->Comm, nc.V);
gnc.A=sumReduce( Dm->Comm, nc.A);
gnc.H=sumReduce( Dm->Comm, nc.H);
gnc.X=sumReduce( Dm->Comm, nc.X);
gnd.V=sumReduce( Dm->Comm, nd.V);
gnd.A=sumReduce( Dm->Comm, nd.A);
gnd.H=sumReduce( Dm->Comm, nd.H);
gnd.X=sumReduce( Dm->Comm, nd.X);
gnc.V=Dm->Comm.sumReduce( nc.V);
gnc.A=Dm->Comm.sumReduce( nc.A);
gnc.H=Dm->Comm.sumReduce( nc.H);
gnc.X=Dm->Comm.sumReduce( nc.X);
gnd.V=Dm->Comm.sumReduce( nd.V);
gnd.A=Dm->Comm.sumReduce( nd.A);
gnd.H=Dm->Comm.sumReduce( nd.H);
gnd.X=Dm->Comm.sumReduce( nd.X);
gnd.Nc = nd.Nc;
// wetting
for (k=0; k<Nz; k++){
@ -492,14 +537,14 @@ void SubPhase::Full(){
wd.H -= wc.H;
wd.X -= wc.X;
// compute global entities
gwc.V=sumReduce( Dm->Comm, wc.V);
gwc.A=sumReduce( Dm->Comm, wc.A);
gwc.H=sumReduce( Dm->Comm, wc.H);
gwc.X=sumReduce( Dm->Comm, wc.X);
gwd.V=sumReduce( Dm->Comm, wd.V);
gwd.A=sumReduce( Dm->Comm, wd.A);
gwd.H=sumReduce( Dm->Comm, wd.H);
gwd.X=sumReduce( Dm->Comm, wd.X);
gwc.V=Dm->Comm.sumReduce( wc.V);
gwc.A=Dm->Comm.sumReduce( wc.A);
gwc.H=Dm->Comm.sumReduce( wc.H);
gwc.X=Dm->Comm.sumReduce( wc.X);
gwd.V=Dm->Comm.sumReduce( wd.V);
gwd.A=Dm->Comm.sumReduce( wd.A);
gwd.H=Dm->Comm.sumReduce( wd.H);
gwd.X=Dm->Comm.sumReduce( wd.X);
gwd.Nc = wd.Nc;
/* Set up geometric analysis of interface region */
@ -527,20 +572,20 @@ void SubPhase::Full(){
iwn.A = morph_i->A();
iwn.H = morph_i->H();
iwn.X = morph_i->X();
giwn.V=sumReduce( Dm->Comm, iwn.V);
giwn.A=sumReduce( Dm->Comm, iwn.A);
giwn.H=sumReduce( Dm->Comm, iwn.H);
giwn.X=sumReduce( Dm->Comm, iwn.X);
giwn.V=Dm->Comm.sumReduce( iwn.V);
giwn.A=Dm->Comm.sumReduce( iwn.A);
giwn.H=Dm->Comm.sumReduce( iwn.H);
giwn.X=Dm->Comm.sumReduce( iwn.X);
// measure only the connected part
iwnc.Nc = morph_i->MeasureConnectedPathway();
iwnc.V = morph_i->V();
iwnc.A = morph_i->A();
iwnc.H = morph_i->H();
iwnc.X = morph_i->X();
giwnc.V=sumReduce( Dm->Comm, iwnc.V);
giwnc.A=sumReduce( Dm->Comm, iwnc.A);
giwnc.H=sumReduce( Dm->Comm, iwnc.H);
giwnc.X=sumReduce( Dm->Comm, iwnc.X);
giwnc.V=Dm->Comm.sumReduce( iwnc.V);
giwnc.A=Dm->Comm.sumReduce( iwnc.A);
giwnc.H=Dm->Comm.sumReduce( iwnc.H);
giwnc.X=Dm->Comm.sumReduce( iwnc.X);
giwnc.Nc = iwnc.Nc;
double vol_nc_bulk = 0.0;
@ -631,46 +676,46 @@ void SubPhase::Full(){
}
}
gnd.M=sumReduce( Dm->Comm, nd.M);
gnd.Px=sumReduce( Dm->Comm, nd.Px);
gnd.Py=sumReduce( Dm->Comm, nd.Py);
gnd.Pz=sumReduce( Dm->Comm, nd.Pz);
gnd.K=sumReduce( Dm->Comm, nd.K);
gnd.M=Dm->Comm.sumReduce( nd.M);
gnd.Px=Dm->Comm.sumReduce( nd.Px);
gnd.Py=Dm->Comm.sumReduce( nd.Py);
gnd.Pz=Dm->Comm.sumReduce( nd.Pz);
gnd.K=Dm->Comm.sumReduce( nd.K);
gwd.M=sumReduce( Dm->Comm, wd.M);
gwd.Px=sumReduce( Dm->Comm, wd.Px);
gwd.Py=sumReduce( Dm->Comm, wd.Py);
gwd.Pz=sumReduce( Dm->Comm, wd.Pz);
gwd.K=sumReduce( Dm->Comm, wd.K);
gwd.M=Dm->Comm.sumReduce( wd.M);
gwd.Px=Dm->Comm.sumReduce( wd.Px);
gwd.Py=Dm->Comm.sumReduce( wd.Py);
gwd.Pz=Dm->Comm.sumReduce( wd.Pz);
gwd.K=Dm->Comm.sumReduce( wd.K);
gnc.M=sumReduce( Dm->Comm, nc.M);
gnc.Px=sumReduce( Dm->Comm, nc.Px);
gnc.Py=sumReduce( Dm->Comm, nc.Py);
gnc.Pz=sumReduce( Dm->Comm, nc.Pz);
gnc.K=sumReduce( Dm->Comm, nc.K);
gnc.M=Dm->Comm.sumReduce( nc.M);
gnc.Px=Dm->Comm.sumReduce( nc.Px);
gnc.Py=Dm->Comm.sumReduce( nc.Py);
gnc.Pz=Dm->Comm.sumReduce( nc.Pz);
gnc.K=Dm->Comm.sumReduce( nc.K);
gwc.M=sumReduce( Dm->Comm, wc.M);
gwc.Px=sumReduce( Dm->Comm, wc.Px);
gwc.Py=sumReduce( Dm->Comm, wc.Py);
gwc.Pz=sumReduce( Dm->Comm, wc.Pz);
gwc.K=sumReduce( Dm->Comm, wc.K);
gwc.M=Dm->Comm.sumReduce( wc.M);
gwc.Px=Dm->Comm.sumReduce( wc.Px);
gwc.Py=Dm->Comm.sumReduce( wc.Py);
gwc.Pz=Dm->Comm.sumReduce( wc.Pz);
gwc.K=Dm->Comm.sumReduce( wc.K);
giwn.Mn=sumReduce( Dm->Comm, iwn.Mn);
giwn.Pnx=sumReduce( Dm->Comm, iwn.Pnx);
giwn.Pny=sumReduce( Dm->Comm, iwn.Pny);
giwn.Pnz=sumReduce( Dm->Comm, iwn.Pnz);
giwn.Kn=sumReduce( Dm->Comm, iwn.Kn);
giwn.Mw=sumReduce( Dm->Comm, iwn.Mw);
giwn.Pwx=sumReduce( Dm->Comm, iwn.Pwx);
giwn.Pwy=sumReduce( Dm->Comm, iwn.Pwy);
giwn.Pwz=sumReduce( Dm->Comm, iwn.Pwz);
giwn.Kw=sumReduce( Dm->Comm, iwn.Kw);
giwn.Mn=Dm->Comm.sumReduce( iwn.Mn);
giwn.Pnx=Dm->Comm.sumReduce( iwn.Pnx);
giwn.Pny=Dm->Comm.sumReduce( iwn.Pny);
giwn.Pnz=Dm->Comm.sumReduce( iwn.Pnz);
giwn.Kn=Dm->Comm.sumReduce( iwn.Kn);
giwn.Mw=Dm->Comm.sumReduce( iwn.Mw);
giwn.Pwx=Dm->Comm.sumReduce( iwn.Pwx);
giwn.Pwy=Dm->Comm.sumReduce( iwn.Pwy);
giwn.Pwz=Dm->Comm.sumReduce( iwn.Pwz);
giwn.Kw=Dm->Comm.sumReduce( iwn.Kw);
// pressure averaging
gnc.p=sumReduce( Dm->Comm, nc.p);
gnd.p=sumReduce( Dm->Comm, nd.p);
gwc.p=sumReduce( Dm->Comm, wc.p);
gwd.p=sumReduce( Dm->Comm, wd.p);
gnc.p=Dm->Comm.sumReduce( nc.p);
gnd.p=Dm->Comm.sumReduce( nd.p);
gwc.p=Dm->Comm.sumReduce( wc.p);
gwd.p=Dm->Comm.sumReduce( wd.p);
if (vol_wc_bulk > 0.0)
wc.p = wc.p /vol_wc_bulk;
@ -681,10 +726,10 @@ void SubPhase::Full(){
if (vol_nd_bulk > 0.0)
nd.p = nd.p /vol_nd_bulk;
vol_wc_bulk=sumReduce( Dm->Comm, vol_wc_bulk);
vol_wd_bulk=sumReduce( Dm->Comm, vol_wd_bulk);
vol_nc_bulk=sumReduce( Dm->Comm, vol_nc_bulk);
vol_nd_bulk=sumReduce( Dm->Comm, vol_nd_bulk);
vol_wc_bulk=Dm->Comm.sumReduce( vol_wc_bulk);
vol_wd_bulk=Dm->Comm.sumReduce( vol_wd_bulk);
vol_nc_bulk=Dm->Comm.sumReduce( vol_nc_bulk);
vol_nd_bulk=Dm->Comm.sumReduce( vol_nd_bulk);
if (vol_wc_bulk > 0.0)
gwc.p = gwc.p /vol_wc_bulk;
@ -720,7 +765,7 @@ void SubPhase::AggregateLabels( const std::string& filename )
}
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
Dm->AggregateLabels( filename );

10
analysis/SubPhase.h
View File

@ -12,7 +12,7 @@
#include "analysis/distance.h"
#include "analysis/Minkowski.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -68,12 +68,16 @@ public:
* b - bulk (total)
*/
// local entities
phase wc,wd,wb,nc,nd,nb;
phase wc,wd,wb,nc,nd,nb,solid;
interface iwn,iwnc;
// global entities
phase gwc,gwd,gwb,gnc,gnd,gnb;
phase gwc,gwd,gwb,gnc,gnd,gnb,gsolid;
interface giwn,giwnc;
/* fluid-solid wetting interaction */
double total_wetting_interaction, count_wetting_interaction;
double total_wetting_interaction_global, count_wetting_interaction_global;
//...........................................................................
int Nx,Ny,Nz;
IntArray PhaseID; // Phase ID array (solid=0, non-wetting=1, wetting=2)

96
analysis/TwoPhase.cpp
View File

@ -37,7 +37,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"
@ -914,7 +914,7 @@ void TwoPhase::ComponentAverages()
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
if (Dm->rank()==0){
printf("Component averages computed locally -- reducing result... \n");
}
@ -922,14 +922,14 @@ void TwoPhase::ComponentAverages()
RecvBuffer.resize(BLOB_AVG_COUNT,NumberComponents_NWP);
/* for (int b=0; b<NumberComponents_NWP; b++){
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(&ComponentAverages_NWP(0,b),&RecvBuffer(0),BLOB_AVG_COUNT);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_NWP(idx,b)=RecvBuffer(idx);
}
*/
MPI_Barrier(Dm->Comm);
MPI_Allreduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP, MPI_DOUBLE,MPI_SUM,Dm->Comm);
// MPI_Reduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT*NumberComponents_NWP);
// Dm->Comm.sumReduce(ComponentAverages_NWP.data(),RecvBuffer.data(),BLOB_AVG_COUNT);
if (Dm->rank()==0){
printf("rescaling... \n");
@ -1025,9 +1025,8 @@ void TwoPhase::ComponentAverages()
// reduce the wetting phase averages
for (int b=0; b<NumberComponents_WP; b++){
MPI_Barrier(Dm->Comm);
// MPI_Allreduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Reduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT,MPI_DOUBLE,MPI_SUM,0,Dm->Comm);
Dm->Comm.barrier();
Dm->Comm.sumReduce(&ComponentAverages_WP(0,b),RecvBuffer.data(),BLOB_AVG_COUNT);
for (int idx=0; idx<BLOB_AVG_COUNT; idx++) ComponentAverages_WP(idx,b)=RecvBuffer(idx);
}
@ -1110,43 +1109,48 @@ void TwoPhase::Reduce()
int i;
double iVol_global=1.0/Volume;
//...........................................................................
MPI_Barrier(Dm->Comm);
MPI_Allreduce(&nwp_volume,&nwp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wp_volume,&wp_volume_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&awn,&awn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&ans,&ans_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&aws,&aws_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&lwns,&lwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&As,&As_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwn,&Jwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kwn,&Kwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KGwns,&KGwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&KNwns,&KNwns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&efawns,&efawns_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwndnw,&wwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&wwnsdnwn,&wwnsdnwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jwnwwndnw,&Jwnwwndnw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
Dm->Comm.barrier();
nwp_volume_global = Dm->Comm.sumReduce( nwp_volume );
wp_volume_global = Dm->Comm.sumReduce( wp_volume );
awn_global = Dm->Comm.sumReduce( awn );
ans_global = Dm->Comm.sumReduce( ans );
aws_global = Dm->Comm.sumReduce( aws );
lwns_global = Dm->Comm.sumReduce( lwns );
As_global = Dm->Comm.sumReduce( As );
Jwn_global = Dm->Comm.sumReduce( Jwn );
Kwn_global = Dm->Comm.sumReduce( Kwn );
KGwns_global = Dm->Comm.sumReduce( KGwns );
KNwns_global = Dm->Comm.sumReduce( KNwns );
efawns_global = Dm->Comm.sumReduce( efawns );
wwndnw_global = Dm->Comm.sumReduce( wwndnw );
wwnsdnwn_global = Dm->Comm.sumReduce( wwnsdnwn );
Jwnwwndnw_global = Dm->Comm.sumReduce( Jwnwwndnw );
// Phase averages
MPI_Allreduce(&vol_w,&vol_w_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vol_n,&vol_n_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&paw,&paw_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&pan,&pan_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vaw(0),&vaw_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&van(0),&van_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawn(0),&vawn_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&vawns(0),&vawns_global(0),3,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gwn(0),&Gwn_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gns(0),&Gns_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Gws(0),&Gws_global(0),6,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trawn,&trawn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trJwn,&trJwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&trRwn,&trRwn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&euler,&euler_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&An,&An_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Jn,&Jn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&Kn,&Kn_global,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Barrier(Dm->Comm);
vol_w_global = Dm->Comm.sumReduce( vol_w );
vol_n_global = Dm->Comm.sumReduce( vol_n );
paw_global = Dm->Comm.sumReduce( paw );
pan_global = Dm->Comm.sumReduce( pan );
for (int idx=0; idx<3; idx++)
vaw_global(idx) = Dm->Comm.sumReduce( vaw(idx) );
for (int idx=0; idx<3; idx++)
van_global(idx) = Dm->Comm.sumReduce( van(idx));
for (int idx=0; idx<3; idx++)
vawn_global(idx) = Dm->Comm.sumReduce( vawn(idx) );
for (int idx=0; idx<3; idx++)
vawns_global(idx) = Dm->Comm.sumReduce( vawns(idx) );
for (int idx=0; idx<6; idx++){
Gwn_global(idx) = Dm->Comm.sumReduce( Gwn(idx) );
Gns_global(idx) = Dm->Comm.sumReduce( Gns(idx) );
Gws_global(idx) = Dm->Comm.sumReduce( Gws(idx) );
}
trawn_global = Dm->Comm.sumReduce( trawn );
trJwn_global = Dm->Comm.sumReduce( trJwn );
trRwn_global = Dm->Comm.sumReduce( trRwn );
euler_global = Dm->Comm.sumReduce( euler );
An_global = Dm->Comm.sumReduce( An );
Jn_global = Dm->Comm.sumReduce( Jn );
Kn_global = Dm->Comm.sumReduce( Kn );
Dm->Comm.barrier();
// Normalize the phase averages
// (density of both components = 1.0)

2
analysis/TwoPhase.h
View File

@ -28,7 +28,7 @@
#include "common/Domain.h"
#include "common/Communication.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "IO/MeshDatabase.h"
#include "IO/Reader.h"
#include "IO/Writer.h"

83
analysis/analysis.cpp
View File

@ -204,7 +204,7 @@ int ComputeLocalPhaseComponent(const IntArray &PhaseID, int &VALUE, BlobIDArray
/******************************************************************
* Reorder the global blob ids *
******************************************************************/
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, MPI_Comm comm )
static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int ngz, const Utilities::MPI& comm )
{
if ( N_blobs==0 )
return 0;
@ -228,7 +228,7 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
}
}
ASSERT(max_id<N_blobs);
MPI_Allreduce(local_size,global_size,N_blobs,MPI_DOUBLE,MPI_SUM,comm);
comm.sumReduce(local_size,global_size,N_blobs);
std::vector<std::pair<double,int> > map1(N_blobs);
int N_blobs2 = 0;
for (int i=0; i<N_blobs; i++) {
@ -251,12 +251,12 @@ static int ReorderBlobIDs2( BlobIDArray& ID, int N_blobs, int ngx, int ngy, int
PROFILE_STOP("ReorderBlobIDs2",1);
return N_blobs2;
}
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm )
void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm )
{
PROFILE_START("ReorderBlobIDs");
int tmp = ID.max()+1;
int N_blobs = 0;
MPI_Allreduce(&tmp,&N_blobs,1,MPI_INT,MPI_MAX,comm);
N_blobs = comm.maxReduce( tmp );
ReorderBlobIDs2(ID,N_blobs,1,1,1,comm);
PROFILE_STOP("ReorderBlobIDs");
}
@ -276,30 +276,29 @@ static void updateRemoteIds(
int N_send, const std::vector<int>& N_recv,
int64_t *send_buf, std::vector<int64_t*>& recv_buf,
std::map<int64_t,int64_t>& remote_map,
MPI_Comm comm )
const Utilities::MPI& comm )
{
std::vector<MPI_Request> send_req(neighbors.size());
std::vector<MPI_Request> recv_req(neighbors.size());
std::vector<MPI_Status> status(neighbors.size());
std::map<int64_t,global_id_info_struct>::const_iterator it = map.begin();
auto it = map.begin();
ASSERT(N_send==(int)map.size());
for (size_t i=0; i<map.size(); i++, ++it) {
send_buf[2*i+0] = it->first;
send_buf[2*i+1] = it->second.new_id;
}
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Isend( send_buf, 2*N_send, MPI_LONG_LONG, neighbors[i], 0, comm, &send_req[i] );
MPI_Irecv( recv_buf[i], 2*N_recv[i], MPI_LONG_LONG, neighbors[i], 0, comm, &recv_req[i] );
send_req[i] = comm.Isend( send_buf, 2*N_send, neighbors[i], 0 );
recv_req[i] = comm.Irecv( recv_buf[i], 2*N_recv[i], neighbors[i], 0 );
}
for (it=map.begin(); it!=map.end(); ++it) {
remote_map[it->first] = it->second.new_id;
}
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Wait(&recv_req[i],&status[i]);
comm.wait( recv_req[i] );
for (int j=0; j<N_recv[i]; j++)
remote_map[recv_buf[i][2*j+0]] = recv_buf[i][2*j+1];
}
MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
comm.waitAll(neighbors.size(),getPtr(send_req));
}
// Compute a new local id for each local id
static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map,
@ -320,18 +319,18 @@ static bool updateLocalIds( const std::map<int64_t,int64_t>& remote_map,
return changed;
}
static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
int nblobs, BlobIDArray& IDs, MPI_Comm comm )
int nblobs, BlobIDArray& IDs, const Utilities::MPI& comm )
{
PROFILE_START("LocalToGlobalIDs",1);
const int rank = rank_info.rank[1][1][1];
int nprocs = comm_size(comm);
int nprocs = comm.getSize();
const int ngx = (IDs.size(0)-nx)/2;
const int ngy = (IDs.size(1)-ny)/2;
const int ngz = (IDs.size(2)-nz)/2;
// Get the number of blobs for each rank
std::vector<int> N_blobs(nprocs,0);
PROFILE_START("LocalToGlobalIDs-Allgather",1);
MPI_Allgather(&nblobs,1,MPI_INT,getPtr(N_blobs),1,MPI_INT,comm);
comm.allGather(nblobs,getPtr(N_blobs));
PROFILE_STOP("LocalToGlobalIDs-Allgather",1);
int64_t N_blobs_tot = 0;
int offset = 0;
@ -379,13 +378,12 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
std::vector<int> N_recv(neighbors.size(),0);
std::vector<MPI_Request> send_req(neighbors.size());
std::vector<MPI_Request> recv_req(neighbors.size());
std::vector<MPI_Status> status(neighbors.size());
for (size_t i=0; i<neighbors.size(); i++) {
MPI_Isend( &N_send, 1, MPI_INT, neighbors[i], 0, comm, &send_req[i] );
MPI_Irecv( &N_recv[i], 1, MPI_INT, neighbors[i], 0, comm, &recv_req[i] );
send_req[i] = comm.Isend( &N_send, 1, neighbors[i], 0 );
recv_req[i] = comm.Irecv( &N_recv[i], 1, neighbors[i], 0 );
}
MPI_Waitall(neighbors.size(),getPtr(send_req),getPtr(status));
MPI_Waitall(neighbors.size(),getPtr(recv_req),getPtr(status));
comm.waitAll(neighbors.size(),getPtr(send_req));
comm.waitAll(neighbors.size(),getPtr(recv_req));
// Allocate memory for communication
int64_t *send_buf = new int64_t[2*N_send];
std::vector<int64_t*> recv_buf(neighbors.size());
@ -414,8 +412,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
bool changed = updateLocalIds( remote_map, map );
// Check if we are finished
int test = changed ? 1:0;
int result = 0;
MPI_Allreduce(&test,&result,1,MPI_INT,MPI_SUM,comm);
int result = comm.sumReduce( test );
if ( result==0 )
break;
}
@ -451,7 +448,7 @@ static int LocalToGlobalIDs( int nx, int ny, int nz, const RankInfoStruct& rank_
}
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
BlobIDArray& GlobalBlobID, MPI_Comm comm )
BlobIDArray& GlobalBlobID, const Utilities::MPI& comm )
{
PROFILE_START("ComputeGlobalBlobIDs");
// First compute the local ids
@ -462,7 +459,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
return nglobal;
}
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm )
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm )
{
PROFILE_START("ComputeGlobalPhaseComponent");
// First compute the local ids
@ -478,37 +475,27 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
* Compute the mapping of blob ids between timesteps *
******************************************************************/
typedef std::map<BlobIDType,std::map<BlobIDType,int64_t> > map_type;
template<class TYPE> inline MPI_Datatype getMPIType();
template<> inline MPI_Datatype getMPIType<int32_t>() { return MPI_INT; }
template<> inline MPI_Datatype getMPIType<int64_t>() {
if ( sizeof(int64_t)==sizeof(long int) )
return MPI_LONG;
else if ( sizeof(int64_t)==sizeof(double) )
return MPI_DOUBLE;
}
template<class TYPE>
void gatherSet( std::set<TYPE>& set, MPI_Comm comm )
void gatherSet( std::set<TYPE>& set, const Utilities::MPI& comm )
{
int nprocs = comm_size(comm);
MPI_Datatype type = getMPIType<TYPE>();
int nprocs = comm.getSize();
std::vector<TYPE> send_data(set.begin(),set.end());
int send_count = send_data.size();
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
comm.allGather( send_count, getPtr(recv_count) );
for (int i=1; i<nprocs; i++)
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
std::vector<TYPE> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
MPI_Allgatherv(getPtr(send_data),send_count,type,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
comm.allGather( getPtr(send_data), send_count, getPtr(recv_data),
getPtr(recv_count), getPtr(recv_disp), true );
for (size_t i=0; i<recv_data.size(); i++)
set.insert(recv_data[i]);
}
void gatherSrcIDMap( map_type& src_map, MPI_Comm comm )
void gatherSrcIDMap( map_type& src_map, const Utilities::MPI& comm )
{
int nprocs = comm_size(comm);
MPI_Datatype type = getMPIType<int64_t>();
int nprocs = comm.getSize();
std::vector<int64_t> send_data;
for (map_type::const_iterator it=src_map.begin(); it!=src_map.end(); ++it) {
for (auto it=src_map.begin(); it!=src_map.end(); ++it) {
int id = it->first;
const std::map<BlobIDType,int64_t>& src_ids = it->second;
send_data.push_back(id);
@ -521,21 +508,21 @@ void gatherSrcIDMap( map_type& src_map, MPI_Comm comm )
}
int send_count = send_data.size();
std::vector<int> recv_count(nprocs,0), recv_disp(nprocs,0);
MPI_Allgather(&send_count,1,MPI_INT,getPtr(recv_count),1,MPI_INT,comm);
comm.allGather(send_count,getPtr(recv_count));
for (int i=1; i<nprocs; i++)
recv_disp[i] = recv_disp[i-1] + recv_count[i-1];
std::vector<int64_t> recv_data(recv_disp[nprocs-1]+recv_count[nprocs-1]);
MPI_Allgatherv(getPtr(send_data),send_count,type,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),type,comm);
comm.allGather(getPtr(send_data),send_count,
getPtr(recv_data),getPtr(recv_count),getPtr(recv_disp),true);
size_t i=0;
src_map.clear();
while ( i < recv_data.size() ) {
BlobIDType id = recv_data[i];
size_t count = recv_data[i+1];
i += 2;
std::map<BlobIDType,int64_t>& src_ids = src_map[id];
auto& src_ids = src_map[id];
for (size_t j=0; j<count; j++,i+=2) {
std::map<BlobIDType,int64_t>::iterator it = src_ids.find(recv_data[i]);
auto it = src_ids.find(recv_data[i]);
if ( it == src_ids.end() )
src_ids.insert(std::pair<BlobIDType,int64_t>(recv_data[i],recv_data[i+1]));
else
@ -554,7 +541,7 @@ void addSrcDstIDs( BlobIDType src_id, map_type& src_map, map_type& dst_map,
}
}
ID_map_struct computeIDMap( int nx, int ny, int nz,
const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm )
const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm )
{
ASSERT(ID1.size()==ID2.size());
PROFILE_START("computeIDMap");
@ -796,7 +783,7 @@ void renumberIDs( const std::vector<BlobIDType>& new_ids, BlobIDArray& IDs )
******************************************************************/
void writeIDMap( const ID_map_struct& map, long long int timestep, const std::string& filename )
{
int rank = MPI_WORLD_RANK();
int rank = Utilities::MPI( MPI_COMM_WORLD ).getRank();
if ( rank!=0 )
return;
bool empty = map.created.empty() && map.destroyed.empty() &&

8
analysis/analysis.h
View File

@ -74,7 +74,7 @@ int ComputeLocalPhaseComponent( const IntArray &PhaseID, int &VALUE, IntArray &C
*/
int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const DoubleArray& Phase, const DoubleArray& SignDist, double vF, double vS,
BlobIDArray& GlobalBlobID, MPI_Comm comm );
BlobIDArray& GlobalBlobID, const Utilities::MPI& comm );
/*!
@ -91,7 +91,7 @@ int ComputeGlobalBlobIDs( int nx, int ny, int nz, const RankInfoStruct& rank_inf
* @return Return the number of components in the specified phase
*/
int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& rank_info,
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, MPI_Comm comm );
const IntArray &PhaseID, int &VALUE, BlobIDArray &GlobalBlobID, const Utilities::MPI& comm );
/*!
@ -103,7 +103,7 @@ int ComputeGlobalPhaseComponent( int nx, int ny, int nz, const RankInfoStruct& r
* @param[in] nz Number of elements in the z-direction
* @param[in/out] ID The ids of the blobs
*/
void ReorderBlobIDs( BlobIDArray& ID, MPI_Comm comm );
void ReorderBlobIDs( BlobIDArray& ID, const Utilities::MPI& comm );
typedef std::pair<BlobIDType,std::vector<BlobIDType> > BlobIDSplitStruct;
@ -136,7 +136,7 @@ struct ID_map_struct {
* @param[in] ID1 The blob ids at the first timestep
* @param[in] ID2 The blob ids at the second timestep
*/
ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, MPI_Comm comm );
ID_map_struct computeIDMap( int nx, int ny, int nz, const BlobIDArray& ID1, const BlobIDArray& ID2, const Utilities::MPI& comm );
/*!

144
analysis/distance.cpp
View File

@ -192,154 +192,12 @@ void CalcVecDist( Array<Vec> &d, const Array<int> &ID0, const Domain &Dm,
// Update distance
double err = calcVecUpdateInterior( d, dx[0], dx[1], dx[2] );
// Check if we are finished
err = maxReduce( Dm.Comm, err );
err = Dm.Comm.maxReduce( err );
if ( err < tol )
break;
}
}
double Eikonal(DoubleArray &Distance, const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic){
/*
* This routine converts the data in the Distance array to a signed distance
* by solving the equation df/dt = sign(1-|grad f|), where Distance provides
* the values of f on the mesh associated with domain Dm
* It has been tested with segmented data initialized to values [-1,1]
* and will converge toward the signed distance to the surface bounding the associated phases
*
* Reference:
* Min C (2010) On reinitializing level set functions, Journal of Computational Physics229
*/
int i,j,k;
double dt=0.1;
double Dx,Dy,Dz;
double Dxp,Dxm,Dyp,Dym,Dzp,Dzm;
double Dxxp,Dxxm,Dyyp,Dyym,Dzzp,Dzzm;
double sign,norm;
double LocalVar,GlobalVar,LocalMax,GlobalMax;
int xdim,ydim,zdim;
xdim=Dm.Nx-2;
ydim=Dm.Ny-2;
zdim=Dm.Nz-2;
//fillHalo<double> fillData(Dm.Comm, Dm.rank_info,xdim,ydim,zdim,1,1,1,0,1);
fillHalo<double> fillData( Dm.Comm, Dm.rank_info, {xdim, ydim, zdim}, {1,1,1}, 50, 1, {true,true,true}, periodic );
// Arrays to store the second derivatives
DoubleArray Dxx(Dm.Nx,Dm.Ny,Dm.Nz);
DoubleArray Dyy(Dm.Nx,Dm.Ny,Dm.Nz);
DoubleArray Dzz(Dm.Nx,Dm.Ny,Dm.Nz);
int count = 0;
while (count < timesteps){
// Communicate the halo of values
fillData.fill(Distance);
// Compute second order derivatives
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
Dxx(i,j,k) = Distance(i+1,j,k) + Distance(i-1,j,k) - 2*Distance(i,j,k);
Dyy(i,j,k) = Distance(i,j+1,k) + Distance(i,j-1,k) - 2*Distance(i,j,k);
Dzz(i,j,k) = Distance(i,j,k+1) + Distance(i,j,k-1) - 2*Distance(i,j,k);
}
}
}
fillData.fill(Dxx);
fillData.fill(Dyy);
fillData.fill(Dzz);
LocalMax=LocalVar=0.0;
// Execute the next timestep
for (k=1;k<Dm.Nz-1;k++){
for (j=1;j<Dm.Ny-1;j++){
for (i=1;i<Dm.Nx-1;i++){
int n = k*Dm.Nx*Dm.Ny + j*Dm.Nx + i;
sign = -1;
if (ID(i,j,k) == 1) sign = 1;
// local second derivative terms
Dxxp = minmod(Dxx(i,j,k),Dxx(i+1,j,k));
Dyyp = minmod(Dyy(i,j,k),Dyy(i,j+1,k));
Dzzp = minmod(Dzz(i,j,k),Dzz(i,j,k+1));
Dxxm = minmod(Dxx(i,j,k),Dxx(i-1,j,k));
Dyym = minmod(Dyy(i,j,k),Dyy(i,j-1,k));
Dzzm = minmod(Dzz(i,j,k),Dzz(i,j,k-1));
/* //............Compute upwind derivatives ...................
Dxp = Distance(i+1,j,k) - Distance(i,j,k) + 0.5*Dxxp;
Dyp = Distance(i,j+1,k) - Distance(i,j,k) + 0.5*Dyyp;
Dzp = Distance(i,j,k+1) - Distance(i,j,k) + 0.5*Dzzp;
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
*/
Dxp = Distance(i+1,j,k)- Distance(i,j,k) - 0.5*Dxxp;
Dyp = Distance(i,j+1,k)- Distance(i,j,k) - 0.5*Dyyp;
Dzp = Distance(i,j,k+1)- Distance(i,j,k) - 0.5*Dzzp;
Dxm = Distance(i,j,k) - Distance(i-1,j,k) + 0.5*Dxxm;
Dym = Distance(i,j,k) - Distance(i,j-1,k) + 0.5*Dyym;
Dzm = Distance(i,j,k) - Distance(i,j,k-1) + 0.5*Dzzm;
// Compute upwind derivatives for Godunov Hamiltonian
if (sign < 0.0){
if (Dxp + Dxm > 0.f) Dx = Dxp*Dxp;
else Dx = Dxm*Dxm;
if (Dyp + Dym > 0.f) Dy = Dyp*Dyp;
else Dy = Dym*Dym;
if (Dzp + Dzm > 0.f) Dz = Dzp*Dzp;
else Dz = Dzm*Dzm;
}
else{
if (Dxp + Dxm < 0.f) Dx = Dxp*Dxp;
else Dx = Dxm*Dxm;
if (Dyp + Dym < 0.f) Dy = Dyp*Dyp;
else Dy = Dym*Dym;
if (Dzp + Dzm < 0.f) Dz = Dzp*Dzp;
else Dz = Dzm*Dzm;
}
//Dx = max(Dxp*Dxp,Dxm*Dxm);
//Dy = max(Dyp*Dyp,Dym*Dym);
//Dz = max(Dzp*Dzp,Dzm*Dzm);
norm=sqrt(Dx + Dy + Dz);
if (norm > 1.0) norm=1.0;
Distance(i,j,k) += dt*sign*(1.0 - norm);
LocalVar += dt*sign*(1.0 - norm);
if (fabs(dt*sign*(1.0 - norm)) > LocalMax)
LocalMax = fabs(dt*sign*(1.0 - norm));
}
}
}
MPI_Allreduce(&LocalVar,&GlobalVar,1,MPI_DOUBLE,MPI_SUM,Dm.Comm);
MPI_Allreduce(&LocalMax,&GlobalMax,1,MPI_DOUBLE,MPI_MAX,Dm.Comm);
GlobalVar /= Dm.Volume;
count++;
if (count%50 == 0 && Dm.rank()==0 )
printf("Time=%i, Max variation=%f, Global variation=%f \n",count,GlobalMax,GlobalVar);
if (fabs(GlobalMax) < 1e-5){
if (Dm.rank()==0) printf("Exiting with max tolerance of 1e-5 \n");
count=timesteps;
}
}
return GlobalVar;
}
// Explicit instantiations
template void CalcDist<float>( Array<float>&, const Array<char>&, const Domain&, const std::array<bool,3>&, const std::array<double,3>& );

22
analysis/distance.h
View File

@ -32,16 +32,6 @@ struct Vec {
};
inline bool operator<(const Vec& l, const Vec& r){ return l.x*l.x+l.y*l.y+l.z*l.z < r.x*r.x+r.y*r.y+r.z*r.z; }
inline double minmod(double &a, double &b){
double value;
value = a;
if ( a*b < 0.0) value=0.0;
else if (fabs(a) > fabs(b)) value = b;
return value;
}
/*!
* @brief Calculate the distance using a simple method
@ -66,16 +56,4 @@ void CalcDist( Array<TYPE> &Distance, const Array<char> &ID, const Domain &Dm,
void CalcVecDist( Array<Vec> &Distance, const Array<int> &ID, const Domain &Dm,
const std::array<bool,3>& periodic = {true,true,true}, const std::array<double,3>& dx = {1,1,1} );
/*!
* @brief Calculate the distance based on solution of Eikonal equation
* @details This routine calculates the signed distance to the nearest domain surface.
* @param[out] Distance Distance function
* @param[in] ID Domain id
* @param[in] Dm Domain information
* @param[in] timesteps number of timesteps to run for Eikonal solver
* @param[in] periodic Directions that are periodic
*/
double Eikonal(DoubleArray &Distance, const Array<char> &ID, Domain &Dm, int timesteps, const std::array<bool,3>& periodic);
#endif

464
analysis/morphology.cpp
View File

@ -1,7 +1,7 @@
#include <analysis/morphology.h>
// Implementation of morphological opening routine
inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){
inline void PackID(const int *list, int count, signed char *sendbuf, signed char *ID){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
@ -13,7 +13,7 @@ inline void PackID(int *list, int count, signed char *sendbuf, signed char *ID){
}
//***************************************************************************************
inline void UnpackID(int *list, int count, signed char *recvbuf, signed char *ID){
inline void UnpackID(const int *list, int count, signed char *recvbuf, signed char *ID){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
@ -58,11 +58,11 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// total Global is the number of nodes in the pore-space
MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
totalGlobal = Dm->Comm.sumReduce( count );
maxdistGlobal = Dm->Comm.sumReduce( maxdist );
double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
double volume_fraction=totalGlobal/volume;
if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
@ -77,44 +77,44 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
// send buffers
sendID_x = new signed char [Dm->sendCount_x];
sendID_y = new signed char [Dm->sendCount_y];
sendID_z = new signed char [Dm->sendCount_z];
sendID_X = new signed char [Dm->sendCount_X];
sendID_Y = new signed char [Dm->sendCount_Y];
sendID_Z = new signed char [Dm->sendCount_Z];
sendID_xy = new signed char [Dm->sendCount_xy];
sendID_yz = new signed char [Dm->sendCount_yz];
sendID_xz = new signed char [Dm->sendCount_xz];
sendID_Xy = new signed char [Dm->sendCount_Xy];
sendID_Yz = new signed char [Dm->sendCount_Yz];
sendID_xZ = new signed char [Dm->sendCount_xZ];
sendID_xY = new signed char [Dm->sendCount_xY];
sendID_yZ = new signed char [Dm->sendCount_yZ];
sendID_Xz = new signed char [Dm->sendCount_Xz];
sendID_XY = new signed char [Dm->sendCount_XY];
sendID_YZ = new signed char [Dm->sendCount_YZ];
sendID_XZ = new signed char [Dm->sendCount_XZ];
sendID_x = new signed char [Dm->sendCount("x")];
sendID_y = new signed char [Dm->sendCount("y")];
sendID_z = new signed char [Dm->sendCount("z")];
sendID_X = new signed char [Dm->sendCount("X")];
sendID_Y = new signed char [Dm->sendCount("Y")];
sendID_Z = new signed char [Dm->sendCount("Z")];
sendID_xy = new signed char [Dm->sendCount("xy")];
sendID_yz = new signed char [Dm->sendCount("yz")];
sendID_xz = new signed char [Dm->sendCount("xz")];
sendID_Xy = new signed char [Dm->sendCount("Xy")];
sendID_Yz = new signed char [Dm->sendCount("Yz")];
sendID_xZ = new signed char [Dm->sendCount("xZ")];
sendID_xY = new signed char [Dm->sendCount("xY")];
sendID_yZ = new signed char [Dm->sendCount("yZ")];
sendID_Xz = new signed char [Dm->sendCount("Xz")];
sendID_XY = new signed char [Dm->sendCount("XY")];
sendID_YZ = new signed char [Dm->sendCount("YZ")];
sendID_XZ = new signed char [Dm->sendCount("XZ")];
//......................................................................................
// recv buffers
recvID_x = new signed char [Dm->recvCount_x];
recvID_y = new signed char [Dm->recvCount_y];
recvID_z = new signed char [Dm->recvCount_z];
recvID_X = new signed char [Dm->recvCount_X];
recvID_Y = new signed char [Dm->recvCount_Y];
recvID_Z = new signed char [Dm->recvCount_Z];
recvID_xy = new signed char [Dm->recvCount_xy];
recvID_yz = new signed char [Dm->recvCount_yz];
recvID_xz = new signed char [Dm->recvCount_xz];
recvID_Xy = new signed char [Dm->recvCount_Xy];
recvID_xZ = new signed char [Dm->recvCount_xZ];
recvID_xY = new signed char [Dm->recvCount_xY];
recvID_yZ = new signed char [Dm->recvCount_yZ];
recvID_Yz = new signed char [Dm->recvCount_Yz];
recvID_Xz = new signed char [Dm->recvCount_Xz];
recvID_XY = new signed char [Dm->recvCount_XY];
recvID_YZ = new signed char [Dm->recvCount_YZ];
recvID_XZ = new signed char [Dm->recvCount_XZ];
recvID_x = new signed char [Dm->recvCount("x")];
recvID_y = new signed char [Dm->recvCount("y")];
recvID_z = new signed char [Dm->recvCount("z")];
recvID_X = new signed char [Dm->recvCount("X")];
recvID_Y = new signed char [Dm->recvCount("Y")];
recvID_Z = new signed char [Dm->recvCount("Z")];
recvID_xy = new signed char [Dm->recvCount("xy")];
recvID_yz = new signed char [Dm->recvCount("yz")];
recvID_xz = new signed char [Dm->recvCount("xz")];
recvID_Xy = new signed char [Dm->recvCount("Xy")];
recvID_xZ = new signed char [Dm->recvCount("xZ")];
recvID_xY = new signed char [Dm->recvCount("xY")];
recvID_yZ = new signed char [Dm->recvCount("yZ")];
recvID_Yz = new signed char [Dm->recvCount("Yz")];
recvID_Xz = new signed char [Dm->recvCount("Xz")];
recvID_XY = new signed char [Dm->recvCount("XY")];
recvID_YZ = new signed char [Dm->recvCount("YZ")];
recvID_XZ = new signed char [Dm->recvCount("XZ")];
//......................................................................................
int sendtag,recvtag;
sendtag = recvtag = 7;
@ -131,9 +131,8 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
// Increase the critical radius until the target saturation is met
double deltaR=0.05; // amount to change the radius in voxel units
double Rcrit_old=0.0;
double Rcrit_old;
double GlobalNumber = 1.f;
int imin,jmin,kmin,imax,jmax,kmax;
if (ErodeLabel == 1){
@ -183,83 +182,65 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
// Pack and send the updated ID values
PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
//......................................................................................
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
//double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -272,7 +253,7 @@ double MorphOpen(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain>
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
void_fraction_new = countGlobal/totalGlobal;
void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
/* if (rank==0){
@ -304,7 +285,7 @@ double morph_open()
fillHalo<char> fillChar(Dm->Comm,Dm->rank_info,{Nx-2,Ny-2,Nz-2},{1,1,1},0,1);
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
count = 0.f;
for (int k=1; k<Nz-1; k++){
@ -317,7 +298,7 @@ double morph_open()
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
return countGlobal;
}
*/
@ -360,11 +341,11 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
// total Global is the number of nodes in the pore-space
MPI_Allreduce(&count,&totalGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
MPI_Allreduce(&maxdist,&maxdistGlobal,1,MPI_DOUBLE,MPI_MAX,Dm->Comm);
totalGlobal = Dm->Comm.sumReduce( count );
maxdistGlobal = Dm->Comm.sumReduce( maxdist );
double volume=double(nprocx*nprocy*nprocz)*double(nx-2)*double(ny-2)*double(nz-2);
double volume_fraction=totalGlobal/volume;
if (rank==0) printf("Volume fraction for morphological opening: %f \n",volume_fraction);
@ -378,44 +359,44 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
signed char *recvID_xy, *recvID_yz, *recvID_xz, *recvID_Xy, *recvID_Yz, *recvID_xZ;
signed char *recvID_xY, *recvID_yZ, *recvID_Xz, *recvID_XY, *recvID_YZ, *recvID_XZ;
// send buffers
sendID_x = new signed char [Dm->sendCount_x];
sendID_y = new signed char [Dm->sendCount_y];
sendID_z = new signed char [Dm->sendCount_z];
sendID_X = new signed char [Dm->sendCount_X];
sendID_Y = new signed char [Dm->sendCount_Y];
sendID_Z = new signed char [Dm->sendCount_Z];
sendID_xy = new signed char [Dm->sendCount_xy];
sendID_yz = new signed char [Dm->sendCount_yz];
sendID_xz = new signed char [Dm->sendCount_xz];
sendID_Xy = new signed char [Dm->sendCount_Xy];
sendID_Yz = new signed char [Dm->sendCount_Yz];
sendID_xZ = new signed char [Dm->sendCount_xZ];
sendID_xY = new signed char [Dm->sendCount_xY];
sendID_yZ = new signed char [Dm->sendCount_yZ];
sendID_Xz = new signed char [Dm->sendCount_Xz];
sendID_XY = new signed char [Dm->sendCount_XY];
sendID_YZ = new signed char [Dm->sendCount_YZ];
sendID_XZ = new signed char [Dm->sendCount_XZ];
sendID_x = new signed char [Dm->sendCount("x")];
sendID_y = new signed char [Dm->sendCount("y")];
sendID_z = new signed char [Dm->sendCount("z")];
sendID_X = new signed char [Dm->sendCount("X")];
sendID_Y = new signed char [Dm->sendCount("Y")];
sendID_Z = new signed char [Dm->sendCount("Z")];
sendID_xy = new signed char [Dm->sendCount("xy")];
sendID_yz = new signed char [Dm->sendCount("yz")];
sendID_xz = new signed char [Dm->sendCount("xz")];
sendID_Xy = new signed char [Dm->sendCount("Xy")];
sendID_Yz = new signed char [Dm->sendCount("Yz")];
sendID_xZ = new signed char [Dm->sendCount("xZ")];
sendID_xY = new signed char [Dm->sendCount("xY")];
sendID_yZ = new signed char [Dm->sendCount("yZ")];
sendID_Xz = new signed char [Dm->sendCount("Xz")];
sendID_XY = new signed char [Dm->sendCount("XY")];
sendID_YZ = new signed char [Dm->sendCount("YZ")];
sendID_XZ = new signed char [Dm->sendCount("XZ")];
//......................................................................................
// recv buffers
recvID_x = new signed char [Dm->recvCount_x];
recvID_y = new signed char [Dm->recvCount_y];
recvID_z = new signed char [Dm->recvCount_z];
recvID_X = new signed char [Dm->recvCount_X];
recvID_Y = new signed char [Dm->recvCount_Y];
recvID_Z = new signed char [Dm->recvCount_Z];
recvID_xy = new signed char [Dm->recvCount_xy];
recvID_yz = new signed char [Dm->recvCount_yz];
recvID_xz = new signed char [Dm->recvCount_xz];
recvID_Xy = new signed char [Dm->recvCount_Xy];
recvID_xZ = new signed char [Dm->recvCount_xZ];
recvID_xY = new signed char [Dm->recvCount_xY];
recvID_yZ = new signed char [Dm->recvCount_yZ];
recvID_Yz = new signed char [Dm->recvCount_Yz];
recvID_Xz = new signed char [Dm->recvCount_Xz];
recvID_XY = new signed char [Dm->recvCount_XY];
recvID_YZ = new signed char [Dm->recvCount_YZ];
recvID_XZ = new signed char [Dm->recvCount_XZ];
recvID_x = new signed char [Dm->recvCount("x")];
recvID_y = new signed char [Dm->recvCount("y")];
recvID_z = new signed char [Dm->recvCount("z")];
recvID_X = new signed char [Dm->recvCount("X")];
recvID_Y = new signed char [Dm->recvCount("Y")];
recvID_Z = new signed char [Dm->recvCount("Z")];
recvID_xy = new signed char [Dm->recvCount("xy")];
recvID_yz = new signed char [Dm->recvCount("yz")];
recvID_xz = new signed char [Dm->recvCount("xz")];
recvID_Xy = new signed char [Dm->recvCount("Xy")];
recvID_xZ = new signed char [Dm->recvCount("xZ")];
recvID_xY = new signed char [Dm->recvCount("xY")];
recvID_yZ = new signed char [Dm->recvCount("yZ")];
recvID_Yz = new signed char [Dm->recvCount("Yz")];
recvID_Xz = new signed char [Dm->recvCount("Xz")];
recvID_XY = new signed char [Dm->recvCount("XY")];
recvID_YZ = new signed char [Dm->recvCount("YZ")];
recvID_XZ = new signed char [Dm->recvCount("XZ")];
//......................................................................................
int sendtag,recvtag;
sendtag = recvtag = 7;
@ -434,7 +415,6 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
double deltaR=0.05; // amount to change the radius in voxel units
double Rcrit_old;
double GlobalNumber = 1.f;
int imin,jmin,kmin,imax,jmax,kmax;
double Rcrit_new = maxdistGlobal;
@ -442,7 +422,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
// Rcrit_new = strtod(argv[2],NULL);
// if (rank==0) printf("Max. distance =%f, Initial critical radius = %f \n",maxdistGlobal,Rcrit_new);
//}
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
FILE *DRAIN = fopen("morphdrain.csv","w");
@ -489,82 +469,64 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
// Pack and send the updated ID values
PackID(Dm->sendList_x, Dm->sendCount_x ,sendID_x, id);
PackID(Dm->sendList_X, Dm->sendCount_X ,sendID_X, id);
PackID(Dm->sendList_y, Dm->sendCount_y ,sendID_y, id);
PackID(Dm->sendList_Y, Dm->sendCount_Y ,sendID_Y, id);
PackID(Dm->sendList_z, Dm->sendCount_z ,sendID_z, id);
PackID(Dm->sendList_Z, Dm->sendCount_Z ,sendID_Z, id);
PackID(Dm->sendList_xy, Dm->sendCount_xy ,sendID_xy, id);
PackID(Dm->sendList_Xy, Dm->sendCount_Xy ,sendID_Xy, id);
PackID(Dm->sendList_xY, Dm->sendCount_xY ,sendID_xY, id);
PackID(Dm->sendList_XY, Dm->sendCount_XY ,sendID_XY, id);
PackID(Dm->sendList_xz, Dm->sendCount_xz ,sendID_xz, id);
PackID(Dm->sendList_Xz, Dm->sendCount_Xz ,sendID_Xz, id);
PackID(Dm->sendList_xZ, Dm->sendCount_xZ ,sendID_xZ, id);
PackID(Dm->sendList_XZ, Dm->sendCount_XZ ,sendID_XZ, id);
PackID(Dm->sendList_yz, Dm->sendCount_yz ,sendID_yz, id);
PackID(Dm->sendList_Yz, Dm->sendCount_Yz ,sendID_Yz, id);
PackID(Dm->sendList_yZ, Dm->sendCount_yZ ,sendID_yZ, id);
PackID(Dm->sendList_YZ, Dm->sendCount_YZ ,sendID_YZ, id);
PackID(Dm->sendList("x"), Dm->sendCount("x") ,sendID_x, id);
PackID(Dm->sendList("X"), Dm->sendCount("X") ,sendID_X, id);
PackID(Dm->sendList("y"), Dm->sendCount("y") ,sendID_y, id);
PackID(Dm->sendList("Y"), Dm->sendCount("Y") ,sendID_Y, id);
PackID(Dm->sendList("z"), Dm->sendCount("z") ,sendID_z, id);
PackID(Dm->sendList("Z"), Dm->sendCount("Z") ,sendID_Z, id);
PackID(Dm->sendList("xy"), Dm->sendCount("xy") ,sendID_xy, id);
PackID(Dm->sendList("Xy"), Dm->sendCount("Xy") ,sendID_Xy, id);
PackID(Dm->sendList("xY"), Dm->sendCount("xY") ,sendID_xY, id);
PackID(Dm->sendList("XY"), Dm->sendCount("XY") ,sendID_XY, id);
PackID(Dm->sendList("xz"), Dm->sendCount("xz") ,sendID_xz, id);
PackID(Dm->sendList("Xz"), Dm->sendCount("Xz") ,sendID_Xz, id);
PackID(Dm->sendList("xZ"), Dm->sendCount("xZ") ,sendID_xZ, id);
PackID(Dm->sendList("XZ"), Dm->sendCount("XZ") ,sendID_XZ, id);
PackID(Dm->sendList("yz"), Dm->sendCount("yz") ,sendID_yz, id);
PackID(Dm->sendList("Yz"), Dm->sendCount("Yz") ,sendID_Yz, id);
PackID(Dm->sendList("yZ"), Dm->sendCount("yZ") ,sendID_yZ, id);
PackID(Dm->sendList("YZ"), Dm->sendCount("YZ") ,sendID_YZ, id);
//......................................................................................
MPI_Sendrecv(sendID_x,Dm->sendCount_x,MPI_CHAR,Dm->rank_x(),sendtag,
recvID_X,Dm->recvCount_X,MPI_CHAR,Dm->rank_X(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_X,Dm->sendCount_X,MPI_CHAR,Dm->rank_X(),sendtag,
recvID_x,Dm->recvCount_x,MPI_CHAR,Dm->rank_x(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_y,Dm->sendCount_y,MPI_CHAR,Dm->rank_y(),sendtag,
recvID_Y,Dm->recvCount_Y,MPI_CHAR,Dm->rank_Y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Y,Dm->sendCount_Y,MPI_CHAR,Dm->rank_Y(),sendtag,
recvID_y,Dm->recvCount_y,MPI_CHAR,Dm->rank_y(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_z,Dm->sendCount_z,MPI_CHAR,Dm->rank_z(),sendtag,
recvID_Z,Dm->recvCount_Z,MPI_CHAR,Dm->rank_Z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Z,Dm->sendCount_Z,MPI_CHAR,Dm->rank_Z(),sendtag,
recvID_z,Dm->recvCount_z,MPI_CHAR,Dm->rank_z(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xy,Dm->sendCount_xy,MPI_CHAR,Dm->rank_xy(),sendtag,
recvID_XY,Dm->recvCount_XY,MPI_CHAR,Dm->rank_XY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XY,Dm->sendCount_XY,MPI_CHAR,Dm->rank_XY(),sendtag,
recvID_xy,Dm->recvCount_xy,MPI_CHAR,Dm->rank_xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xy,Dm->sendCount_Xy,MPI_CHAR,Dm->rank_Xy(),sendtag,
recvID_xY,Dm->recvCount_xY,MPI_CHAR,Dm->rank_xY(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xY,Dm->sendCount_xY,MPI_CHAR,Dm->rank_xY(),sendtag,
recvID_Xy,Dm->recvCount_Xy,MPI_CHAR,Dm->rank_Xy(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xz,Dm->sendCount_xz,MPI_CHAR,Dm->rank_xz(),sendtag,
recvID_XZ,Dm->recvCount_XZ,MPI_CHAR,Dm->rank_XZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_XZ,Dm->sendCount_XZ,MPI_CHAR,Dm->rank_XZ(),sendtag,
recvID_xz,Dm->recvCount_xz,MPI_CHAR,Dm->rank_xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Xz,Dm->sendCount_Xz,MPI_CHAR,Dm->rank_Xz(),sendtag,
recvID_xZ,Dm->recvCount_xZ,MPI_CHAR,Dm->rank_xZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_xZ,Dm->sendCount_xZ,MPI_CHAR,Dm->rank_xZ(),sendtag,
recvID_Xz,Dm->recvCount_Xz,MPI_CHAR,Dm->rank_Xz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yz,Dm->sendCount_yz,MPI_CHAR,Dm->rank_yz(),sendtag,
recvID_YZ,Dm->recvCount_YZ,MPI_CHAR,Dm->rank_YZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_YZ,Dm->sendCount_YZ,MPI_CHAR,Dm->rank_YZ(),sendtag,
recvID_yz,Dm->recvCount_yz,MPI_CHAR,Dm->rank_yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_Yz,Dm->sendCount_Yz,MPI_CHAR,Dm->rank_Yz(),sendtag,
recvID_yZ,Dm->recvCount_yZ,MPI_CHAR,Dm->rank_yZ(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendID_yZ,Dm->sendCount_yZ,MPI_CHAR,Dm->rank_yZ(),sendtag,
recvID_Yz,Dm->recvCount_Yz,MPI_CHAR,Dm->rank_Yz(),recvtag,Dm->Comm,MPI_STATUS_IGNORE);
Dm->Comm.sendrecv(sendID_x,Dm->sendCount("x"),Dm->rank_x(),sendtag,recvID_X,Dm->recvCount("X"),Dm->rank_X(),recvtag);
Dm->Comm.sendrecv(sendID_X,Dm->sendCount("X"),Dm->rank_X(),sendtag,recvID_x,Dm->recvCount("x"),Dm->rank_x(),recvtag);
Dm->Comm.sendrecv(sendID_y,Dm->sendCount("y"),Dm->rank_y(),sendtag,recvID_Y,Dm->recvCount("Y"),Dm->rank_Y(),recvtag);
Dm->Comm.sendrecv(sendID_Y,Dm->sendCount("Y"),Dm->rank_Y(),sendtag,recvID_y,Dm->recvCount("y"),Dm->rank_y(),recvtag);
Dm->Comm.sendrecv(sendID_z,Dm->sendCount("z"),Dm->rank_z(),sendtag,recvID_Z,Dm->recvCount("Z"),Dm->rank_Z(),recvtag);
Dm->Comm.sendrecv(sendID_Z,Dm->sendCount("Z"),Dm->rank_Z(),sendtag,recvID_z,Dm->recvCount("z"),Dm->rank_z(),recvtag);
Dm->Comm.sendrecv(sendID_xy,Dm->sendCount("xy"),Dm->rank_xy(),sendtag,recvID_XY,Dm->recvCount("XY"),Dm->rank_XY(),recvtag);
Dm->Comm.sendrecv(sendID_XY,Dm->sendCount("XY"),Dm->rank_XY(),sendtag,recvID_xy,Dm->recvCount("xy"),Dm->rank_xy(),recvtag);
Dm->Comm.sendrecv(sendID_Xy,Dm->sendCount("Xy"),Dm->rank_Xy(),sendtag,recvID_xY,Dm->recvCount("xY"),Dm->rank_xY(),recvtag);
Dm->Comm.sendrecv(sendID_xY,Dm->sendCount("xY"),Dm->rank_xY(),sendtag,recvID_Xy,Dm->recvCount("Xy"),Dm->rank_Xy(),recvtag);
Dm->Comm.sendrecv(sendID_xz,Dm->sendCount("xz"),Dm->rank_xz(),sendtag,recvID_XZ,Dm->recvCount("XZ"),Dm->rank_XZ(),recvtag);
Dm->Comm.sendrecv(sendID_XZ,Dm->sendCount("XZ"),Dm->rank_XZ(),sendtag,recvID_xz,Dm->recvCount("xz"),Dm->rank_xz(),recvtag);
Dm->Comm.sendrecv(sendID_Xz,Dm->sendCount("Xz"),Dm->rank_Xz(),sendtag,recvID_xZ,Dm->recvCount("xZ"),Dm->rank_xZ(),recvtag);
Dm->Comm.sendrecv(sendID_xZ,Dm->sendCount("xZ"),Dm->rank_xZ(),sendtag,recvID_Xz,Dm->recvCount("Xz"),Dm->rank_Xz(),recvtag);
Dm->Comm.sendrecv(sendID_yz,Dm->sendCount("yz"),Dm->rank_yz(),sendtag,recvID_YZ,Dm->recvCount("YZ"),Dm->rank_YZ(),recvtag);
Dm->Comm.sendrecv(sendID_YZ,Dm->sendCount("YZ"),Dm->rank_YZ(),sendtag,recvID_yz,Dm->recvCount("yz"),Dm->rank_yz(),recvtag);
Dm->Comm.sendrecv(sendID_Yz,Dm->sendCount("Yz"),Dm->rank_Yz(),sendtag,recvID_yZ,Dm->recvCount("yZ"),Dm->rank_yZ(),recvtag);
Dm->Comm.sendrecv(sendID_yZ,Dm->sendCount("yZ"),Dm->rank_yZ(),sendtag,recvID_Yz,Dm->recvCount("Yz"),Dm->rank_Yz(),recvtag);
//......................................................................................
UnpackID(Dm->recvList_x, Dm->recvCount_x ,recvID_x, id);
UnpackID(Dm->recvList_X, Dm->recvCount_X ,recvID_X, id);
UnpackID(Dm->recvList_y, Dm->recvCount_y ,recvID_y, id);
UnpackID(Dm->recvList_Y, Dm->recvCount_Y ,recvID_Y, id);
UnpackID(Dm->recvList_z, Dm->recvCount_z ,recvID_z, id);
UnpackID(Dm->recvList_Z, Dm->recvCount_Z ,recvID_Z, id);
UnpackID(Dm->recvList_xy, Dm->recvCount_xy ,recvID_xy, id);
UnpackID(Dm->recvList_Xy, Dm->recvCount_Xy ,recvID_Xy, id);
UnpackID(Dm->recvList_xY, Dm->recvCount_xY ,recvID_xY, id);
UnpackID(Dm->recvList_XY, Dm->recvCount_XY ,recvID_XY, id);
UnpackID(Dm->recvList_xz, Dm->recvCount_xz ,recvID_xz, id);
UnpackID(Dm->recvList_Xz, Dm->recvCount_Xz ,recvID_Xz, id);
UnpackID(Dm->recvList_xZ, Dm->recvCount_xZ ,recvID_xZ, id);
UnpackID(Dm->recvList_XZ, Dm->recvCount_XZ ,recvID_XZ, id);
UnpackID(Dm->recvList_yz, Dm->recvCount_yz ,recvID_yz, id);
UnpackID(Dm->recvList_Yz, Dm->recvCount_Yz ,recvID_Yz, id);
UnpackID(Dm->recvList_yZ, Dm->recvCount_yZ ,recvID_yZ, id);
UnpackID(Dm->recvList_YZ, Dm->recvCount_YZ ,recvID_YZ, id);
UnpackID(Dm->recvList("x"), Dm->recvCount("x") ,recvID_x, id);
UnpackID(Dm->recvList("X"), Dm->recvCount("X") ,recvID_X, id);
UnpackID(Dm->recvList("y"), Dm->recvCount("y") ,recvID_y, id);
UnpackID(Dm->recvList("Y"), Dm->recvCount("Y") ,recvID_Y, id);
UnpackID(Dm->recvList("z"), Dm->recvCount("z") ,recvID_z, id);
UnpackID(Dm->recvList("Z"), Dm->recvCount("Z") ,recvID_Z, id);
UnpackID(Dm->recvList("xy"), Dm->recvCount("xy") ,recvID_xy, id);
UnpackID(Dm->recvList("Xy"), Dm->recvCount("Xy") ,recvID_Xy, id);
UnpackID(Dm->recvList("xY"), Dm->recvCount("xY") ,recvID_xY, id);
UnpackID(Dm->recvList("XY"), Dm->recvCount("XY") ,recvID_XY, id);
UnpackID(Dm->recvList("xz"), Dm->recvCount("xz") ,recvID_xz, id);
UnpackID(Dm->recvList("Xz"), Dm->recvCount("Xz") ,recvID_Xz, id);
UnpackID(Dm->recvList("xZ"), Dm->recvCount("xZ") ,recvID_xZ, id);
UnpackID(Dm->recvList("XZ"), Dm->recvCount("XZ") ,recvID_XZ, id);
UnpackID(Dm->recvList("yz"), Dm->recvCount("yz") ,recvID_yz, id);
UnpackID(Dm->recvList("Yz"), Dm->recvCount("Yz") ,recvID_Yz, id);
UnpackID(Dm->recvList("yZ"), Dm->recvCount("yZ") ,recvID_yZ, id);
UnpackID(Dm->recvList("YZ"), Dm->recvCount("YZ") ,recvID_YZ, id);
//......................................................................................
MPI_Allreduce(&LocalNumber,&GlobalNumber,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
// double GlobalNumber = Dm->Comm.sumReduce( LocalNumber );
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
@ -580,10 +542,9 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
// Extract only the connected part of NWP
BlobIDstruct new_index;
double vF=0.0; double vS=0.0;
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=0; k<nz; k++){
for (int j=0; j<ny; j++){
@ -619,7 +580,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
ComputeGlobalBlobIDs(nx-2,ny-2,nz-2,Dm->rank_info,phase,SignDist,vF,vS,phase_label,Dm->Comm);
MPI_Barrier(Dm->Comm);
Dm->Comm.barrier();
for (int k=1; k<nz-1; k++){
for (int j=1; j<ny-1; j++){
@ -645,7 +606,7 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
}
}
}
MPI_Allreduce(&count,&countGlobal,1,MPI_DOUBLE,MPI_SUM,Dm->Comm);
countGlobal = Dm->Comm.sumReduce( count );
void_fraction_new = countGlobal/totalGlobal;
void_fraction_diff_new = abs(void_fraction_new-VoidFraction);
if (rank==0){
@ -685,13 +646,14 @@ double MorphDrain(DoubleArray &SignDist, signed char *id, std::shared_ptr<Domain
return final_void_fraction;
}
//double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth)
double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id, std::shared_ptr<Domain> Dm, double TargetGrowth, double WallFactor)
{
int Nx = Dm->Nx;
int Ny = Dm->Ny;
int Nz = Dm->Nz;
int rank = Dm->rank();
double count=0.0;
for (int k=1; k<Nz-1; k++){
for (int j=1; j<Ny-1; j++){
@ -702,7 +664,7 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
}
}
}
double count_original=sumReduce( Dm->Comm, count);
double count_original = Dm->Comm.sumReduce( count);
// Estimate morph_delta
double morph_delta = 0.0;
@ -722,7 +684,9 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
double walldist=BoundaryDist(i,j,k);
//double wallweight = 1.0 / (1+exp(-5.f*(walldist-1.f)));
double wallweight = WallFactor/ (1+exp(-5.f*(walldist-1.f)));
//wallweight = 1.0;
if (fabs(wallweight*morph_delta) > MAX_DISPLACEMENT) MAX_DISPLACEMENT= fabs(wallweight*morph_delta);
if (Dist(i,j,k) - wallweight*morph_delta < 0.0){
@ -731,19 +695,21 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
}
}
}
count=sumReduce( Dm->Comm, count);
MAX_DISPLACEMENT = maxReduce( Dm->Comm, MAX_DISPLACEMENT);
count = Dm->Comm.sumReduce( count );
MAX_DISPLACEMENT = Dm->Comm.maxReduce( MAX_DISPLACEMENT );
GrowthEstimate = count - count_original;
ERROR = fabs((GrowthEstimate-TargetGrowth) /TargetGrowth);
if (rank == 0) printf(" delta=%f, growth=%f, max. displacement = %f \n",morph_delta, GrowthEstimate, MAX_DISPLACEMENT);
// Now adjust morph_delta
double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
GrowthPrevious = GrowthEstimate;
morph_delta_previous = morph_delta;
morph_delta += step_size;
if (fabs(GrowthEstimate - GrowthPrevious) > 0.0) {
double step_size = (TargetGrowth - GrowthEstimate)*(morph_delta - morph_delta_previous) / (GrowthEstimate - GrowthPrevious);
GrowthPrevious = GrowthEstimate;
morph_delta_previous = morph_delta;
morph_delta += step_size;
}
if (morph_delta / morph_delta_previous > 2.0 ) morph_delta = morph_delta_previous*2.0;
//MAX_DISPLACEMENT *= max(TargetGrowth/GrowthEstimate,1.25);
if (morph_delta > 0.0 ){
@ -768,14 +734,16 @@ double MorphGrow(DoubleArray &BoundaryDist, DoubleArray &Dist, Array<char> &id,
for (int j=1; j<Ny-1; j++){
for (int i=1; i<Nx-1; i++){
double walldist=BoundaryDist(i,j,k);
double wallweight = WallFactor / (1+exp(-5.f*(walldist-1.f)));
//double wallweight = 1.0 / (1+exp(-5.f*(walldist-1.f)));
//wallweight = 1.0;
double wallweight = WallFactor / (1+exp(-5.f*(walldist-1.f)));
Dist(i,j,k) -= wallweight*morph_delta;
if (Dist(i,j,k) < 0.0) count+=1.0;
}
}
}
count=sumReduce( Dm->Comm, count);
count = Dm->Comm.sumReduce( count );
return count;
}

1332
analysis/runAnalysis.cpp
View File

File diff suppressed because it is too large Load Diff

74
analysis/runAnalysis.h
View File

@ -17,41 +17,51 @@
#ifndef RunAnalysis_H_INC
#define RunAnalysis_H_INC
#include "analysis/analysis.h"
#include "analysis/TwoPhase.h"
#include "analysis/SubPhase.h"
#include "analysis/TwoPhase.h"
#include "analysis/analysis.h"
#include "common/Communication.h"
#include "common/ScaLBL.h"
#include "threadpool/thread_pool.h"
#include "models/ColorModel.h"
#include <limits.h>
typedef std::shared_ptr<std::pair<int,IntArray>> BlobIDstruct;
typedef std::shared_ptr<std::vector<BlobIDType>> BlobIDList;
// Types of analysis
enum class AnalysisType : uint64_t { AnalyzeNone=0, IdentifyBlobs=0x01, CopyPhaseIndicator=0x02,
CopySimState=0x04, ComputeAverages=0x08, CreateRestart=0x10, WriteVis=0x20, ComputeSubphase=0x40 };
enum class AnalysisType : uint64_t {
AnalyzeNone = 0,
IdentifyBlobs = 0x01,
CopyPhaseIndicator = 0x02,
CopySimState = 0x04,
ComputeAverages = 0x08,
CreateRestart = 0x10,
WriteVis = 0x20,
ComputeSubphase = 0x40
};
//! Class to run the analysis in multiple threads
class runAnalysis
{
public:
//! Constructor
runAnalysis(std::shared_ptr<Database> db, const RankInfoStruct& rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr <Domain> dm, int Np, bool Regular, IntArray Map );
runAnalysis( std::shared_ptr<Database> db, const RankInfoStruct &rank_info,
std::shared_ptr<ScaLBL_Communicator> ScaLBL_Comm, std::shared_ptr<Domain> dm, int Np,
bool Regular, IntArray Map );
runAnalysis( ScaLBL_ColorModel &ColorModel);
//! Destructor
~runAnalysis();
//! Run the next analysis
void run(int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
void run( int timestep, std::shared_ptr<Database> db, TwoPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData(int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages, const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den);
void basic( int timestep, std::shared_ptr<Database> db, SubPhase &Averages, const double *Phi,
double *Pressure, double *Velocity, double *fq, double *Den );
void WriteVisData( int timestep, std::shared_ptr<Database> vis_db, SubPhase &Averages,
const double *Phi, double *Pressure, double *Velocity, double *fq, double *Den );
//! Finish all active analysis
void finish();
@ -60,7 +70,8 @@ public:
* \brief Set the affinities
* \details This function will create the analysis threads and set the affinity
* of this thread and all analysis threads. If MPI_THREAD_MULTIPLE is not
* enabled, the analysis threads will be disabled and the analysis will run in the current thread.
* enabled, the analysis threads will be disabled and the analysis will run in the current
* thread.
* @param[in] method Method used to control the affinities:
* none - Don't use threads (runs all analysis in the current thread)
* default - Create the specified number of threads, but don't load balance
@ -69,38 +80,36 @@ public:
* that all threads run on independent cores
* @param[in] N_threads Number of threads, only used by some of the methods
*/
void createThreads( const std::string& method = "default", int N_threads = 4 );
void createThreads( const std::string &method = "default", int N_threads = 4 );
private:
runAnalysis();
// Determine the analysis to perform
AnalysisType computeAnalysisType( int timestep );
public:
class commWrapper
{
public:
MPI_Comm comm;
public:
Utilities::MPI comm;
int tag;
runAnalysis *analysis;
commWrapper( int tag, MPI_Comm comm, runAnalysis *analysis );
commWrapper( ) = delete;
commWrapper( int tag, const Utilities::MPI &comm, runAnalysis *analysis );
commWrapper() = delete;
commWrapper( const commWrapper &rhs ) = delete;
commWrapper& operator=( const commWrapper &rhs ) = delete;
commWrapper &operator=( const commWrapper &rhs ) = delete;
commWrapper( commWrapper &&rhs );
~commWrapper();
};
// Get a comm (not thread safe)
commWrapper getComm( );
commWrapper getComm();
private:
int d_N[3];
std::array<int, 3> d_n; // Number of local cells
std::array<int, 3> d_N; // Number of local cells with ghosts
int d_Np;
int d_rank;
int d_restart_interval, d_analysis_interval, d_blobid_interval, d_visualization_interval;
@ -110,14 +119,13 @@ private:
ThreadPool d_tpool;
RankInfoStruct d_rank_info;
IntArray d_Map;
BlobIDstruct d_last_ids;
BlobIDstruct d_last_index;
BlobIDList d_last_id_map;
std::shared_ptr<std::pair<int, IntArray>> d_last_ids;
std::shared_ptr<std::pair<int, IntArray>> d_last_index;
std::shared_ptr<std::vector<BlobIDType>> d_last_id_map;
std::vector<IO::MeshDataStruct> d_meshData;
fillHalo<double> d_fillData;
std::string d_restartFile;
MPI_Comm d_comm;
MPI_Comm d_comms[1024];
Utilities::MPI d_comm;
Utilities::MPI d_comms[1024];
volatile bool d_comm_used[1024];
std::shared_ptr<ScaLBL_Communicator> d_ScaLBL_Comm;
@ -130,8 +138,6 @@ private:
// Friends
friend commWrapper::~commWrapper();
};
#endif

11
analysis/uCT.cpp
View File

@ -244,8 +244,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
Array<float>& Mean, Array<float>& Dist1, Array<float>& Dist2 )
{
PROFILE_SCOPED(timer,"filter_final");
int rank;
MPI_Comm_rank(Dm.Comm,&rank);
int rank = Dm.Comm.getRank();
int Nx = Dm.Nx-2;
int Ny = Dm.Ny-2;
int Nz = Dm.Nz-2;
@ -258,7 +257,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
float tmp = 0;
for (size_t i=0; i<Dist0.length(); i++)
tmp += Dist0(i)*Dist0(i);
tmp = sqrt( sumReduce(Dm.Comm,tmp) / sumReduce(Dm.Comm,(float)Dist0.length()) );
tmp = sqrt( Dm.Comm.sumReduce(tmp) / Dm.Comm.sumReduce<float>(Dist0.length()) );
const float dx1 = 0.3*tmp;
const float dx2 = 1.05*dx1;
if (rank==0)
@ -301,7 +300,7 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
Phase.fill(1);
ComputeGlobalBlobIDs( Nx, Ny, Nz, Dm.rank_info, Phase, SignDist, 0, 0, GlobalBlobID, Dm.Comm );
fillInt.fill(GlobalBlobID);
int N_blobs = maxReduce(Dm.Comm,GlobalBlobID.max()+1);
int N_blobs = Dm.Comm.maxReduce(GlobalBlobID.max()+1);
std::vector<float> mean(N_blobs,0);
std::vector<int> count(N_blobs,0);
for (int k=1; k<=Nz; k++) {
@ -337,8 +336,8 @@ void filter_final( Array<char>& ID, Array<float>& Dist,
}
}
}
mean = sumReduce(Dm.Comm,mean);
count = sumReduce(Dm.Comm,count);
mean = Dm.Comm.sumReduce(mean);
count = Dm.Comm.sumReduce(count);
for (size_t i=0; i<mean.size(); i++)
mean[i] /= count[i];
/*if (rank==0) {

362
cmake/FindMPI.cmake
View File

@ -1,362 +0,0 @@
# - Message Passing Interface (MPI) module.
#
# The Message Passing Interface (MPI) is a library used to write
# high-performance parallel applications that use message passing, and
# is typically deployed on a cluster. MPI is a standard interface
# (defined by the MPI forum) for which many implementations are
# available. All of these implementations have somewhat different
# compilation approaches (different include paths, libraries to link
# against, etc.), and this module tries to smooth out those differences.
#
# This module will set the following variables:
# MPI_FOUND TRUE if we have found MPI
# MPI_COMPILE_FLAGS Compilation flags for MPI programs
# MPI_INCLUDE_PATH Include path(s) for MPI header
# MPI_LINK_FLAGS Linking flags for MPI programs
# MPI_LIBRARY First MPI library to link against (cached)
# MPI_EXTRA_LIBRARY Extra MPI libraries to link against (cached)
# MPI_LIBRARIES All libraries to link MPI programs against
# MPIEXEC Executable for running MPI programs
# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving it the
# number of processors to run on
# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly before the
# executable to run.
# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after all other flags.
#
# This module will attempt to auto-detect these settings, first by
# looking for a MPI compiler, which many MPI implementations provide
# as a pass-through to the native compiler to simplify the compilation
# of MPI programs. The MPI compiler is stored in the cache variable
# MPI_COMPILER, and will attempt to look for commonly-named drivers
# mpic++, mpicxx, mpiCC, or mpicc. If the compiler driver is found and
# recognized, it will be used to set all of the module variables. To
# skip this auto-detection, set MPI_LIBRARY and MPI_INCLUDE_PATH in
# the CMake cache.
#
# If no compiler driver is found or the compiler driver is not
# recognized, this module will then search for common include paths
# and library names to try to detect MPI.
#
# If CMake initially finds a different MPI than was intended, and you
# want to use the MPI compiler auto-detection for a different MPI
# implementation, set MPI_COMPILER to the MPI compiler driver you want
# to use (e.g., mpicxx) and then set MPI_LIBRARY to the string
# MPI_LIBRARY-NOTFOUND. When you re-configure, auto-detection of MPI
# will run again with the newly-specified MPI_COMPILER.
#
# When using MPIEXEC to execute MPI applications, you should typically
# use all of the MPIEXEC flags as follows:
# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS ${MPIEXEC_PREFLAGS} EXECUTABLE
# ${MPIEXEC_POSTFLAGS} ARGS
# where PROCS is the number of processors on which to execute the program,
# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the
# MPI program.
#=============================================================================
# Copyright 2001-2009 Kitware, Inc.
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
# This module is maintained by David Partyka <dave.partyka@kitware.com>.
# A set of directories to search through in addition to the standard system paths
# that find_program will search through.
# Microsoft HPC SDK is automatically added to the system path
# Argonne National Labs MPICH2 sets a registry key that we can use.
set(_MPI_PACKAGE_DIR
mpi
mpich
openmpi
lib/mpi
lib/mpich
lib/openmpi
"MPICH/SDK"
"Microsoft Compute Cluster Pack"
"Microsoft HPC Pack 2008 R2"
)
set(_MPI_PREFIX_PATH)
if(WIN32)
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..")
list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]")
endif()
foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH})
foreach(MpiPackageDir ${_MPI_PREFIX_PATH})
if(EXISTS ${SystemPrefixDir}/${MpiPackageDir})
list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}")
endif()
endforeach(MpiPackageDir)
endforeach(SystemPrefixDir)
# Most mpi distros have some form of mpiexec which gives us something we can reliably look for.
find_program(MPIEXEC
NAMES mpiexec mpirun lamexec
PATHS ${_MPI_PREFIX_PATH}
PATH_SUFFIXES bin
DOC "Executable for running MPI programs."
)
# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin).
# This gives us a fairly reliable base directory to search for /bin /lib and /include from.
get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH)
get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH)
# If there is an mpi compiler find it and interogate (farther below) it for the include
# and lib dirs otherwise we will continue to search from ${_MPI_BASE_DIR}.
find_program(MPI_COMPILER
NAMES mpic++ mpicxx mpiCC mpicc
HINTS "${_MPI_BASE_DIR}"
PATH_SUFFIXES bin
DOC "MPI compiler. Used only to detect MPI compilation flags.")
mark_as_advanced(MPI_COMPILER)
set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.")
set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.")
set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.")
set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.")
mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS
MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILER)
# Check whether the -showme:compile option works. This indicates
# that we have either Open MPI or a newer version of LAM-MPI, and
# implies that -showme:link will also work.
# Note that Windows distros do not have an mpi compiler to interogate.
exec_program(${MPI_COMPILER}
ARGS -showme:compile
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
if (MPI_COMPILER_RETURN EQUAL 0)
# If we appear to have -showme:compile, then we should also have
# -showme:link. Try it.
exec_program(${MPI_COMPILER}
ARGS -showme:link
OUTPUT_VARIABLE MPI_LINK_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
# Note that we probably have -showme:incdirs and -showme:libdirs
# as well.
set(MPI_COMPILER_MAY_HAVE_INCLIBDIRS TRUE)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# Older versions of LAM-MPI have "-showme". Try it.
exec_program(${MPI_COMPILER}
ARGS -showme
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# Do nothing: we have our command lines now
else (MPI_COMPILER_RETURN EQUAL 0)
# MPICH uses "-show". Try it.
exec_program(${MPI_COMPILER}
ARGS -show
OUTPUT_VARIABLE MPI_COMPILE_CMDLINE
RETURN_VALUE MPI_COMPILER_RETURN)
endif (MPI_COMPILER_RETURN EQUAL 0)
if (MPI_COMPILER_RETURN EQUAL 0)
# We have our command lines, but we might need to copy
# MPI_COMPILE_CMDLINE into MPI_LINK_CMDLINE, if the underlying
if (NOT MPI_LINK_CMDLINE)
SET(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE})
endif (NOT MPI_LINK_CMDLINE)
else (MPI_COMPILER_RETURN EQUAL 0)
message(STATUS "Unable to determine MPI from MPI driver ${MPI_COMPILER}")
endif (MPI_COMPILER_RETURN EQUAL 0)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Do nothing: we already have MPI_INCLUDE_PATH and MPI_LIBRARY in
# the cache, and we don't want to override those settings.
elseif (MPI_COMPILE_CMDLINE)
# Extract compile flags from the compile command line.
string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}")
set(MPI_COMPILE_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_COMPILE_FLAGS})
if (MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}")
else(MPI_COMPILE_FLAGS_WORK)
set(MPI_COMPILE_FLAGS_WORK ${FLAG})
endif(MPI_COMPILE_FLAGS_WORK)
endforeach(FLAG)
# Extract include paths from compile command line
string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}")
set(MPI_INCLUDE_PATH_WORK)
foreach(IPATH ${MPI_ALL_INCLUDE_PATHS})
string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH})
string(REGEX REPLACE "//" "/" IPATH ${IPATH})
list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH})
endforeach(IPATH)
if (NOT MPI_INCLUDE_PATH_WORK)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any include paths on it,
# but we may have -showme:incdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:incdirs
OUTPUT_VARIABLE MPI_INCLUDE_PATH_WORK
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_INCLUDE_PATH_WORK)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_INCLUDE_PATH_WORK)
if (NOT MPI_INCLUDE_PATH_WORK)
# If all else fails, just search for mpi.h in the normal include
# paths.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include
)
set(MPI_INCLUDE_PATH_WORK ${MPI_INCLUDE_PATH})
endif (NOT MPI_INCLUDE_PATH_WORK)
# Extract linker paths from the link command line
string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_PATH)
foreach(LPATH ${MPI_ALL_LINK_PATHS})
string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH})
string(REGEX REPLACE "//" "/" LPATH ${LPATH})
list(APPEND MPI_LINK_PATH ${LPATH})
endforeach(LPATH)
if (NOT MPI_LINK_PATH)
if (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
# The compile command line didn't have any linking paths on it,
# but we may have -showme:libdirs. Use it.
exec_program(${MPI_COMPILER}
ARGS -showme:libdirs
OUTPUT_VARIABLE MPI_LINK_PATH
RETURN_VALUE MPI_COMPILER_RETURN)
separate_arguments(MPI_LINK_PATH)
endif (MPI_COMPILER_MAY_HAVE_INCLIBDIRS)
endif (NOT MPI_LINK_PATH)
# Extract linker flags from the link command line
string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}")
set(MPI_LINK_FLAGS_WORK)
foreach(FLAG ${MPI_ALL_LINK_FLAGS})
if (MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}")
else(MPI_LINK_FLAGS_WORK)
set(MPI_LINK_FLAGS_WORK ${FLAG})
endif(MPI_LINK_FLAGS_WORK)
endforeach(FLAG)
if ( MPI_LINK_FLAGS_WORK )
string ( REGEX REPLACE "^ " "" MPI_LINK_FLAGS_WORK ${MPI_LINK_FLAGS_WORK} )
endif ()
# Extract the set of libraries to link against from the link command
# line
string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}")
# Determine full path names for all of the libraries that one needs
# to link against in an MPI program
set(MPI_LIBRARIES)
foreach(LIB ${MPI_LIBNAMES})
string(REGEX REPLACE "^ ?-l" "" LIB ${LIB})
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE)
find_library(MPI_LIB ${LIB} HINTS ${MPI_LINK_PATH})
if (MPI_LIB)
list(APPEND MPI_LIBRARIES ${MPI_LIB})
elseif (NOT MPI_FIND_QUIETLY)
message(WARNING "Unable to find MPI library ${LIB}")
endif ()
endforeach(LIB)
set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI detection" FORCE)
# Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and
# MPI_EXTRA_LIBRARY.
list(LENGTH MPI_LIBRARIES MPI_NUMLIBS)
list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK)
set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE)
else (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE)
endif (MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED)
if (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES})
list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0)
set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE)
else (MPI_NUMLIBS GREATER 1)
set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE)
endif (MPI_NUMLIBS GREATER 1)
# Set up all of the appropriate cache entries
set(MPI_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI compilation flags" FORCE)
set(MPI_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI include path" FORCE)
set(MPI_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI linking flags" FORCE)
else (MPI_COMPILE_CMDLINE)
# No MPI compiler to interogate so attempt to find everything with find functions.
find_path(MPI_INCLUDE_PATH mpi.h
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES include Inc
)
# Decide between 32-bit and 64-bit libraries for Microsoft's MPI
if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8)
set(MS_MPI_ARCH_DIR amd64)
else()
set(MS_MPI_ARCH_DIR i386)
endif()
find_library(MPI_LIBRARY
NAMES mpi mpich msmpi
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}
)
find_library(MPI_EXTRA_LIBRARY
NAMES mpi++
HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH}
PATH_SUFFIXES lib
DOC "Extra MPI libraries to link against.")
set(MPI_COMPILE_FLAGS "" CACHE STRING "MPI compilation flags")
set(MPI_LINK_FLAGS "" CACHE STRING "MPI linking flags")
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
# Set up extra variables to conform to
if (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY})
else (MPI_EXTRA_LIBRARY)
set(MPI_LIBRARIES ${MPI_LIBRARY})
endif (MPI_EXTRA_LIBRARY)
if (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND TRUE)
else (MPI_INCLUDE_PATH AND MPI_LIBRARY)
set(MPI_FOUND FALSE)
endif (MPI_INCLUDE_PATH AND MPI_LIBRARY)
#include("${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake")
# handle the QUIETLY and REQUIRED arguments
#find_package_handle_standard_args(MPI DEFAULT_MSG MPI_LIBRARY MPI_INCLUDE_PATH)
mark_as_advanced(MPI_INCLUDE_PATH MPI_COMPILE_FLAGS MPI_LINK_FLAGS MPI_LIBRARY
MPI_EXTRA_LIBRARY)
# unset to cleanup namespace
unset(_MPI_PACKAGE_DIR)
unset(_MPI_PREFIX_PATH)
unset(_MPI_BASE_DIR)

41
cmake/Find_TIMER.cmake
View File

@ -4,7 +4,7 @@
# CONFIGURE_TIMER( DEFAULT_USE_TIMER NULL_TIMER_DIR )
# This function assumes that USE_TIMER is set to indicate if the timer should be used
# If USE_TIMER is set, TIMER_DIRECTORY specifies the install path for the timer
# If USE_TIMER is not set we will create a summy timer that does nothing.
# If USE_TIMER is not set we will create a dummy timer that does nothing.
# The input argument DEFAULT_USE_TIMER specifies if the timer library is included by default.
# The input argument NULL_TIMER_DIR specifies the location to install the dummy timer.
# If it is an empty string, the default install path "${CMAKE_CURRENT_BINARY_DIR}/null_timer" is used.
@ -13,7 +13,7 @@
# TIMER_CXXFLAGS - C++ flags for the timer library
# TIMER_LDFLAGS - Linker flags to link the timer library
# TIMER_LDLIBS - Linker libraries to link the timer library
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR QUIET )
# Determine if we want to use the timer utility
CHECK_ENABLE_FLAG( USE_TIMER ${DEFAULT_USE_TIMER} )
SET( TIMER_INCLUDE )
@ -33,20 +33,23 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FIND_LIBRARY( TIMER_LIBS NAMES timerutility PATHS ${TIMER_DIRECTORY}/lib NO_DEFAULT_PATH )
SET( TIMER_INCLUDE ${TIMER_DIRECTORY}/include )
SET( TIMER_CXXFLAGS "-DUSE_TIMER -I${TIMER_DIRECTORY}/include" )
SET( TIMER_LDFLAGS -L${TIMER_DIRECTORY}/lib )
SET( TIMER_LDLIBS -ltimerutility )
SET( TIMER_LDFLAGS )
SET( TIMER_LDLIBS "${TIMER_LIBS}" )
ELSE()
MESSAGE( FATAL_ERROR "Default search for TIMER is not yet supported. Use -D TIMER_DIRECTORY=" )
ENDIF()
SET(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
SET( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} "${TIMER_DIRECTORY}/lib" PARENT_SCOPE )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
ADD_DEFINITIONS( -DUSE_TIMER )
MESSAGE( "Using timer utility" )
MESSAGE( " TIMER_LIBRARIES = ${TIMER_LIBS}" )
IF ( NOT QUIET )
MESSAGE( STATUS "Using timer utility" )
MESSAGE( STATUS " TIMER_LIBRARIES = ${TIMER_LIBS}" )
ENDIF()
ELSE()
IF ( "${NULL_TIMER_DIR}" STREQUAL "" )
SET( NULL_TIMER_DIR "${CMAKE_CURRENT_BINARY_DIR}/null_timer" )
ENDIF()
# Write ProfilerApp.h
FILE(WRITE "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_STOP(...) do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_START2(...) do {} while(0)\n" )
@ -61,9 +64,25 @@ FUNCTION( CONFIGURE_TIMER DEFAULT_USE_TIMER NULL_TIMER_DIR )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_TRACE() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_ENABLE_MEMORY() do {} while(0)\n" )
FILE(APPEND "${NULL_TIMER_DIR}/ProfilerApp.h" "#define PROFILE_DISABLE_MEMORY() do {} while(0)\n" )
# Write MemoryApp.h
FILE(WRITE "${NULL_TIMER_DIR}/MemoryApp.h" "#include <cstring>\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "class MemoryApp final {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "public:\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " struct MemoryStats {\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " size_t bytes_new, bytes_delete, N_new, N_delete, tot_bytes_used, system_memory, stack_used, stack_size;\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " MemoryStats() { memset(this,0,sizeof(MemoryStats)); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " };\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline void print( std::ostream& ) {}\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getTotalMemoryUsage() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline size_t getSystemMemory() { return 0; }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" " static inline MemoryStats getMemoryStats() { return MemoryStats(); }\n" )
FILE(APPEND "${NULL_TIMER_DIR}/MemoryApp.h" "};\n" )
SET( TIMER_INCLUDE "${NULL_TIMER_DIR}" )
INCLUDE_DIRECTORIES( "${TIMER_INCLUDE}" )
MESSAGE( "Disabling timer utility" )
IF ( NOT QUIET )
MESSAGE( STATUS "Disabling timer utility" )
ENDIF()
ENDIF()
SET( TIMER_INCLUDE "${TIMER_INCLUDE}" PARENT_SCOPE )
SET( TIMER_CXXFLAGS "${TIMER_CXXFLAGS}" PARENT_SCOPE )
@ -88,12 +107,12 @@ MACRO( CHECK_ENABLE_FLAG FLAG DEFAULT )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ${FLAG} STREQUAL "" )
SET( ${FLAG} ${DEFAULT} )
ELSEIF( ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "FALSE" ) OR ( ${${FLAG}} STREQUAL "false" ) OR ( ${${FLAG}} STREQUAL "0" ) OR ( ${${FLAG}} STREQUAL "OFF" ) )
SET( ${FLAG} 0 )
ELSEIF( ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
ELSEIF( ( ${${FLAG}} STREQUAL "TRUE" ) OR ( ${${FLAG}} STREQUAL "true" ) OR ( ${${FLAG}} STREQUAL "1" ) OR ( ${${FLAG}} STREQUAL "ON" ) )
SET( ${FLAG} 1 )
ELSE()
MESSAGE( "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
MESSAGE( FATAL_ERROR "Bad value for ${FLAG} (${${FLAG}}); use true or false" )
ENDIF ()
ENDMACRO()

3
cmake/ctest_script.cmake
View File

@ -32,7 +32,6 @@ SET( CMAKE_MAKE_PROGRAM $ENV{CMAKE_MAKE_PROGRAM} )
SET( CTEST_CMAKE_GENERATOR $ENV{CTEST_CMAKE_GENERATOR} )
SET( LDLIBS $ENV{LDLIBS} )
SET( LDFLAGS $ENV{LDFLAGS} )
SET( MPI_COMPILER $ENV{MPI_COMPILER} )
SET( MPI_DIRECTORY $ENV{MPI_DIRECTORY} )
SET( MPI_INCLUDE $ENV{MPI_INCLUDE} )
SET( MPI_LINK_FLAGS $ENV{MPI_LINK_FLAGS} )
@ -198,7 +197,7 @@ SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_FLAGS='${CFLAGS}';-DCMAKE_CXX_FLA
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DLDFLAGS:STRING='${FLAGS}';-DLDLIBS:STRING='${LDLIBS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_GCOV:BOOL=${ENABLE_GCOV}" )
IF ( USE_MPI )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC}")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC}")
IF ( NOT USE_VALGRIND )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_MPI_FOR_SERIAL_TESTS:BOOL=true")
ENDIF()

132
cmake/libraries.cmake
View File

@ -41,93 +41,61 @@ ENDMACRO()
# Macro to find and configure the MPI libraries
MACRO( CONFIGURE_MPI )
# Determine if we want to use MPI
CHECK_ENABLE_FLAG(USE_MPI 1 )
CHECK_ENABLE_FLAG( USE_MPI 1 )
IF ( USE_MPI )
# Check if we specified the MPI directory
IF ( MPI_DIRECTORY )
# Check the provided MPI directory for include files
VERIFY_PATH( "${MPI_DIRECTORY}" )
IF ( EXISTS "${MPI_DIRECTORY}/include/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/include" )
ELSEIF ( EXISTS "${MPI_DIRECTORY}/Inc/mpi.h" )
SET( MPI_INCLUDE_PATH "${MPI_DIRECTORY}/Inc" )
ELSE()
MESSAGE( FATAL_ERROR "mpi.h not found in ${MPI_DIRECTORY}/include" )
ENDIF ()
INCLUDE_DIRECTORIES ( ${MPI_INCLUDE_PATH} )
SET ( MPI_INCLUDE ${MPI_INCLUDE_PATH} )
# Set MPI libraries
IF ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPI_LIB NAMES msmpi PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFEC_LIB NAMES msmpifec PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/x64" NO_DEFAULT_PATH )
FIND_LIBRARY( MSMPIFMC_LIB NAMES msmpifmc PATHS "${MPI_DIRECTORY}/Lib/amd64" NO_DEFAULT_PATH )
SET( MPI_LIBRARIES ${MSMPI_LIB} ${MSMPIFEC_LIB} ${MSMPIFMC_LIB} )
ENDIF()
# Set the mpi executable
IF ( MPIEXEC )
# User specified the MPI command directly, use as is
ELSEIF ( MPIEXEC_CMD )
# User specified the name of the MPI executable
SET ( MPIEXEC ${MPI_DIRECTORY}/bin/${MPIEXEC_CMD} )
IF ( NOT EXISTS ${MPIEXEC} )
MESSAGE( FATAL_ERROR "${MPIEXEC_CMD} not found in ${MPI_DIRECTORY}/bin" )
ENDIF ()
ELSE ()
# Search for the MPI executable in the current directory
FIND_PROGRAM( MPIEXEC NAMES mpiexec mpirun lamexec PATHS ${MPI_DIRECTORY}/bin NO_DEFAULT_PATH )
IF ( NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Could not locate mpi executable" )
ENDIF()
ENDIF ()
# Set MPI flags
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
ENDIF()
ELSEIF ( MPI_COMPILER )
# The mpi compiler should take care of everything
IF ( MPI_INCLUDE )
INCLUDE_DIRECTORIES( ${MPI_INCLUDE} )
ENDIF()
MESSAGE( "Configuring MPI" )
IF ( MPIEXEC )
SET( MPIEXEC_EXECUTABLE ${MPIEXEC} )
ENDIF()
IF ( NOT MPI_SKIP_SEARCH )
FIND_PACKAGE( MPI )
ELSE()
# Perform the default search for MPI
INCLUDE ( FindMPI )
IF ( NOT MPI_FOUND )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
MESSAGE( FATAL_ERROR "Did not find MPI" )
ENDIF ()
INCLUDE_DIRECTORIES( "${MPI_INCLUDE_PATH}" )
SET( MPI_INCLUDE "${MPI_INCLUDE_PATH}" )
# Write mpi test
SET( MPI_TEST_SRC "${CMAKE_CURRENT_BINARY_DIR}/test_mpi.cpp" )
FILE(WRITE ${MPI_TEST_SRC} "#include <mpi.h>\n" )
FILE(APPEND ${MPI_TEST_SRC} "int main(int argc, char** argv) {\n" )
FILE(APPEND ${MPI_TEST_SRC} " MPI_Init(NULL, NULL);\n")
FILE(APPEND ${MPI_TEST_SRC} " MPI_Finalize();\n" )
FILE(APPEND ${MPI_TEST_SRC} "}\n" )
# Test the compile
IF ( CMAKE_CXX_COMPILER )
SET( TMP_FLAGS -DINCLUDE_DIRECTORIES=${MPI_CXX_INCLUDE_PATH} )
TRY_COMPILE( MPI_TEST_CXX ${CMAKE_CURRENT_BINARY_DIR} ${MPI_TEST_SRC}
CMAKE_FLAGS ${TMP_FLAGS}
LINK_OPTIONS ${MPI_CXX_LINK_FLAGS}
LINK_LIBRARIES ${MPI_CXX_LIBRARIES}
OUTPUT_VARIABLE OUT_TXT)
IF ( NOT ${MPI_TEST} )
MESSAGE( FATAL_ERROR "Skipping MPI search and default compile fails:\n${OUT_TXT}" )
ENDIF()
SET( MPI_C_FOUND TRUE )
SET( MPI_CXX_FOUND TRUE )
SET( MPI_Fortran_FOUND TRUE )
ENDIF()
ENDIF()
# Check if we need to use MPI for serial tests
CHECK_ENABLE_FLAG( USE_MPI_FOR_SERIAL_TESTS 0 )
# Set defaults if they have not been set
IF ( NOT MPIEXEC )
SET( MPIEXEC mpirun )
STRING( STRIP "${MPI_CXX_COMPILE_FLAGS}" MPI_CXX_COMPILE_FLAGS )
STRING( STRIP "${MPI_CXX_LINK_FLAGS}" MPI_CXX_LINK_FLAGS )
STRING( STRIP "${MPI_CXX_LIBRARIES}" MPI_CXX_LIBRARIES )
MESSAGE( " MPI_CXX_FOUND = ${MPI_CXX_FOUND}" )
MESSAGE( " MPI_CXX_COMPILER = ${MPI_CXX_COMPILER}" )
MESSAGE( " MPI_CXX_COMPILE_FLAGS = ${MPI_CXX_COMPILE_FLAGS}" )
MESSAGE( " MPI_CXX_INCLUDE_PATH = ${MPI_CXX_INCLUDE_PATH}" )
MESSAGE( " MPI_CXX_LINK_FLAGS = ${MPI_CXX_LINK_FLAGS}" )
MESSAGE( " MPI_CXX_LIBRARIES = ${MPI_CXX_LIBRARIES}" )
MESSAGE( " MPIEXEC = ${MPIEXEC}" )
MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
MESSAGE( " MPIEXEC_PREFLAGS = ${MPIEXEC_PREFLAGS}" )
MESSAGE( " MPIEXEC_POSTFLAGS = ${MPIEXEC_POSTFLAGS}" )
ADD_DEFINITIONS( -DUSE_MPI )
INCLUDE_DIRECTORIES( ${MPI_CXX_INCLUDE_PATH} )
SET( MPI_LIBRARIES ${MPI_CXX_LIBRARIES} )
SET( MPI_LINK_FLAGS ${MPI_CXX_LINK_FLAGS} )
IF ( NOT MPI_CXX_FOUND )
MESSAGE( FATAL_ERROR "MPI not found" )
ENDIF()
IF ( NOT MPIEXEC_NUMPROC_FLAG )
SET( MPIEXEC_NUMPROC_FLAG "-np" )
IF ( USE_MPI AND NOT MPIEXEC )
MESSAGE( FATAL_ERROR "Unable to find MPIEXEC, please set it before continuing" )
ENDIF()
# Set the definitions
ADD_DEFINITIONS( "-DUSE_MPI" )
MESSAGE( "Using MPI" )
MESSAGE( " MPIEXEC = ${MPIEXEC}" )
MESSAGE( " MPIEXEC_NUMPROC_FLAG = ${MPIEXEC_NUMPROC_FLAG}" )
MESSAGE( " MPI_INCLUDE = ${MPI_INCLUDE}" )
MESSAGE( " MPI_LINK_FLAGS = ${MPI_LINK_FLAGS}" )
MESSAGE( " MPI_LIBRARIES = ${MPI_LIBRARIES}" )
ELSE()
SET( USE_MPI_FOR_SERIAL_TESTS 0 )
SET( MPIEXEC "" )
SET( MPIEXEC_NUMPROC_FLAG "" )
SET( MPI_INCLUDE "" )
SET( MPI_LINK_FLAGS "" )
SET( MPI_LIBRARIES "" )
MESSAGE( "Not using MPI, all parallel tests will be disabled" )
ENDIF()
ENDMACRO()

4
cmake/macros.cmake
View File

@ -681,8 +681,8 @@ MACRO( TARGET_LINK_EXTERNAL_LIBRARIES TARGET_NAME )
FOREACH ( tmp ${BLAS_LAPACK_LIBS} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( MPI_LIBRARIES )
TARGET_LINK_LIBRARIES( ${EXE} ${ARGN} ${tmp} )
FOREACH ( tmp ${MPI_LIBRARIES} )
TARGET_LINK_LIBRARIES( ${TARGET_NAME} ${ARGN} ${tmp} )
ENDFOREACH()
FOREACH ( tmp ${CMAKE_C_IMPLICIT_LINK_LIBRARIES}
${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES} ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES} )

237
common/Communication.h
View File

@ -17,7 +17,7 @@
#ifndef COMMUNICATION_H_INC
#define COMMUNICATION_H_INC
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
#include "common/Array.h"
@ -54,7 +54,7 @@ struct RankInfoStruct {
//! Redistribute domain data (dst may be smaller than the src)
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm );
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm );
/*!
@ -75,7 +75,7 @@ public:
* @param[in] fill Fill {faces,edges,corners}
* @param[in] periodic Periodic dimensions
*/
fillHalo( MPI_Comm comm, const RankInfoStruct& info,
fillHalo( const Utilities::MPI& comm, const RankInfoStruct& info,
std::array<int,3> n, std::array<int,3> ng, int tag, int depth,
std::array<bool,3> fill = {true,true,true},
std::array<bool,3> periodic = {true,true,true} );
@ -83,6 +83,10 @@ public:
//! Destructor
~fillHalo( );
fillHalo() = delete;
fillHalo(const fillHalo&) = delete;
fillHalo& operator=(const fillHalo&) = delete;
/*!
* @brief Communicate the halos
* @param[in] array The array on which we fill the halos
@ -99,7 +103,7 @@ public:
private:
MPI_Comm comm;
Utilities::MPI comm;
RankInfoStruct info;
std::array<int,3> n, ng;
int depth;
@ -109,18 +113,13 @@ private:
TYPE *mem;
TYPE *send[3][3][3], *recv[3][3][3];
MPI_Request send_req[3][3][3], recv_req[3][3][3];
size_t N_type;
MPI_Datatype datatype;
fillHalo(); // Private empty constructor
fillHalo(const fillHalo&); // Private copy constructor
fillHalo& operator=(const fillHalo&); // Private assignment operator
void pack( const Array<TYPE>& array, int i, int j, int k, TYPE *buffer );
void unpack( Array<TYPE>& array, int i, int j, int k, const TYPE *buffer );
};
//***************************************************************************************
inline void PackMeshData(int *list, int count, double *sendbuf, double *data){
inline void PackMeshData(const int *list, int count, double *sendbuf, double *data){
// Fill in the phase ID values from neighboring processors
// This packs up the values that need to be sent from one processor to another
int idx,n;
@ -129,7 +128,7 @@ inline void PackMeshData(int *list, int count, double *sendbuf, double *data){
sendbuf[idx] = data[n];
}
}
inline void UnpackMeshData(int *list, int count, double *recvbuf, double *data){
inline void UnpackMeshData(const int *list, int count, double *recvbuf, double *data){
// Fill in the phase ID values from neighboring processors
// This unpacks the values once they have been recieved from neighbors
int idx,n;
@ -152,7 +151,7 @@ void InitializeRanks( const int rank, const int nprocx, const int nprocy, const
//***************************************************************************************
inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateSendRecvCounts( const Utilities::MPI& comm, int sendtag, int recvtag,
int rank_x, int rank_y, int rank_z,
int rank_X, int rank_Y, int rank_Z,
int rank_xy, int rank_XY, int rank_xY, int rank_Xy,
@ -170,54 +169,53 @@ inline void CommunicateSendRecvCounts( MPI_Comm Communicator, int sendtag, int r
int& recvCount_yz, int& recvCount_YZ, int& recvCount_yZ, int& recvCount_Yz )
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(&sendCount_x, 1,MPI_INT,rank_x,sendtag+0,Communicator,&req1[0]);
MPI_Irecv(&recvCount_X, 1,MPI_INT,rank_X,recvtag+0,Communicator,&req2[0]);
MPI_Isend(&sendCount_X, 1,MPI_INT,rank_X,sendtag+1,Communicator,&req1[1]);
MPI_Irecv(&recvCount_x, 1,MPI_INT,rank_x,recvtag+1,Communicator,&req2[1]);
MPI_Isend(&sendCount_y, 1,MPI_INT,rank_y,sendtag+2,Communicator,&req1[2]);
MPI_Irecv(&recvCount_Y, 1,MPI_INT,rank_Y,recvtag+2,Communicator,&req2[2]);
MPI_Isend(&sendCount_Y, 1,MPI_INT,rank_Y,sendtag+3,Communicator,&req1[3]);
MPI_Irecv(&recvCount_y, 1,MPI_INT,rank_y,recvtag+3,Communicator,&req2[3]);
MPI_Isend(&sendCount_z, 1,MPI_INT,rank_z,sendtag+4,Communicator,&req1[4]);
MPI_Irecv(&recvCount_Z, 1,MPI_INT,rank_Z,recvtag+4,Communicator,&req2[4]);
MPI_Isend(&sendCount_Z, 1,MPI_INT,rank_Z,sendtag+5,Communicator,&req1[5]);
MPI_Irecv(&recvCount_z, 1,MPI_INT,rank_z,recvtag+5,Communicator,&req2[5]);
req1[0] = comm.Isend(&sendCount_x,1,rank_x,sendtag+0);
req2[0] = comm.Irecv(&recvCount_X,1,rank_X,recvtag+0);
req1[1] = comm.Isend(&sendCount_X,1,rank_X,sendtag+1);
req2[1] = comm.Irecv(&recvCount_x,1,rank_x,recvtag+1);
req1[2] = comm.Isend(&sendCount_y,1,rank_y,sendtag+2);
req2[2] = comm.Irecv(&recvCount_Y,1,rank_Y,recvtag+2);
req1[3] = comm.Isend(&sendCount_Y,1,rank_Y,sendtag+3);
req2[3] = comm.Irecv(&recvCount_y,1,rank_y,recvtag+3);
req1[4] = comm.Isend(&sendCount_z,1,rank_z,sendtag+4);
req2[4] = comm.Irecv(&recvCount_Z,1,rank_Z,recvtag+4);
req1[5] = comm.Isend(&sendCount_Z,1,rank_Z,sendtag+5);
req2[5] = comm.Irecv(&recvCount_z,1,rank_z,recvtag+5);
MPI_Isend(&sendCount_xy, 1,MPI_INT,rank_xy,sendtag+6,Communicator,&req1[6]);
MPI_Irecv(&recvCount_XY, 1,MPI_INT,rank_XY,recvtag+6,Communicator,&req2[6]);
MPI_Isend(&sendCount_XY, 1,MPI_INT,rank_XY,sendtag+7,Communicator,&req1[7]);
MPI_Irecv(&recvCount_xy, 1,MPI_INT,rank_xy,recvtag+7,Communicator,&req2[7]);
MPI_Isend(&sendCount_Xy, 1,MPI_INT,rank_Xy,sendtag+8,Communicator,&req1[8]);
MPI_Irecv(&recvCount_xY, 1,MPI_INT,rank_xY,recvtag+8,Communicator,&req2[8]);
MPI_Isend(&sendCount_xY, 1,MPI_INT,rank_xY,sendtag+9,Communicator,&req1[9]);
MPI_Irecv(&recvCount_Xy, 1,MPI_INT,rank_Xy,recvtag+9,Communicator,&req2[9]);
req1[6] = comm.Isend(&sendCount_xy,1,rank_xy,sendtag+6);
req2[6] = comm.Irecv(&recvCount_XY,1,rank_XY,recvtag+6);
req1[7] = comm.Isend(&sendCount_XY,1,rank_XY,sendtag+7);
req2[7] = comm.Irecv(&recvCount_xy,1,rank_xy,recvtag+7);
req1[8] = comm.Isend(&sendCount_Xy,1,rank_Xy,sendtag+8);
req2[8] = comm.Irecv(&recvCount_xY,1,rank_xY,recvtag+8);
req1[9] = comm.Isend(&sendCount_xY,1,rank_xY,sendtag+9);
req2[9] = comm.Irecv(&recvCount_Xy,1,rank_Xy,recvtag+9);
MPI_Isend(&sendCount_xz, 1,MPI_INT,rank_xz,sendtag+10,Communicator,&req1[10]);
MPI_Irecv(&recvCount_XZ, 1,MPI_INT,rank_XZ,recvtag+10,Communicator,&req2[10]);
MPI_Isend(&sendCount_XZ, 1,MPI_INT,rank_XZ,sendtag+11,Communicator,&req1[11]);
MPI_Irecv(&recvCount_xz, 1,MPI_INT,rank_xz,recvtag+11,Communicator,&req2[11]);
MPI_Isend(&sendCount_Xz, 1,MPI_INT,rank_Xz,sendtag+12,Communicator,&req1[12]);
MPI_Irecv(&recvCount_xZ, 1,MPI_INT,rank_xZ,recvtag+12,Communicator,&req2[12]);
MPI_Isend(&sendCount_xZ, 1,MPI_INT,rank_xZ,sendtag+13,Communicator,&req1[13]);
MPI_Irecv(&recvCount_Xz, 1,MPI_INT,rank_Xz,recvtag+13,Communicator,&req2[13]);
req1[10] = comm.Isend(&sendCount_xz,1,rank_xz,sendtag+10);
req2[10] = comm.Irecv(&recvCount_XZ,1,rank_XZ,recvtag+10);
req1[11] = comm.Isend(&sendCount_XZ,1,rank_XZ,sendtag+11);
req2[11] = comm.Irecv(&recvCount_xz,1,rank_xz,recvtag+11);
req1[12] = comm.Isend(&sendCount_Xz,1,rank_Xz,sendtag+12);
req2[12] = comm.Irecv(&recvCount_xZ,1,rank_xZ,recvtag+12);
req1[13] = comm.Isend(&sendCount_xZ,1,rank_xZ,sendtag+13);
req2[13] = comm.Irecv(&recvCount_Xz,1,rank_Xz,recvtag+13);
MPI_Isend(&sendCount_yz, 1,MPI_INT,rank_yz,sendtag+14,Communicator,&req1[14]);
MPI_Irecv(&recvCount_YZ, 1,MPI_INT,rank_YZ,recvtag+14,Communicator,&req2[14]);
MPI_Isend(&sendCount_YZ, 1,MPI_INT,rank_YZ,sendtag+15,Communicator,&req1[15]);
MPI_Irecv(&recvCount_yz, 1,MPI_INT,rank_yz,recvtag+15,Communicator,&req2[15]);
MPI_Isend(&sendCount_Yz, 1,MPI_INT,rank_Yz,sendtag+16,Communicator,&req1[16]);
MPI_Irecv(&recvCount_yZ, 1,MPI_INT,rank_yZ,recvtag+16,Communicator,&req2[16]);
MPI_Isend(&sendCount_yZ, 1,MPI_INT,rank_yZ,sendtag+17,Communicator,&req1[17]);
MPI_Irecv(&recvCount_Yz, 1,MPI_INT,rank_Yz,recvtag+17,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
MPI_Barrier(Communicator);
req1[14] = comm.Isend(&sendCount_yz,1,rank_yz,sendtag+14);
req2[14] = comm.Irecv(&recvCount_YZ,1,rank_YZ,recvtag+14);
req1[15] = comm.Isend(&sendCount_YZ,1,rank_YZ,sendtag+15);
req2[15] = comm.Irecv(&recvCount_yz,1,rank_yz,recvtag+15);
req1[16] = comm.Isend(&sendCount_Yz,1,rank_Yz,sendtag+16);
req2[16] = comm.Irecv(&recvCount_yZ,1,rank_yZ,recvtag+16);
req1[17] = comm.Isend(&sendCount_yZ,1,rank_yZ,sendtag+17);
req2[17] = comm.Irecv(&recvCount_Yz,1,rank_Yz,recvtag+17);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
comm.barrier();
}
//***************************************************************************************
inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvtag,
inline void CommunicateRecvLists( const Utilities::MPI& comm, int sendtag, int recvtag,
int *sendList_x, int *sendList_y, int *sendList_z, int *sendList_X, int *sendList_Y, int *sendList_Z,
int *sendList_xy, int *sendList_XY, int *sendList_xY, int *sendList_Xy,
int *sendList_xz, int *sendList_XZ, int *sendList_xZ, int *sendList_Xz,
@ -238,53 +236,52 @@ inline void CommunicateRecvLists( MPI_Comm Communicator, int sendtag, int recvta
int rank_Xy, int rank_xz, int rank_XZ, int rank_xZ, int rank_Xz, int rank_yz, int rank_YZ, int rank_yZ, int rank_Yz)
{
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
MPI_Isend(sendList_x, sendCount_x,MPI_INT,rank_x,sendtag,Communicator,&req1[0]);
MPI_Irecv(recvList_X, recvCount_X,MPI_INT,rank_X,recvtag,Communicator,&req2[0]);
MPI_Isend(sendList_X, sendCount_X,MPI_INT,rank_X,sendtag,Communicator,&req1[1]);
MPI_Irecv(recvList_x, recvCount_x,MPI_INT,rank_x,recvtag,Communicator,&req2[1]);
MPI_Isend(sendList_y, sendCount_y,MPI_INT,rank_y,sendtag,Communicator,&req1[2]);
MPI_Irecv(recvList_Y, recvCount_Y,MPI_INT,rank_Y,recvtag,Communicator,&req2[2]);
MPI_Isend(sendList_Y, sendCount_Y,MPI_INT,rank_Y,sendtag,Communicator,&req1[3]);
MPI_Irecv(recvList_y, recvCount_y,MPI_INT,rank_y,recvtag,Communicator,&req2[3]);
MPI_Isend(sendList_z, sendCount_z,MPI_INT,rank_z,sendtag,Communicator,&req1[4]);
MPI_Irecv(recvList_Z, recvCount_Z,MPI_INT,rank_Z,recvtag,Communicator,&req2[4]);
MPI_Isend(sendList_Z, sendCount_Z,MPI_INT,rank_Z,sendtag,Communicator,&req1[5]);
MPI_Irecv(recvList_z, recvCount_z,MPI_INT,rank_z,recvtag,Communicator,&req2[5]);
req1[0] = comm.Isend(sendList_x,sendCount_x,rank_x,sendtag);
req2[0] = comm.Irecv(recvList_X,recvCount_X,rank_X,recvtag);
req1[1] = comm.Isend(sendList_X,sendCount_X,rank_X,sendtag);
req2[1] = comm.Irecv(recvList_x,recvCount_x,rank_x,recvtag);
req1[2] = comm.Isend(sendList_y,sendCount_y,rank_y,sendtag);
req2[2] = comm.Irecv(recvList_Y,recvCount_Y,rank_Y,recvtag);
req1[3] = comm.Isend(sendList_Y,sendCount_Y,rank_Y,sendtag);
req2[3] = comm.Irecv(recvList_y,recvCount_y,rank_y,recvtag);
req1[4] = comm.Isend(sendList_z,sendCount_z,rank_z,sendtag);
req2[4] = comm.Irecv(recvList_Z,recvCount_Z,rank_Z,recvtag);
req1[5] = comm.Isend(sendList_Z,sendCount_Z,rank_Z,sendtag);
req2[5] = comm.Irecv(recvList_z,recvCount_z,rank_z,recvtag);
MPI_Isend(sendList_xy, sendCount_xy,MPI_INT,rank_xy,sendtag,Communicator,&req1[6]);
MPI_Irecv(recvList_XY, recvCount_XY,MPI_INT,rank_XY,recvtag,Communicator,&req2[6]);
MPI_Isend(sendList_XY, sendCount_XY,MPI_INT,rank_XY,sendtag,Communicator,&req1[7]);
MPI_Irecv(recvList_xy, recvCount_xy,MPI_INT,rank_xy,recvtag,Communicator,&req2[7]);
MPI_Isend(sendList_Xy, sendCount_Xy,MPI_INT,rank_Xy,sendtag,Communicator,&req1[8]);
MPI_Irecv(recvList_xY, recvCount_xY,MPI_INT,rank_xY,recvtag,Communicator,&req2[8]);
MPI_Isend(sendList_xY, sendCount_xY,MPI_INT,rank_xY,sendtag,Communicator,&req1[9]);
MPI_Irecv(recvList_Xy, recvCount_Xy,MPI_INT,rank_Xy,recvtag,Communicator,&req2[9]);
req1[6] = comm.Isend(sendList_xy,sendCount_xy,rank_xy,sendtag);
req2[6] = comm.Irecv(recvList_XY,recvCount_XY,rank_XY,recvtag);
req1[7] = comm.Isend(sendList_XY,sendCount_XY,rank_XY,sendtag);
req2[7] = comm.Irecv(recvList_xy,recvCount_xy,rank_xy,recvtag);
req1[8] = comm.Isend(sendList_Xy,sendCount_Xy,rank_Xy,sendtag);
req2[8] = comm.Irecv(recvList_xY,recvCount_xY,rank_xY,recvtag);
req1[9] = comm.Isend(sendList_xY,sendCount_xY,rank_xY,sendtag);
req2[9] = comm.Irecv(recvList_Xy,recvCount_Xy,rank_Xy,recvtag);
MPI_Isend(sendList_xz, sendCount_xz,MPI_INT,rank_xz,sendtag,Communicator,&req1[10]);
MPI_Irecv(recvList_XZ, recvCount_XZ,MPI_INT,rank_XZ,recvtag,Communicator,&req2[10]);
MPI_Isend(sendList_XZ, sendCount_XZ,MPI_INT,rank_XZ,sendtag,Communicator,&req1[11]);
MPI_Irecv(recvList_xz, recvCount_xz,MPI_INT,rank_xz,recvtag,Communicator,&req2[11]);
MPI_Isend(sendList_Xz, sendCount_Xz,MPI_INT,rank_Xz,sendtag,Communicator,&req1[12]);
MPI_Irecv(recvList_xZ, recvCount_xZ,MPI_INT,rank_xZ,recvtag,Communicator,&req2[12]);
MPI_Isend(sendList_xZ, sendCount_xZ,MPI_INT,rank_xZ,sendtag,Communicator,&req1[13]);
MPI_Irecv(recvList_Xz, recvCount_Xz,MPI_INT,rank_Xz,recvtag,Communicator,&req2[13]);
req1[10] = comm.Isend(sendList_xz,sendCount_xz,rank_xz,sendtag);
req2[10] = comm.Irecv(recvList_XZ,recvCount_XZ,rank_XZ,recvtag);
req1[11] = comm.Isend(sendList_XZ,sendCount_XZ,rank_XZ,sendtag);
req2[11] = comm.Irecv(recvList_xz,recvCount_xz,rank_xz,recvtag);
req1[12] = comm.Isend(sendList_Xz,sendCount_Xz,rank_Xz,sendtag);
req2[12] = comm.Irecv(recvList_xZ,recvCount_xZ,rank_xZ,recvtag);
req1[13] = comm.Isend(sendList_xZ,sendCount_xZ,rank_xZ,sendtag);
req2[13] = comm.Irecv(recvList_Xz,recvCount_Xz,rank_Xz,recvtag);
MPI_Isend(sendList_yz, sendCount_yz,MPI_INT,rank_yz,sendtag,Communicator,&req1[14]);
MPI_Irecv(recvList_YZ, recvCount_YZ,MPI_INT,rank_YZ,recvtag,Communicator,&req2[14]);
MPI_Isend(sendList_YZ, sendCount_YZ,MPI_INT,rank_YZ,sendtag,Communicator,&req1[15]);
MPI_Irecv(recvList_yz, recvCount_yz,MPI_INT,rank_yz,recvtag,Communicator,&req2[15]);
MPI_Isend(sendList_Yz, sendCount_Yz,MPI_INT,rank_Yz,sendtag,Communicator,&req1[16]);
MPI_Irecv(recvList_yZ, recvCount_yZ,MPI_INT,rank_yZ,recvtag,Communicator,&req2[16]);
MPI_Isend(sendList_yZ, sendCount_yZ,MPI_INT,rank_yZ,sendtag,Communicator,&req1[17]);
MPI_Irecv(recvList_Yz, recvCount_Yz,MPI_INT,rank_Yz,recvtag,Communicator,&req2[17]);
MPI_Waitall(18,req1,stat1);
MPI_Waitall(18,req2,stat2);
req1[14] = comm.Isend(sendList_yz,sendCount_yz,rank_yz,sendtag);
req2[14] = comm.Irecv(recvList_YZ,recvCount_YZ,rank_YZ,recvtag);
req1[15] = comm.Isend(sendList_YZ,sendCount_YZ,rank_YZ,sendtag);
req2[15] = comm.Irecv(recvList_yz,recvCount_yz,rank_yz,recvtag);
req1[16] = comm.Isend(sendList_Yz,sendCount_Yz,rank_Yz,sendtag);
req2[16] = comm.Irecv(recvList_yZ,recvCount_yZ,rank_yZ,recvtag);
req1[17] = comm.Isend(sendList_yZ,sendCount_yZ,rank_yZ,sendtag);
req2[17] = comm.Irecv(recvList_Yz,recvCount_Yz,rank_Yz,recvtag);
comm.waitAll( 18, req1 );
comm.waitAll( 18, req2 );
}
//***************************************************************************************
inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
inline void CommunicateMeshHalo(DoubleArray &Mesh, const Utilities::MPI& comm,
double *sendbuf_x,double *sendbuf_y,double *sendbuf_z,double *sendbuf_X,double *sendbuf_Y,double *sendbuf_Z,
double *sendbuf_xy,double *sendbuf_XY,double *sendbuf_xY,double *sendbuf_Xy,
double *sendbuf_xz,double *sendbuf_XZ,double *sendbuf_xZ,double *sendbuf_Xz,
@ -334,42 +331,24 @@ inline void CommunicateMeshHalo(DoubleArray &Mesh, MPI_Comm Communicator,
PackMeshData(sendList_yZ, sendCount_yZ ,sendbuf_yZ, MeshData);
PackMeshData(sendList_YZ, sendCount_YZ ,sendbuf_YZ, MeshData);
//......................................................................................
MPI_Sendrecv(sendbuf_x,sendCount_x,MPI_DOUBLE,rank_x,sendtag,
recvbuf_X,recvCount_X,MPI_DOUBLE,rank_X,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_X,sendCount_X,MPI_DOUBLE,rank_X,sendtag,
recvbuf_x,recvCount_x,MPI_DOUBLE,rank_x,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_y,sendCount_y,MPI_DOUBLE,rank_y,sendtag,
recvbuf_Y,recvCount_Y,MPI_DOUBLE,rank_Y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Y,sendCount_Y,MPI_DOUBLE,rank_Y,sendtag,
recvbuf_y,recvCount_y,MPI_DOUBLE,rank_y,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_z,sendCount_z,MPI_DOUBLE,rank_z,sendtag,
recvbuf_Z,recvCount_Z,MPI_DOUBLE,rank_Z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Z,sendCount_Z,MPI_DOUBLE,rank_Z,sendtag,
recvbuf_z,recvCount_z,MPI_DOUBLE,rank_z,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xy,sendCount_xy,MPI_DOUBLE,rank_xy,sendtag,
recvbuf_XY,recvCount_XY,MPI_DOUBLE,rank_XY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XY,sendCount_XY,MPI_DOUBLE,rank_XY,sendtag,
recvbuf_xy,recvCount_xy,MPI_DOUBLE,rank_xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xy,sendCount_Xy,MPI_DOUBLE,rank_Xy,sendtag,
recvbuf_xY,recvCount_xY,MPI_DOUBLE,rank_xY,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xY,sendCount_xY,MPI_DOUBLE,rank_xY,sendtag,
recvbuf_Xy,recvCount_Xy,MPI_DOUBLE,rank_Xy,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xz,sendCount_xz,MPI_DOUBLE,rank_xz,sendtag,
recvbuf_XZ,recvCount_XZ,MPI_DOUBLE,rank_XZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_XZ,sendCount_XZ,MPI_DOUBLE,rank_XZ,sendtag,
recvbuf_xz,recvCount_xz,MPI_DOUBLE,rank_xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Xz,sendCount_Xz,MPI_DOUBLE,rank_Xz,sendtag,
recvbuf_xZ,recvCount_xZ,MPI_DOUBLE,rank_xZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_xZ,sendCount_xZ,MPI_DOUBLE,rank_xZ,sendtag,
recvbuf_Xz,recvCount_Xz,MPI_DOUBLE,rank_Xz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yz,sendCount_yz,MPI_DOUBLE,rank_yz,sendtag,
recvbuf_YZ,recvCount_YZ,MPI_DOUBLE,rank_YZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_YZ,sendCount_YZ,MPI_DOUBLE,rank_YZ,sendtag,
recvbuf_yz,recvCount_yz,MPI_DOUBLE,rank_yz,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_Yz,sendCount_Yz,MPI_DOUBLE,rank_Yz,sendtag,
recvbuf_yZ,recvCount_yZ,MPI_DOUBLE,rank_yZ,recvtag,Communicator,MPI_STATUS_IGNORE);
MPI_Sendrecv(sendbuf_yZ,sendCount_yZ,MPI_DOUBLE,rank_yZ,sendtag,
recvbuf_Yz,recvCount_Yz,MPI_DOUBLE,rank_Yz,recvtag,Communicator,MPI_STATUS_IGNORE);
comm.sendrecv(sendbuf_x,sendCount_x,rank_x,sendtag,recvbuf_X,recvCount_X,rank_X,recvtag);
comm.sendrecv(sendbuf_X,sendCount_X,rank_X,sendtag,recvbuf_x,recvCount_x,rank_x,recvtag);
comm.sendrecv(sendbuf_y,sendCount_y,rank_y,sendtag,recvbuf_Y,recvCount_Y,rank_Y,recvtag);
comm.sendrecv(sendbuf_Y,sendCount_Y,rank_Y,sendtag,recvbuf_y,recvCount_y,rank_y,recvtag);
comm.sendrecv(sendbuf_z,sendCount_z,rank_z,sendtag,recvbuf_Z,recvCount_Z,rank_Z,recvtag);
comm.sendrecv(sendbuf_Z,sendCount_Z,rank_Z,sendtag,recvbuf_z,recvCount_z,rank_z,recvtag);
comm.sendrecv(sendbuf_xy,sendCount_xy,rank_xy,sendtag,recvbuf_XY,recvCount_XY,rank_XY,recvtag);
comm.sendrecv(sendbuf_XY,sendCount_XY,rank_XY,sendtag,recvbuf_xy,recvCount_xy,rank_xy,recvtag);
comm.sendrecv(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag,recvbuf_xY,recvCount_xY,rank_xY,recvtag);
comm.sendrecv(sendbuf_xY,sendCount_xY,rank_xY,sendtag,recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag);
comm.sendrecv(sendbuf_xz,sendCount_xz,rank_xz,sendtag,recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag);
comm.sendrecv(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag,recvbuf_xz,recvCount_xz,rank_xz,recvtag);
comm.sendrecv(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag,recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag);
comm.sendrecv(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag,recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag);
comm.sendrecv(sendbuf_yz,sendCount_yz,rank_yz,sendtag,recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag);
comm.sendrecv(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag,recvbuf_yz,recvCount_yz,rank_yz,recvtag);
comm.sendrecv(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag,recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag);
comm.sendrecv(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag,recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag);
//........................................................................................
UnpackMeshData(recvList_x, recvCount_x ,recvbuf_x, MeshData);
UnpackMeshData(recvList_X, recvCount_X ,recvbuf_X, MeshData);

53
common/Communication.hpp
View File

@ -34,9 +34,8 @@
#define COMMUNICATION_HPP_INC
#include "common/Communication.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Utilities.h"
//#include "ProfilerApp.h"
/********************************************************
@ -44,17 +43,19 @@
********************************************************/
template<class TYPE>
Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src_data,
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, MPI_Comm comm )
const RankInfoStruct& dst_rank, std::array<int,3> dst_size, const Utilities::MPI& comm )
{
#ifdef USE_MPI
if ( comm.getSize() == 1 ) {
return src_data.subset( { 0, (size_t) dst_size[0]-1, 0, (size_t) dst_size[1]-1, 0, (size_t) dst_size[2]-1 } );
}
// Get the src size
std::array<int,3> src_size;
int size0[3] = { (int) src_data.size(0), (int) src_data.size(1), (int) src_data.size(2) };
MPI_Allreduce( size0, src_size.data(), 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( size0, src_size.data(), 3 );
if ( !src_data.empty() )
ASSERT( src_size[0] == size0[0] && src_size[1] == size0[1] && src_size[2] == size0[2] );
// Check that dst_size matches on all ranks
MPI_Allreduce( dst_size.data(), size0, 3, MPI_INT, MPI_MAX, comm );
comm.maxReduce( dst_size.data(), size0, 3 );
ASSERT( dst_size[0] == size0[0] && dst_size[1] == size0[1] && dst_size[2] == size0[2] );
// Function to get overlap range
auto calcOverlap = []( int i1[3], int i2[3], int j1[3], int j2[3] ) {
@ -92,7 +93,7 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
}
std::vector<MPI_Request> send_request( send_rank.size() );
for (size_t i=0; i<send_rank.size(); i++)
MPI_Isend( send_data[i].data(), sizeof(TYPE)*send_data[i].length(), MPI_BYTE, send_rank[i], 5462, comm, &send_request[i]);
send_request[i] = comm.Isend( send_data[i].data(), send_data[i].length(), send_rank[i], 5462 );
// Unpack data from the appropriate ranks (including myself)
Array<TYPE> dst_data( dst_size[0], dst_size[1], dst_size[2] );
int i1[3] = { dst_size[0] * dst_rank.ix, dst_size[1] * dst_rank.jy, dst_size[2] * dst_rank.kz };
@ -107,17 +108,14 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
continue;
int rank = src_rank.getRankForBlock(i,j,k);
Array<TYPE> data( index[1] - index[0] + 1, index[3] - index[2] + 1, index[5] - index[4] + 1 );
MPI_Recv( data.data(), sizeof(TYPE)*data.length(), MPI_BYTE, rank, 5462, comm, MPI_STATUS_IGNORE );
comm.recv( data.data(), data.length(), rank, 5462 );
dst_data.copySubset( index, data );
}
}
}
// Free data
MPI_Waitall( send_request.size(), send_request.data(), MPI_STATUSES_IGNORE );
comm.waitAll( send_request.size(), send_request.data() );
return dst_data;
#else
return src_data.subset( { 0, dst_size[0]-1, 0, dst_size[1]-1, 0, dst_size[2]-1 );
#endif
}
@ -126,27 +124,11 @@ Array<TYPE> redistribute( const RankInfoStruct& src_rank, const Array<TYPE>& src
* Structure to fill halo cells *
********************************************************/
template<class TYPE>
fillHalo<TYPE>::fillHalo( MPI_Comm comm_, const RankInfoStruct& info_,
fillHalo<TYPE>::fillHalo( const Utilities::MPI& comm_, const RankInfoStruct& info_,
std::array<int,3> n_, std::array<int,3> ng_, int tag0, int depth_,
std::array<bool,3> fill, std::array<bool,3> periodic ):
comm(comm_), info(info_), n(n_), ng(ng_), depth(depth_)
{
if ( std::is_same<TYPE,double>() ) {
N_type = 1;
datatype = MPI_DOUBLE;
} else if ( std::is_same<TYPE,float>() ) {
N_type = 1;
datatype = MPI_FLOAT;
} else if ( sizeof(TYPE)%sizeof(double)==0 ) {
N_type = sizeof(TYPE) / sizeof(double);
datatype = MPI_DOUBLE;
} else if ( sizeof(TYPE)%sizeof(float)==0 ) {
N_type = sizeof(TYPE) / sizeof(float);
datatype = MPI_FLOAT;
} else {
N_type = sizeof(TYPE);
datatype = MPI_BYTE;
}
// Set the fill pattern
memset(fill_pattern,0,sizeof(fill_pattern));
if ( fill[0] ) {
@ -283,8 +265,8 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Irecv( recv[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[2-i][2-j][2-k], comm, &recv_req[i][j][k] );
recv_req[i][j][k] = comm.Irecv( recv[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[2-i][2-j][2-k] );
}
}
}
@ -295,19 +277,18 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
if ( !fill_pattern[i][j][k] )
continue;
pack( data, i-1, j-1, k-1, send[i][j][k] );
MPI_Isend( send[i][j][k], N_type*depth2*N_send_recv[i][j][k], datatype,
info.rank[i][j][k], tag[i][j][k], comm, &send_req[i][j][k] );
send_req[i][j][k] = comm.Isend( send[i][j][k], depth2*N_send_recv[i][j][k],
info.rank[i][j][k], tag[i][j][k] );
}
}
}
// Recv the dst data and unpack (we recive in reverse order to match the sends)
MPI_Status status;
for (int i=2; i>=0; i--) {
for (int j=2; j>=0; j--) {
for (int k=2; k>=0; k--) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&recv_req[i][j][k],&status);
comm.wait( recv_req[i][j][k] );
unpack( data, i-1, j-1, k-1, recv[i][j][k] );
}
}
@ -318,7 +299,7 @@ void fillHalo<TYPE>::fill( Array<TYPE>& data )
for (int k=0; k<3; k++) {
if ( !fill_pattern[i][j][k] )
continue;
MPI_Wait(&send_req[i][j][k],&status);
comm.wait( send_req[i][j][k] );
}
}
}

957
common/Domain.cpp
View File

File diff suppressed because it is too large Load Diff

64
common/Domain.h
View File

@ -21,6 +21,7 @@
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <vector>
#include <math.h>
#include <time.h>
#include <exception>
@ -28,7 +29,7 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/Database.h"
@ -79,7 +80,7 @@ private:
class Domain{
public:
//! Default constructor
Domain( std::shared_ptr<Database> db, MPI_Comm Communicator);
Domain( std::shared_ptr<Database> db, const Utilities::MPI& Communicator);
//! Obsolete constructor
Domain( int nx, int ny, int nz, int rnk, int npx, int npy, int npz,
@ -132,12 +133,10 @@ public: // Public variables (need to create accessors instead)
double porosity;
RankInfoStruct rank_info;
MPI_Comm Comm; // MPI Communicator for this domain
Utilities::MPI Comm; // MPI Communicator for this domain
int BoundaryCondition;
MPI_Group Group; // Group of processors associated with this domain
//**********************************
// MPI ranks for all 18 neighbors
//**********************************
@ -173,32 +172,24 @@ public: // Public variables (need to create accessors instead)
// Get the actual D3Q19 communication counts (based on location of solid phase)
// Discrete velocity set symmetry implies the sendcount = recvcount
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
//......................................................................................
int *sendList_x, *sendList_y, *sendList_z, *sendList_X, *sendList_Y, *sendList_Z;
int *sendList_xy, *sendList_yz, *sendList_xz, *sendList_Xy, *sendList_Yz, *sendList_xZ;
int *sendList_xY, *sendList_yZ, *sendList_Xz, *sendList_XY, *sendList_YZ, *sendList_XZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
//......................................................................................
int *recvList_x, *recvList_y, *recvList_z, *recvList_X, *recvList_Y, *recvList_Z;
int *recvList_xy, *recvList_yz, *recvList_xz, *recvList_Xy, *recvList_Yz, *recvList_xZ;
int *recvList_xY, *recvList_yZ, *recvList_Xz, *recvList_XY, *recvList_YZ, *recvList_XZ;
inline int recvCount( const char* dir ) const { return getRecvList( dir ).size(); }
inline int sendCount( const char* dir ) const { return getSendList( dir ).size(); }
inline const int* recvList( const char* dir ) const { return getRecvList( dir ).data(); }
inline const int* sendList( const char* dir ) const { return getSendList( dir ).data(); }
//......................................................................................
// Solid indicator function
signed char *id;
std::vector<signed char> id;
void ReadIDs();
void Decomp( const std::string& filename );
void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
void CommunicateMeshHalo(DoubleArray &Mesh);
void CommInit();
int PoreCount();
void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
void AggregateLabels( const std::string& filename );
void AggregateLabels( const std::string& filename, DoubleArray &UserData );
private:
@ -208,22 +199,18 @@ private:
//......................................................................................
MPI_Request req1[18], req2[18];
MPI_Status stat1[18],stat2[18];
//......................................................................................
std::vector<int> sendList_x, sendList_y, sendList_z, sendList_X, sendList_Y, sendList_Z;
std::vector<int> sendList_xy, sendList_yz, sendList_xz, sendList_Xy, sendList_Yz, sendList_xZ;
std::vector<int> sendList_xY, sendList_yZ, sendList_Xz, sendList_XY, sendList_YZ, sendList_XZ;
//......................................................................................
std::vector<int> recvList_x, recvList_y, recvList_z, recvList_X, recvList_Y, recvList_Z;
std::vector<int> recvList_xy, recvList_yz, recvList_xz, recvList_Xy, recvList_Yz, recvList_xZ;
std::vector<int> recvList_xY, recvList_yZ, recvList_Xz, recvList_XY, recvList_YZ, recvList_XZ;
//......................................................................................
const std::vector<int>& getRecvList( const char* dir ) const;
const std::vector<int>& getSendList( const char* dir ) const;
int *sendBuf_x, *sendBuf_y, *sendBuf_z, *sendBuf_X, *sendBuf_Y, *sendBuf_Z;
int *sendBuf_xy, *sendBuf_yz, *sendBuf_xz, *sendBuf_Xy, *sendBuf_Yz, *sendBuf_xZ;
int *sendBuf_xY, *sendBuf_yZ, *sendBuf_Xz, *sendBuf_XY, *sendBuf_YZ, *sendBuf_XZ;
//......................................................................................
int *recvBuf_x, *recvBuf_y, *recvBuf_z, *recvBuf_X, *recvBuf_Y, *recvBuf_Z;
int *recvBuf_xy, *recvBuf_yz, *recvBuf_xz, *recvBuf_Xy, *recvBuf_Yz, *recvBuf_xZ;
int *recvBuf_xY, *recvBuf_yZ, *recvBuf_Xz, *recvBuf_XY, *recvBuf_YZ, *recvBuf_XZ;
//......................................................................................
double *sendData_x, *sendData_y, *sendData_z, *sendData_X, *sendData_Y, *sendData_Z;
double *sendData_xy, *sendData_yz, *sendData_xz, *sendData_Xy, *sendData_Yz, *sendData_xZ;
double *sendData_xY, *sendData_yZ, *sendData_Xz, *sendData_XY, *sendData_YZ, *sendData_XZ;
double *recvData_x, *recvData_y, *recvData_z, *recvData_X, *recvData_Y, *recvData_Z;
double *recvData_xy, *recvData_yz, *recvData_xz, *recvData_Xy, *recvData_Yz, *recvData_xZ;
double *recvData_xY, *recvData_yZ, *recvData_Xz, *recvData_XY, *recvData_YZ, *recvData_XZ;
};
@ -261,9 +248,6 @@ private:
};
//void ReadFromFile(const std::string& Filename,const std::string& Datatype, double *UserData);
//void ReadFromFile(const std::string& Filename, DoubleArray &Mesh);
void WriteCheckpoint(const char *FILENAME, const double *cDen, const double *cfq, size_t Np);
void ReadCheckpoint(char *FILENAME, double *cDen, double *cfq, size_t Np);

1176
common/MPI.I Normal file
View File

File diff suppressed because it is too large Load Diff

3810
common/MPI.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

1170
common/MPI.h Normal file
View File

File diff suppressed because it is too large Load Diff

282
common/MPI_Helpers.cpp
View File

@ -1,282 +0,0 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include "common/MPI_Helpers.h"
#include "common/Utilities.h"
/********************************************************
* Return the MPI data type *
********************************************************/
template<> MPI_Datatype getMPItype<char>() {
return MPI_CHAR;
}
template<> MPI_Datatype getMPItype<unsigned char>() {
return MPI_UNSIGNED_CHAR;
}
template<> MPI_Datatype getMPItype<int>() {
return MPI_INT;
}
template<> MPI_Datatype getMPItype<long>() {
return MPI_LONG;
}
template<> MPI_Datatype getMPItype<unsigned long>() {
return MPI_UNSIGNED_LONG;
}
template<> MPI_Datatype getMPItype<long long>() {
return MPI_LONG_LONG;
}
template<> MPI_Datatype getMPItype<float>() {
return MPI_FLOAT;
}
template<> MPI_Datatype getMPItype<double>() {
return MPI_DOUBLE;
}
/********************************************************
* Concrete implimentations for packing/unpacking *
********************************************************/
// unsigned char
template<>
size_t packsize<unsigned char>( const unsigned char& )
{
return sizeof(unsigned char);
}
template<>
void pack<unsigned char>( const unsigned char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(unsigned char));
}
template<>
void unpack<unsigned char>( unsigned char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(unsigned char));
}
// char
template<>
size_t packsize<char>( const char& )
{
return sizeof(char);
}
template<>
void pack<char>( const char& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(char));
}
template<>
void unpack<char>( char& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(char));
}
// int
template<>
size_t packsize<int>( const int& )
{
return sizeof(int);
}
template<>
void pack<int>( const int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<int>( int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// unsigned int
template<>
size_t packsize<unsigned int>( const unsigned int& )
{
return sizeof(unsigned int);
}
template<>
void pack<unsigned int>( const unsigned int& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(int));
}
template<>
void unpack<unsigned int>( unsigned int& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(int));
}
// size_t
template<>
size_t packsize<size_t>( const size_t& )
{
return sizeof(size_t);
}
template<>
void pack<size_t>( const size_t& rhs, char *buffer )
{
memcpy(buffer,&rhs,sizeof(size_t));
}
template<>
void unpack<size_t>( size_t& data, const char *buffer )
{
memcpy(&data,buffer,sizeof(size_t));
}
// std::string
template<>
size_t packsize<std::string>( const std::string& rhs )
{
return rhs.size()+1;
}
template<>
void pack<std::string>( const std::string& rhs, char *buffer )
{
memcpy(buffer,rhs.c_str(),rhs.size()+1);
}
template<>
void unpack<std::string>( std::string& data, const char *buffer )
{
data = std::string(buffer);
}
/********************************************************
* Fake MPI routines *
********************************************************/
#ifndef USE_MPI
int MPI_Init(int*,char***)
{
return 0;
}
int MPI_Init_thread(int*,char***, int required, int *provided )
{
*provided = required;
return 0;
}
int MPI_Finalize()
{
return 0;
}
int MPI_Comm_size( MPI_Comm, int *size )
{
*size = 1;
return 0;
}
int MPI_Comm_rank( MPI_Comm, int *rank )
{
*rank = 0;
return 0;
}
int MPI_Barrier( MPI_Comm )
{
return 0;
}
int MPI_Waitall( int, MPI_Request[], MPI_Status[] )
{
return 0;
}
int MPI_Wait( MPI_Request*, MPI_Status* )
{
return 0;
}
int MPI_Bcast( void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm )
{
return 0;
}
int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm, MPI_Request *request)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Request *request)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag,
MPI_Comm comm, MPI_Status *status)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, MPI_Comm comm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm)
{
ERROR("Not implimented yet");
return 0;
}
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
{
*newcomm = comm;
return 0;
}
double MPI_Wtime( void )
{
return 0.0;
}
int MPI_Comm_free(MPI_Comm *group)
{
return 0;
}
int MPI_Group_free(MPI_Group *group)
{
return 0;
}
#endif

255
common/MPI_Helpers.h
View File

@ -1,255 +0,0 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
// This file contains wrappers for MPI routines and functions to pack/unpack data structures
#ifndef MPI_WRAPPERS_INC
#define MPI_WRAPPERS_INC
#include <string.h>
#include <vector>
#include <set>
#include <map>
#ifdef USE_MPI
// Inlcude MPI
#include "mpi.h"
#else
// Create fake MPI types
typedef int MPI_Comm;
typedef int MPI_Request;
typedef int MPI_Status;
#define MPI_COMM_WORLD 0
#define MPI_COMM_SELF 0
#define MPI_COMM_NULL -1
#define MPI_GROUP_NULL -2
#define MPI_STATUS_IGNORE NULL
enum MPI_Datatype { MPI_LOGICAL, MPI_CHAR, MPI_UNSIGNED_CHAR, MPI_INT,
MPI_UNSIGNED, MPI_LONG, MPI_UNSIGNED_LONG, MPI_LONG_LONG, MPI_FLOAT, MPI_DOUBLE };
enum MPI_Op { MPI_MIN, MPI_MAX, MPI_SUM };
typedef int MPI_Group;
#define MPI_THREAD_SINGLE 0
#define MPI_THREAD_FUNNELED 1
#define MPI_THREAD_SERIALIZED 2
#define MPI_THREAD_MULTIPLE 3
// Fake MPI functions
int MPI_Init(int*,char***);
int MPI_Init_thread( int *argc, char ***argv, int required, int *provided );
int MPI_Finalize();
int MPI_Comm_size( MPI_Comm, int *size );
int MPI_Comm_rank( MPI_Comm, int *rank );
int MPI_Barrier(MPI_Comm);
int MPI_Wait(MPI_Request*,MPI_Status*);
int MPI_Waitall(int,MPI_Request[],MPI_Status[]);
int MPI_Bcast(void*,int,MPI_Datatype,int,MPI_Comm);
int MPI_Send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm);
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status);
int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm, MPI_Request *request);
int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source,
int tag, MPI_Comm comm, MPI_Request *request);
int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
MPI_Datatype datatype, MPI_Op op, MPI_Comm comm);
int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
MPI_Comm comm);
int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, MPI_Comm comm);
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag,
MPI_Comm comm, MPI_Status *status);
int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
MPI_Op op, int root, MPI_Comm comm);
double MPI_Wtime( void );
int MPI_Comm_group(MPI_Comm comm, MPI_Group *group);
int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
int MPI_Comm_free(MPI_Comm *group);
int MPI_Group_free(MPI_Group *group);
int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
#endif
//! Get the size of the MPI_Comm
// Note: this is a thread and interrupt safe function
inline int comm_size( MPI_Comm comm ) {
int size = 1;
MPI_Comm_size( comm, &size );
return size;
}
//! Get the rank of the MPI_Comm
// Note: this is a thread and interrupt safe function
inline int comm_rank( MPI_Comm comm ) {
int rank = 1;
MPI_Comm_rank( comm, &rank );
return rank;
}
//! Get the size of MPI_COMM_WORLD
inline int MPI_WORLD_SIZE( ) {
return comm_size( MPI_COMM_WORLD );
}
//! Get the size of MPI_COMM_WORLD
inline int MPI_WORLD_RANK( ) {
return comm_rank( MPI_COMM_WORLD );
}
//! Return the appropriate MPI datatype for a class
template<class TYPE>
MPI_Datatype getMPItype();
//! Template function to return the buffer size required to pack a class
template<class TYPE>
size_t packsize( const TYPE& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const TYPE& rhs, char *buffer );
//! Template function to unpack a class from a buffer
template<class TYPE>
void unpack( TYPE& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::vector
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::pair
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::map
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer );
//! Template function to return the buffer size required to pack a std::set
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs );
//! Template function to pack a class to a buffer
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer );
//! Template function to pack a class to a buffer
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer );
// Helper functions
inline double sumReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_SUM,comm);
return y;
}
inline float sumReduce( MPI_Comm comm, float x )
{
float y = 0;
MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_SUM,comm);
return y;
}
inline int sumReduce( MPI_Comm comm, int x )
{
int y = 0;
MPI_Allreduce(&x,&y,1,MPI_INT,MPI_SUM,comm);
return y;
}
inline long long sumReduce( MPI_Comm comm, long long x )
{
long long y = 0;
MPI_Allreduce(&x,&y,1,MPI_LONG_LONG,MPI_SUM,comm);
return y;
}
inline bool sumReduce( MPI_Comm comm, bool x )
{
int y = sumReduce( comm, x?1:0 );
return y>0;
}
inline std::vector<float> sumReduce( MPI_Comm comm, const std::vector<float>& x )
{
auto y = x;
MPI_Allreduce(x.data(),y.data(),x.size(),MPI_FLOAT,MPI_SUM,comm);
return y;
}
inline std::vector<int> sumReduce( MPI_Comm comm, const std::vector<int>& x )
{
auto y = x;
MPI_Allreduce(x.data(),y.data(),x.size(),MPI_INT,MPI_SUM,comm);
return y;
}
inline double maxReduce( MPI_Comm comm, double x )
{
double y = 0;
MPI_Allreduce(&x,&y,1,MPI_DOUBLE,MPI_MAX,comm);
return y;
}
inline float maxReduce( MPI_Comm comm, float x )
{
float y = 0;
MPI_Allreduce(&x,&y,1,MPI_FLOAT,MPI_MAX,comm);
return y;
}
inline int maxReduce( MPI_Comm comm, int x )
{
int y = 0;
MPI_Allreduce(&x,&y,1,MPI_INT,MPI_MAX,comm);
return y;
}
#endif
#include "common/MPI_Helpers.hpp"

186
common/MPI_Helpers.hpp
View File

@ -1,186 +0,0 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
// This file contains wrappers for MPI routines and functions to pack/unpack data structures
#ifndef MPI_WRAPPERS_HPP
#define MPI_WRAPPERS_HPP
#include "common/MPI_Helpers.h"
#include <string.h>
#include <vector>
#include <set>
#include <map>
/********************************************************
* Default instantiations for std::vector *
********************************************************/
template<class TYPE>
size_t packsize( const std::vector<TYPE>& rhs )
{
size_t bytes = sizeof(size_t);
for (size_t i=0; i<rhs.size(); i++)
bytes += packsize(rhs[i]);
return bytes;
}
template<class TYPE>
void pack( const std::vector<TYPE>& rhs, char *buffer )
{
size_t size = rhs.size();
memcpy(buffer,&size,sizeof(size_t));
size_t pos = sizeof(size_t);
for (size_t i=0; i<rhs.size(); i++) {
pack(rhs[i],&buffer[pos]);
pos += packsize(rhs[i]);
}
}
template<class TYPE>
void unpack( std::vector<TYPE>& data, const char *buffer )
{
size_t size;
memcpy(&size,buffer,sizeof(size_t));
data.clear();
data.resize(size);
size_t pos = sizeof(size_t);
for (size_t i=0; i<data.size(); i++) {
unpack(data[i],&buffer[pos]);
pos += packsize(data[i]);
}
}
/********************************************************
* Default instantiations for std::pair *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::pair<TYPE1,TYPE2>& rhs )
{
return packsize(rhs.first)+packsize(rhs.second);
}
template<class TYPE1, class TYPE2>
void pack( const std::pair<TYPE1,TYPE2>& rhs, char *buffer )
{
pack(rhs.first,buffer);
pack(rhs.second,&buffer[packsize(rhs.first)]);
}
template<class TYPE1, class TYPE2>
void unpack( std::pair<TYPE1,TYPE2>& data, const char *buffer )
{
unpack(data.first,buffer);
unpack(data.second,&buffer[packsize(data.first)]);
}
/********************************************************
* Default instantiations for std::map *
********************************************************/
template<class TYPE1, class TYPE2>
size_t packsize( const std::map<TYPE1,TYPE2>& rhs )
{
size_t bytes = sizeof(size_t);
typename std::map<TYPE1,TYPE2>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
bytes += packsize(it->first);
bytes += packsize(it->second);
}
return bytes;
}
template<class TYPE1, class TYPE2>
void pack( const std::map<TYPE1,TYPE2>& rhs, char *buffer )
{
size_t N = rhs.size();
pack(N,buffer);
size_t pos = sizeof(size_t);
typename std::map<TYPE1,TYPE2>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
pack(it->first,&buffer[pos]); pos+=packsize(it->first);
pack(it->second,&buffer[pos]); pos+=packsize(it->second);
}
}
template<class TYPE1, class TYPE2>
void unpack( std::map<TYPE1,TYPE2>& data, const char *buffer )
{
size_t N = 0;
unpack(N,buffer);
size_t pos = sizeof(size_t);
data.clear();
for (size_t i=0; i<N; i++) {
std::pair<TYPE1,TYPE2> tmp;
unpack(tmp.first,&buffer[pos]); pos+=packsize(tmp.first);
unpack(tmp.second,&buffer[pos]); pos+=packsize(tmp.second);
data.insert(tmp);
}
}
/********************************************************
* Default instantiations for std::set *
********************************************************/
template<class TYPE>
size_t packsize( const std::set<TYPE>& rhs )
{
size_t bytes = sizeof(size_t);
typename std::set<TYPE>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
bytes += packsize(*it);
}
return bytes;
}
template<class TYPE>
void pack( const std::set<TYPE>& rhs, char *buffer )
{
size_t N = rhs.size();
pack(N,buffer);
size_t pos = sizeof(size_t);
typename std::set<TYPE>::const_iterator it;
for (it=rhs.begin(); it!=rhs.end(); ++it) {
pack(*it); pos+=packsize(*it);
}
}
template<class TYPE>
void unpack( std::set<TYPE>& data, const char *buffer )
{
size_t N = 0;
unpack(N,buffer);
size_t pos = sizeof(size_t);
data.clear();
for (size_t i=0; i<N; i++) {
TYPE tmp;
unpack(tmp,&buffer[pos]); pos+=packsize(tmp);
data.insert(tmp);
}
}
#endif

4
common/ReadMicroCT.cpp
View File

@ -64,11 +64,11 @@ Array<uint8_t> readMicroCT( const std::string& filename )
// Read the compressed micro CT data and distribute
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm )
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm )
{
// Get the local problem info
auto n = domain.getVector<int>( "n" );
int rank = comm_rank(MPI_COMM_WORLD);
int rank = comm.getRank();
auto nproc = domain.getVector<int>( "nproc" );
RankInfoStruct rankInfo( rank, nproc[0], nproc[1], nproc[2] );

3
common/ReadMicroCT.h
View File

@ -5,11 +5,12 @@
#include "common/Array.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/MPI.h"
Array<uint8_t> readMicroCT( const std::string& filename );
Array<uint8_t> readMicroCT( const Database& domain, MPI_Comm comm );
Array<uint8_t> readMicroCT( const Database& domain, const Utilities::MPI& comm );
#endif

1225
common/ScaLBL.cpp
View File

File diff suppressed because it is too large Load Diff

284
common/ScaLBL.h
View File

@ -62,7 +62,6 @@ extern "C" void ScaLBL_UnpackDenD3Q7(int *list, int count, double *recvbuf, int
extern "C" void ScaLBL_D3Q19_Init(double *Dist, int Np);
extern "C" void ScaLBL_D3Q19_Momentum(double *dist, double *vel, int Np);
extern "C" void ScaLBL_D3Q19_Pressure(double *dist, double *press, int Np);
@ -93,104 +92,6 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int f
extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double rlx_eff, double Fx, double Fy, double Fz,
double *Poros,double *Perm, double *Velocity,double Den,double *Pressure);
// GREYSCALE FREE-ENERGY MODEL (Two-component)
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFE(double *dist, double *Aq, double *Bq, double *Den,
// double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
// double *Poros,double *Perm, double *Velocity,double *Pressure);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFE(int *neighborList, double *dist, double *Aq, double *Bq, double *Den,
// double *DenGradA, double *DenGradB, double *SolidForce, int start, int finish, int Np,
// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double Gsc, double Gx, double Gy, double Gz,
// double *Poros,double *Perm, double *Velocity,double *Pressure);
//
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleFEChem(double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
// double Gx, double Gy, double Gz,
// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleFEChem(int *neighborList, double *dist, double *Cq, double *Phi, double *SolidForce, int start, int finish, int Np,
// double tauA,double tauB,double tauA_eff,double tauB_eff,double rhoA,double rhoB,double gamma,double kappaA,double kappaB,double lambdaA,double lambdaB,
// double Gx, double Gy, double Gz,
// double *Poros,double *Perm, double *Velocity,double *Pressure,double *PressureGrad,double *PressTensorGrad,double *PhiLap);
//
//extern "C" void ScaLBL_D3Q7_GreyscaleFE_Init(double *Den, double *Cq, double *PhiLap, double gamma, double kappaA, double kappaB, double lambdaA, double lambdaB, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleFE_IMRT_Init(double *dist, double *Den, double rhoA, double rhoB, int Np);
//
//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEDensity(int *NeighborList, double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEDensity(double *Aq, double *Bq, double *Den, double *Phi, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q7_AAodd_GreyscaleFEPhi(int *NeighborList, double *Cq, double *Phi, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q7_AAeven_GreyscaleFEPhi(double *Cq, double *Phi, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Gradient(int *neighborList, double *Den, double *DenGrad, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Laplacian(int *neighborList, double *Den, double *DenLap, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleFE_Pressure(double *dist, double *Den, double *Porosity,double *Velocity,
// double *Pressure, double rhoA,double rhoB, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleFE_PressureTensor(int *neighborList, double *Phi,double *Pressure, double *PressTensor, double *PhiLap,
// double kappaA,double kappaB,double lambdaA,double lambdaB, int start, int finish, int Np);
// GREYSCALE SHAN-CHEN MODEL (Two-component)
//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Init(int *Map, double *distA, double *distB, double *DenA, double *DenB, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_Density(int *NeighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_Density(int *Map, double *distA, double *distB, double *DenA, double *DenB, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_MRT(int *neighborList, int *Mpa, double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB,
// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
// int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_MRT(int *Map,double *distA, double *distB, double *DenA,double *DenB, double *DenGradA, double *DenGradB,
// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
// int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleSC_BGK(int *neighborList, int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB,
// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
// int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleSC_BGK(int *Map, double *distA, double *distB, double *DenA, double *DenB, double *DenGradA, double *DenGradB,
// double *SolidForceA, double *SolidForceB, double *Poros,double *Perm, double *Velocity,double *Pressure,
// double tauA,double tauB,double tauA_eff,double tauB_eff, double Gsc, double Gx, double Gy, double Gz,
// int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_GreyscaleSC_Gradient(int *neighborList, int *Map, double *Den, double *DenGrad, int strideY, int strideZ,int start, int finish, int Np);
//
//extern "C" void ScaLBL_GreyscaleSC_BC_z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
//
//extern "C" void ScaLBL_GreyscaleSC_BC_Z(int *list, int *Map, double *DenA, double *DenB, double vA, double vB, int count);
//
//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_z(int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
//
//extern "C" void ScaLBL_GreyscaleSC_AAeven_Pressure_BC_Z(int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
//
//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_z(int *neighborList, int *list, double *distA, double *distB, double dinA, double dinB, int count, int N);
//
//extern "C" void ScaLBL_GreyscaleSC_AAodd_Pressure_BC_Z(int *neighborList, int *list, double *distA, double *distB, double doutA, double doutB, int count, int N);
// GREYSCALE COLOR MODEL (Two-component)
//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np);
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *ColorGrad,double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
// double rhoA, double rhoB, double tauA, double tauB,double tauA_eff,double tauB_eff, double alpha, double beta,
// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
//
//extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *ColorGrad,double *Phi, double *GreySolidGrad, double *Poros,double *Perm,double *Vel,
// double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
// double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den,
double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,double *Pressure,
@ -202,6 +103,52 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
double rhoA, double rhoB, double tauA, double tauB, double tauA_eff,double tauB_eff, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
// ION TRANSPORT MODEL
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np);
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np);
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np);
// LBM Poisson solver
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList,int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np);
//maybe deprecated
//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
// int strideY, int strideZ,int start, int finish, int Np);
// LBM Stokes Model (adapted from MRT model)
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB,
double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np);
extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np);
// MRT MODEL
extern "C" void ScaLBL_D3Q19_AAeven_MRT(double *dist, int start, int finish, int Np, double rlx_setA, double rlx_setB, double Fx,
double Fy, double Fz);
@ -210,7 +157,6 @@ extern "C" void ScaLBL_D3Q19_AAodd_MRT(int *d_neighborList, double *dist, int st
double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz);
// COLOR MODEL
extern "C" void ScaLBL_D3Q19_AAeven_Color(int *Map, double *dist, double *Aq, double *Bq, double *Den, double *Phi,
double *Vel, double rhoA, double rhoB, double tauA, double tauB, double alpha, double beta,
double Fx, double Fy, double Fz, int strideY, int strideZ, int start, int finish, int Np);
@ -225,8 +171,16 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *NeighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_Gradient(int *Map, double *Phi, double *ColorGrad, int start, int finish, int Np, int Nx, int Ny, int Nz);
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz);
extern "C" void ScaLBL_PhaseField_Init(int *Map, double *Phi, double *Den, double *Aq, double *Bq, int start, int finish, int Np);
// Density functional hydrodynamics LBM
@ -246,6 +200,44 @@ extern "C" void ScaLBL_D3Q7_AAeven_DFH(double *Aq, double *Bq, double *Den, doub
extern "C" void ScaLBL_D3Q19_Gradient_DFH(int *NeighborList, double *Phi, double *ColorGrad, int start, int finish, int Np);
// FREE ENERGY LEE MODEL
extern "C" void ScaLBL_D3Q19_FreeLeeModel_TwoFluid_Init(double *gqbar, double *mu_phi, double *ColorGrad, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_D3Q19_FreeLeeModel_SingleFluid_Init(double *gqbar, double Fx, double Fy, double Fz, int Np);
extern "C" void ScaLBL_FreeLeeModel_PhaseField_Init(int *Map, double *Phi, double *Den, double *hq, double *ColorGrad,
double rhonA, double rhoB, double tauM, double W, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAodd_FreeLeeModel_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
//extern "C" void ScaLBL_D3Q7_AAeven_FreeLeeModel_PhaseField(int *Map, double *hq, double *Den, double *Phi,
// double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_FreeLee_PhaseField(int *neighborList, int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_FreeLee_PhaseField( int *Map, double *hq, double *Den, double *Phi, double *ColorGrad, double *Vel,
double rhoA, double rhoB, double tauM, double W, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q7_ComputePhaseField(int *Map, double *hq, double *Den, double *Phi, double rhoA, double rhoB, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel(int *neighborList, int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel(int *Map, double *dist, double *Den, double *Phi, double *mu_phi, double *Vel, double *Pressure, double *ColorGrad,
double rhoA, double rhoB, double tauA, double tauB, double kappa, double beta, double W, double Fx, double Fy, double Fz,
int strideY, int strideZ, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAodd_FreeLeeModel_SingleFluid_BGK(int *neighborList, double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q19_AAeven_FreeLeeModel_SingleFluid_BGK(double *dist, double *Vel, double *Pressure,
double tau, double rho0, double Fx, double Fy, double Fz, int start, int finish, int Np);
extern "C" void ScaLBL_D3Q9_MGTest(int *Map, double *Phi,double *ColorGrad,int strideY, int strideZ, int start, int finish, int Np);
// BOUNDARY CONDITION ROUTINES
extern "C" void ScaLBL_D3Q19_AAodd_Pressure_BC_z(int *neighborList, int *list, double *dist, double din, int count, int Np);
@ -278,6 +270,38 @@ extern "C" void ScaLBL_SetSlice_z(double *Phi, double value, int Nx, int Ny, int
extern "C" void ScaLBL_CopySlice_z(double *Phi, int Nx, int Ny, int Nz, int Source, int Destination);
extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np);
extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count);
extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count);
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double Cin, double tau, double *VelocityZ, int count, int Np);
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, double tau, double *VelocityZ, int count, int Np);
class ScaLBL_Communicator{
public:
//......................................................................................
@ -286,9 +310,9 @@ public:
//ScaLBL_Communicator(Domain &Dm, IntArray &Map);
~ScaLBL_Communicator();
//......................................................................................
MPI_Comm MPI_COMM_SCALBL; // MPI Communicator
unsigned long int CommunicationCount,SendCount,RecvCount;
int Nx,Ny,Nz,N;
int n_bb_d3q7, n_bb_d3q19;
int BoundaryCondition;
int next;
@ -311,11 +335,16 @@ public:
int FirstInterior();
int LastInterior();
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np);
double GetPerformance(int *NeighborList, double *fq, int Np);
int MemoryOptimizedLayoutAA(IntArray &Map, int *neighborList, signed char *id, int Np, int width);
void Barrier(){
ScaLBL_DeviceBarrier();
MPI_COMM_SCALBL.barrier();
};
void SendD3Q19AA(double *dist);
void RecvD3Q19AA(double *dist);
// void BiSendD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
// void BiRecvD3Q7(double *A_even, double *A_odd, double *B_even, double *B_odd);
void SendD3Q7AA(double *fq, int Component);
void RecvD3Q7AA(double *fq, int Component);
void BiSendD3Q7AA(double *Aq, double *Bq);
void BiRecvD3Q7AA(double *Aq, double *Bq);
void TriSendD3Q7AA(double *Aq, double *Bq, double *Cq);
@ -324,28 +353,35 @@ public:
void RecvHalo(double *data);
void RecvGrad(double *Phi, double *Gradient);
void RegularLayout(IntArray map, const double *data, DoubleArray &regdata);
void SetupBounceBackList(IntArray &Map, signed char *id, int Np);
void SolidDirichletD3Q7(double *fq, double *BoundaryValue);
void SolidNeumannD3Q7(double *fq, double *BoundaryValue);
// Routines to set boundary conditions
void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB);
void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time);
void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
void D3Q19_Reflection_BC_z(double *fq);
void D3Q19_Reflection_BC_Z(double *fq);
double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
// Routines to set boundary conditions
void Color_BC_z(int *Map, double *Phi, double *Den, double vA, double vB);
void Color_BC_Z(int *Map, double *Phi, double *Den, double vA, double vB);
void D3Q19_Pressure_BC_z(int *neighborList, double *fq, double din, int time);
void D3Q19_Pressure_BC_Z(int *neighborList, double *fq, double dout, int time);
void D3Q19_Reflection_BC_z(double *fq);
void D3Q19_Reflection_BC_Z(double *fq);
double D3Q19_Flux_BC_z(int *neighborList, double *fq, double flux, int time);
void D3Q7_Poisson_Potential_BC_z(int *neighborList, double *fq, double Vin, int time);
void D3Q7_Poisson_Potential_BC_Z(int *neighborList, double *fq, double Vout, int time);
void Poisson_D3Q7_BC_z(int *Map, double *Psi, double Vin);
void Poisson_D3Q7_BC_Z(int *Map, double *Psi, double Vout);
void D3Q7_Ion_Concentration_BC_z(int *neighborList, double *fq, double Cin, int time);
void D3Q7_Ion_Concentration_BC_Z(int *neighborList, double *fq, double Cout, int time);
void D3Q7_Ion_Flux_BC_z(int *neighborList, double *fq, double Cin, double tau, double *VelocityZ, int time);
void D3Q7_Ion_Flux_BC_Z(int *neighborList, double *fq, double Cout, double tau, double *VelocityZ, int time);
void GreyscaleSC_BC_z(int *Map, double *DenA, double *DenB, double vA, double vB);
void GreyscaleSC_BC_Z(int *Map, double *DenA, double *DenB, double vA, double vB);
void GreyscaleSC_Pressure_BC_z(int *neighborList, double *fqA, double *fqB, double dinA, double dinB, int time);
void GreyscaleSC_Pressure_BC_Z(int *neighborList, double *fqA, double *fqB, double doutA, double doutB, int time);
// void TestSendD3Q19(double *f_even, double *f_odd);
// void TestRecvD3Q19(double *f_even, double *f_odd);
// Debugging and unit testing functions
void PrintD3Q19();
// Debugging and unit testing functions
void PrintD3Q19();
private:
//void D3Q19_MapRecv_OLD(int q, int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist);
void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, int *list, int start, int count, int *d3q19_recvlist);
void D3Q19_MapRecv(int Cqx, int Cqy, int Cqz, const int *list, int start, int count, int *d3q19_recvlist);
bool Lock; // use Lock to make sure only one call at a time to protect data in transit
// only one set of Send requests can be active at any time (per instance)
@ -356,9 +392,8 @@ private:
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Group Group; // Group of processors associated with this domain
Utilities::MPI MPI_COMM_SCALBL; // MPI Communicator for this domain
MPI_Request req1[18],req2[18];
MPI_Status stat1[18],stat2[18];
//......................................................................................
// MPI ranks for all 18 neighbors
//......................................................................................
@ -393,6 +428,9 @@ private:
int *dvcRecvDist_xy, *dvcRecvDist_yz, *dvcRecvDist_xz, *dvcRecvDist_Xy, *dvcRecvDist_Yz, *dvcRecvDist_xZ;
int *dvcRecvDist_xY, *dvcRecvDist_yZ, *dvcRecvDist_Xz, *dvcRecvDist_XY, *dvcRecvDist_YZ, *dvcRecvDist_XZ;
//......................................................................................
int *bb_dist;
int *bb_interactions;
//......................................................................................
};

2
common/SpherePack.cpp
View File

@ -41,7 +41,7 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/Database.h"
#include "common/SpherePack.h"

2
common/SpherePack.h
View File

@ -28,7 +28,7 @@
#include "common/Array.h"
#include "common/Utilities.h"
#include "common/MPI_Helpers.h"
#include "common/MPI.h"
#include "common/Communication.h"
#include "common/Database.h"

24
common/Utilities.cpp
View File

@ -24,7 +24,7 @@
#endif
#ifdef USE_MPI
#include "mpi.h"
#include "common/MPI.h"
#endif
#include <algorithm>
@ -53,7 +53,7 @@ static std::mutex Utilities_mutex;
/****************************************************************************
* Function to perform the default startup/shutdown sequences *
****************************************************************************/
void Utilities::startup( int argc, char **argv )
void Utilities::startup( int argc, char **argv, bool multiple )
{
NULL_USE( argc );
NULL_USE( argv );
@ -62,15 +62,19 @@ void Utilities::startup( int argc, char **argv )
Utilities::setenv( "MKL_NUM_THREADS", "1" );
// Start MPI
#ifdef USE_MPI
int provided;
MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
if ( provided < MPI_THREAD_MULTIPLE ) {
int rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if ( rank == 0 )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
if ( multiple ) {
int provided;
MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
if ( provided < MPI_THREAD_MULTIPLE ) {
int rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if ( rank == 0 )
std::cerr << "Warning: Failed to start MPI with necessary thread support, thread support will be disabled" << std::endl;
}
StackTrace::globalCallStackInitialize( MPI_COMM_WORLD );
} else {
MPI_Init( &argc, &argv );
}
StackTrace::globalCallStackInitialize( MPI_COMM_WORLD );
#endif
// Set the error handlers
Utilities::setAbortBehavior( true, 3 );

2
common/Utilities.h
View File

@ -47,7 +47,7 @@ using StackTrace::Utilities::sleep_s;
* \param argc argc from main
* \param argv argv from main
*/
void startup( int argc, char **argv );
void startup( int argc, char **argv, bool multiple=true );
/*!
* \brief Stop MPI, error handlers

234
common/Utilities.hpp Normal file
View File

@ -0,0 +1,234 @@
#ifndef included_Utilities_hpp
#define included_Utilities_hpp
#include "Utilities.h"
#include <vector>
namespace Utilities {
/************************************************************************
* templated quicksort routines *
************************************************************************/
template<class T>
void quicksort( std::vector<T> &x )
{
if ( x.size() <= 1u )
return;
T *arr = &x[0];
bool test;
long int i, ir, j, jstack, k, l, istack[100];
T a, tmp_a;
jstack = 0;
l = 0;
ir = x.size() - 1;
while ( 1 ) {
if ( ir - l < 7 ) { // Insertion sort when subarray small enough.
for ( j = l + 1; j <= ir; j++ ) {
a = arr[j];
test = true;
for ( i = j - 1; i >= 0; i-- ) {
if ( arr[i] < a ) {
arr[i + 1] = a;
test = false;
break;
}
arr[i + 1] = arr[i];
}
if ( test ) {
i = l - 1;
arr[i + 1] = a;
}
}
if ( jstack == 0 )
return;
ir = istack[jstack]; // Pop stack and begin a new round of partitioning.
l = istack[jstack - 1];
jstack -= 2;
} else {
k = ( l + ir ) / 2; // Choose median of left, center and right elements as partitioning
// element a. Also rearrange so that a(l) < a(l+1) < a(ir).
tmp_a = arr[k];
arr[k] = arr[l + 1];
arr[l + 1] = tmp_a;
if ( arr[l] > arr[ir] ) {
tmp_a = arr[l];
arr[l] = arr[ir];
arr[ir] = tmp_a;
}
if ( arr[l + 1] > arr[ir] ) {
tmp_a = arr[l + 1];
arr[l + 1] = arr[ir];
arr[ir] = tmp_a;
}
if ( arr[l] > arr[l + 1] ) {
tmp_a = arr[l];
arr[l] = arr[l + 1];
arr[l + 1] = tmp_a;
}
// Scan up to find element > a
j = ir;
a = arr[l + 1]; // Partitioning element.
for ( i = l + 2; i <= ir; i++ ) {
if ( arr[i] < a )
continue;
while ( arr[j] > a ) // Scan down to find element < a.
j--;
if ( j < i )
break; // Pointers crossed. Exit with partitioning complete.
tmp_a = arr[i]; // Exchange elements of both arrays.
arr[i] = arr[j];
arr[j] = tmp_a;
}
arr[l + 1] = arr[j]; // Insert partitioning element in both arrays.
arr[j] = a;
jstack += 2;
// Push pointers to larger subarray on stack, process smaller subarray immediately.
if ( ir - i + 1 >= j - l ) {
istack[jstack] = ir;
istack[jstack - 1] = i;
ir = j - 1;
} else {
istack[jstack] = j - 1;
istack[jstack - 1] = l;
l = i;
}
}
}
}
template<class T1, class T2>
void quicksort( std::vector<T1> &x, std::vector<T2> &y )
{
if ( x.size() <= 1u )
return;
T1 *arr = &x[0];
T2 *brr = &y[0];
bool test;
long int i, ir, j, jstack, k, l, istack[100];
T1 a, tmp_a;
T2 b, tmp_b;
jstack = 0;
l = 0;
ir = x.size() - 1;
while ( 1 ) {
if ( ir - l < 7 ) { // Insertion sort when subarray small enough.
for ( j = l + 1; j <= ir; j++ ) {
a = arr[j];
b = brr[j];
test = true;
for ( i = j - 1; i >= 0; i-- ) {
if ( arr[i] < a ) {
arr[i + 1] = a;
brr[i + 1] = b;
test = false;
break;
}
arr[i + 1] = arr[i];
brr[i + 1] = brr[i];
}
if ( test ) {
i = l - 1;
arr[i + 1] = a;
brr[i + 1] = b;
}
}
if ( jstack == 0 )
return;
ir = istack[jstack]; // Pop stack and begin a new round of partitioning.
l = istack[jstack - 1];
jstack -= 2;
} else {
k = ( l + ir ) / 2; // Choose median of left, center and right elements as partitioning
// element a. Also rearrange so that a(l) ? a(l+1) ? a(ir).
tmp_a = arr[k];
arr[k] = arr[l + 1];
arr[l + 1] = tmp_a;
tmp_b = brr[k];
brr[k] = brr[l + 1];
brr[l + 1] = tmp_b;
if ( arr[l] > arr[ir] ) {
tmp_a = arr[l];
arr[l] = arr[ir];
arr[ir] = tmp_a;
tmp_b = brr[l];
brr[l] = brr[ir];
brr[ir] = tmp_b;
}
if ( arr[l + 1] > arr[ir] ) {
tmp_a = arr[l + 1];
arr[l + 1] = arr[ir];
arr[ir] = tmp_a;
tmp_b = brr[l + 1];
brr[l + 1] = brr[ir];
brr[ir] = tmp_b;
}
if ( arr[l] > arr[l + 1] ) {
tmp_a = arr[l];
arr[l] = arr[l + 1];
arr[l + 1] = tmp_a;
tmp_b = brr[l];
brr[l] = brr[l + 1];
brr[l + 1] = tmp_b;
}
// Scan up to find element > a
j = ir;
a = arr[l + 1]; // Partitioning element.
b = brr[l + 1];
for ( i = l + 2; i <= ir; i++ ) {
if ( arr[i] < a )
continue;
while ( arr[j] > a ) // Scan down to find element < a.
j--;
if ( j < i )
break; // Pointers crossed. Exit with partitioning complete.
tmp_a = arr[i]; // Exchange elements of both arrays.
arr[i] = arr[j];
arr[j] = tmp_a;
tmp_b = brr[i];
brr[i] = brr[j];
brr[j] = tmp_b;
}
arr[l + 1] = arr[j]; // Insert partitioning element in both arrays.
arr[j] = a;
brr[l + 1] = brr[j];
brr[j] = b;
jstack += 2;
// Push pointers to larger subarray on stack, process smaller subarray immediately.
if ( ir - i + 1 >= j - l ) {
istack[jstack] = ir;
istack[jstack - 1] = i;
ir = j - 1;
} else {
istack[jstack] = j - 1;
istack[jstack - 1] = l;
l = i;
}
}
}
}
template<class T>
void unique( std::vector<T> &x )
{
if ( x.size() <= 1 )
return;
// First perform a quicksort
quicksort( x );
// Next remove duplicate entries
size_t pos = 1;
for ( size_t i = 1; i < x.size(); i++ ) {
if ( x[i] != x[pos - 1] ) {
x[pos] = x[i];
pos++;
}
}
if ( pos < x.size() )
x.resize( pos );
}
}
#endif

340
common/WideHalo.cpp Normal file
View File

@ -0,0 +1,340 @@
/*
This class implements support for halo widths larger than 1
*/
#include "common/WideHalo.h"
ScaLBLWideHalo_Communicator::ScaLBLWideHalo_Communicator(std::shared_ptr <Domain> Dm, int width)
{
//......................................................................................
Lock=false; // unlock the communicator
//......................................................................................
// Create a separate copy of the communicator for the device
MPI_COMM_SCALBL = Dm->Comm.dup();
//......................................................................................
// Copy the domain size and communication information directly from Dm
Nx = Dm->Nx;
Ny = Dm->Ny;
Nz = Dm->Nz;
N = Nx*Ny*Nz;
Nxh = Nx + 2*(width - 1);
Nyh = Ny + 2*(width - 1);
Nzh = Nz + 2*(width - 1);
Nh = Nxh*Nyh*Nzh;
Map.resize(Nx,Ny,Nz);
rank=Dm->rank();
iproc = Dm->iproc();
jproc = Dm->jproc();
kproc = Dm->kproc();
nprocx = Dm->nprocx();
nprocy = Dm->nprocy();
nprocz = Dm->nprocz();
rank_info = RankInfoStruct(rank,nprocx,nprocy,nprocz);
rank = rank_info.rank[1][1][1];
rank_X = rank_info.rank[2][1][1];
rank_x = rank_info.rank[0][1][1];
rank_Y = rank_info.rank[1][2][1];
rank_y = rank_info.rank[1][0][1];
rank_Z = rank_info.rank[1][1][2];
rank_z = rank_info.rank[1][1][0];
rank_XY = rank_info.rank[2][2][1];
rank_xy = rank_info.rank[0][0][1];
rank_Xy = rank_info.rank[2][0][1];
rank_xY = rank_info.rank[0][2][1];
rank_XZ = rank_info.rank[2][1][2];
rank_xz = rank_info.rank[0][1][0];
rank_Xz = rank_info.rank[2][1][0];
rank_xZ = rank_info.rank[0][1][2];
rank_YZ = rank_info.rank[1][2][2];
rank_yz = rank_info.rank[1][0][0];
rank_Yz = rank_info.rank[1][2][0];
rank_yZ = rank_info.rank[1][0][2];
rank_XYz = rank_info.rank[2][2][0];
rank_xyz = rank_info.rank[0][0][0];
rank_Xyz = rank_info.rank[2][0][0];
rank_xYz = rank_info.rank[0][2][0];
rank_XYZ = rank_info.rank[2][2][2];
rank_xyZ = rank_info.rank[0][0][2];
rank_XyZ = rank_info.rank[2][0][2];
rank_xYZ = rank_info.rank[0][2][2];
MPI_COMM_SCALBL.barrier();
/* Fill in communications patterns for the lists */
/* Send lists */
sendCount_x =getHaloBlock(width,2*width,width,Nyh-width,width,Nzh-width,dvcSendList_x);
sendCount_X =getHaloBlock(Nxh-2*width,Nxh-width,width,Nyh-width,width,Nzh-width,dvcSendList_X);
sendCount_y =getHaloBlock(width,Nxh-width,width,2*width,width,Nzh-width,dvcSendList_y);
sendCount_Y =getHaloBlock(width,Nxh-width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_Y);
sendCount_z =getHaloBlock(width,Nxh-width,width,Nyh-width,width,2*width,dvcSendList_z);
sendCount_Z =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_Z);
// xy
sendCount_xy =getHaloBlock(width,2*width,width,2*width,width,Nzh-width,dvcSendList_xy);
sendCount_xY =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_xY);
sendCount_Xy =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,width,Nzh-width,dvcSendList_Xy);
sendCount_XY =getHaloBlock(Nxh-2*width,Nxh-width,Nyh-2*width,Nyh-width,width,Nzh-width,dvcSendList_XY);
// xz
sendCount_xz =getHaloBlock(width,2*width,width,Nyh-width,width,2*width,dvcSendList_xz);
sendCount_xZ =getHaloBlock(width,2*width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xZ);
sendCount_Xz =getHaloBlock(Nxh-2*width,Nxh-width,width,Nyh-width,width,2*width,dvcSendList_Xz);
sendCount_XZ =getHaloBlock(Nxh-2*width,Nxh-width,width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_XZ);
// yz
sendCount_yz =getHaloBlock(width,Nxh-width,width,2*width,width,2*width,dvcSendList_yz);
sendCount_yZ =getHaloBlock(width,Nxh-width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_yZ);
sendCount_Yz =getHaloBlock(width,Nxh-width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_Yz);
sendCount_YZ =getHaloBlock(width,Nxh-width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_YZ);
// xyz
sendCount_xyz =getHaloBlock(width,2*width,width,2*width,width,2*width,dvcSendList_xyz);
sendCount_xyZ =getHaloBlock(width,2*width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_xyZ);
sendCount_xYz =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_xYz);
sendCount_xYZ =getHaloBlock(width,2*width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_xYZ);
sendCount_Xyz =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,width,2*width,dvcSendList_Xyz);
sendCount_XyZ =getHaloBlock(Nxh-2*width,Nxh-width,width,2*width,Nzh-2*width,Nzh-width,dvcSendList_XyZ);
sendCount_XYz =getHaloBlock(Nxh-2*width,Nxh-width,Nyh-2*width,Nyh-width,width,2*width,dvcSendList_XYz);
sendCount_XYZ =getHaloBlock(Nxh-2*width,Nxh-width,Nyh-2*width,Nyh-width,Nzh-2*width,Nzh-width,dvcSendList_XYZ);
/* Recv lists */
recvCount_x =getHaloBlock(0,width,width,Nyh-width,width,Nzh-width,dvcRecvList_x);
recvCount_X =getHaloBlock(Nxh-width,Nxh,width,Nyh-width,width,Nzh-width,dvcRecvList_X);
recvCount_y =getHaloBlock(width,Nxh-width,0,width,width,Nzh-width,dvcRecvList_y);
recvCount_Y =getHaloBlock(width,Nxh-width,Nyh-width,Nyh,width,Nzh-width,dvcRecvList_Y);
recvCount_z =getHaloBlock(width,Nxh-width,width,Nyh-width,0,width,dvcRecvList_z);
recvCount_Z =getHaloBlock(width,Nxh-width,width,Nyh-width,Nzh-width,Nzh,dvcRecvList_Z);
//xy
recvCount_xy =getHaloBlock(0,width,0,width,width,Nzh-width,dvcRecvList_xy);
recvCount_xY =getHaloBlock(0,width,Nyh-width,Nyh,width,Nzh-width,dvcRecvList_xY);
recvCount_Xy =getHaloBlock(Nxh-width,Nxh,0,width,width,Nzh-width,dvcRecvList_Xy);
recvCount_XY =getHaloBlock(Nxh-width,Nxh,Nyh-width,Nyh,width,Nzh-width,dvcRecvList_XY);
//xz
recvCount_xz =getHaloBlock(0,width,width,Nyh-width,0,width,dvcRecvList_xz);
recvCount_xZ =getHaloBlock(0,width,width,Nyh-width,Nzh-width,Nzh,dvcRecvList_xZ);
recvCount_Xz =getHaloBlock(Nxh-width,Nxh,width,Nyh-width,0,width,dvcRecvList_Xz);
recvCount_XZ =getHaloBlock(Nxh-width,Nxh,width,Nyh-width,Nzh-width,Nzh,dvcRecvList_XZ);
//yz
recvCount_yz =getHaloBlock(width,Nxh-width,0,width,0,width,dvcRecvList_yz);
recvCount_yZ =getHaloBlock(width,Nxh-width,0,width,Nzh-width,Nzh,dvcRecvList_yZ);
recvCount_Yz =getHaloBlock(width,Nxh-width,Nyh-width,Nyh,0,width,dvcRecvList_Yz);
recvCount_YZ =getHaloBlock(width,Nxh-width,Nyh-width,Nyh,Nzh-width,Nzh,dvcRecvList_YZ);
//xyz
recvCount_xyz =getHaloBlock(0,width,0,width,0,width,dvcRecvList_xyz);
recvCount_xyZ =getHaloBlock(0,width,0,width,Nzh-width,Nzh,dvcRecvList_xyZ);
recvCount_xYz =getHaloBlock(0,width,Nyh-width,Nyh,0,width,dvcRecvList_xYz);
recvCount_xYZ =getHaloBlock(0,width,Nyh-width,Nyh,Nzh-width,Nzh,dvcRecvList_xYZ);
recvCount_Xyz =getHaloBlock(Nxh-width,Nxh,0,width,0,width,dvcRecvList_Xyz);
recvCount_XyZ =getHaloBlock(Nxh-width,Nxh,0,width,Nzh-width,Nzh,dvcRecvList_XyZ);
recvCount_XYz =getHaloBlock(Nxh-width,Nxh,Nyh-width,Nyh,0,width,dvcRecvList_XYz);
recvCount_XYZ =getHaloBlock(Nxh-width,Nxh,Nyh-width,Nyh,Nzh-width,Nzh,dvcRecvList_XYZ);
//......................................................................................
ScaLBL_AllocateZeroCopy((void **) &sendbuf_x, sendCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_X, sendCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_y, sendCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Y, sendCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_z, sendCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Z, sendCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xy, sendCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xY, sendCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xy, sendCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XY, sendCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xz, sendCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xZ, sendCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xz, sendCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XZ, sendCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yz, sendCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_yZ, sendCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Yz, sendCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_YZ, sendCount_YZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xyz, sendCount_xyz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xYz, sendCount_xYz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_Xyz, sendCount_Xyz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XYz, sendCount_XYz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xyZ, sendCount_xyZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_xYZ, sendCount_xYZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XyZ, sendCount_XyZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &sendbuf_XYZ, sendCount_XYZ*sizeof(double)); // Allocate device memory
//......................................................................................
ScaLBL_AllocateZeroCopy((void **) &recvbuf_x, recvCount_x*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_X, recvCount_X*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_y, recvCount_y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Y, recvCount_Y*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_z, recvCount_z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Z, recvCount_Z*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xy, recvCount_xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xY, recvCount_xY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xy, recvCount_Xy*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XY, recvCount_XY*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xz, recvCount_xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xZ, recvCount_xZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xz, recvCount_Xz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XZ, recvCount_XZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yz, recvCount_yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_yZ, recvCount_yZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Yz, recvCount_Yz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_YZ, recvCount_YZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xyz, recvCount_xyz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xYz, recvCount_xYz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_Xyz, recvCount_Xyz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XYz, recvCount_XYz*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xyZ, recvCount_xyZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_xYZ, recvCount_xYZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XyZ, recvCount_XyZ*sizeof(double)); // Allocate device memory
ScaLBL_AllocateZeroCopy((void **) &recvbuf_XYZ, recvCount_XYZ*sizeof(double)); // Allocate device memory
/* Set up a map to the halo width=1 data structure */
for (k=width; k<Nzh-width; k++){
for (j=width; j<Nyh-width; j++){
for (i=width; i<Nxh-width; i++){
int idx = k*Nxh*Nyh + j*Nxh + i;
Map(i-width+1,j-width+1,k-width+1) = idx;
}
}
}
}
void ScaLBLWideHalo_Communicator::Send(double *data){
//...................................................................................
if (Lock==true){
ERROR("ScaLBL Error (SendHalo): ScaLBLWideHalo_Communicator is locked -- did you forget to match Send/Recv calls?");
}
else{
Lock=true;
}
ScaLBL_DeviceBarrier();
//...................................................................................
sendtag = recvtag = 1;
//...................................................................................
ScaLBL_Scalar_Pack(dvcSendList_x, sendCount_x,sendbuf_x, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_y, sendCount_y,sendbuf_y, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_z, sendCount_z,sendbuf_z, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_X, sendCount_X,sendbuf_X, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Y, sendCount_Y,sendbuf_Y, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Z, sendCount_Z,sendbuf_Z, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xy, sendCount_xy,sendbuf_xy, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xY, sendCount_xY,sendbuf_xY, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Xy, sendCount_Xy,sendbuf_Xy, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_XY, sendCount_XY,sendbuf_XY, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xz, sendCount_xz,sendbuf_xz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xZ, sendCount_xZ,sendbuf_xZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Xz, sendCount_Xz,sendbuf_Xz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_XZ, sendCount_XZ,sendbuf_XZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_yz, sendCount_yz,sendbuf_yz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_yZ, sendCount_yZ,sendbuf_yZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Yz, sendCount_Yz,sendbuf_Yz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_YZ, sendCount_YZ,sendbuf_YZ, data, Nh);
/* corners */
ScaLBL_Scalar_Pack(dvcSendList_xyz, sendCount_xyz,sendbuf_xyz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xyZ, sendCount_xyZ,sendbuf_xyZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xYz, sendCount_xYz,sendbuf_xYz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_xYZ, sendCount_xYZ,sendbuf_xYZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_Xyz, sendCount_Xyz,sendbuf_Xyz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_XyZ, sendCount_XyZ,sendbuf_XyZ, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_XYz, sendCount_XYz,sendbuf_XYz, data, Nh);
ScaLBL_Scalar_Pack(dvcSendList_XYZ, sendCount_XYZ,sendbuf_XYZ, data, Nh);
//...................................................................................
// Send / Recv all the phase indcator field values
//...................................................................................
req1[0] = MPI_COMM_SCALBL.Isend(sendbuf_x,sendCount_x,rank_x,sendtag+0);
req2[0] = MPI_COMM_SCALBL.Irecv(recvbuf_X,recvCount_X,rank_X,recvtag+0);
req1[1] = MPI_COMM_SCALBL.Isend(sendbuf_X,sendCount_X,rank_X,sendtag+1);
req2[1] = MPI_COMM_SCALBL.Irecv(recvbuf_x,recvCount_x,rank_x,recvtag+1);
req1[2] = MPI_COMM_SCALBL.Isend(sendbuf_y,sendCount_y,rank_y,sendtag+2);
req2[2] = MPI_COMM_SCALBL.Irecv(recvbuf_Y,recvCount_Y,rank_Y,recvtag+2);
req1[3] = MPI_COMM_SCALBL.Isend(sendbuf_Y,sendCount_Y,rank_Y,sendtag+3);
req2[3] = MPI_COMM_SCALBL.Irecv(recvbuf_y,recvCount_y,rank_y,recvtag+3);
req1[4] = MPI_COMM_SCALBL.Isend(sendbuf_z,sendCount_z,rank_z,sendtag+4);
req2[4] = MPI_COMM_SCALBL.Irecv(recvbuf_Z,recvCount_Z,rank_Z,recvtag+4);
req1[5] = MPI_COMM_SCALBL.Isend(sendbuf_Z,sendCount_Z,rank_Z,sendtag+5);
req2[5] = MPI_COMM_SCALBL.Irecv(recvbuf_z,recvCount_z,rank_z,recvtag+5);
req1[6] = MPI_COMM_SCALBL.Isend(sendbuf_xy,sendCount_xy,rank_xy,sendtag+6);
req2[6] = MPI_COMM_SCALBL.Irecv(recvbuf_XY,recvCount_XY,rank_XY,recvtag+6);
req1[7] = MPI_COMM_SCALBL.Isend(sendbuf_XY,sendCount_XY,rank_XY,sendtag+7);
req2[7] = MPI_COMM_SCALBL.Irecv(recvbuf_xy,recvCount_xy,rank_xy,recvtag+7);
req1[8] = MPI_COMM_SCALBL.Isend(sendbuf_Xy,sendCount_Xy,rank_Xy,sendtag+8);
req2[8] = MPI_COMM_SCALBL.Irecv(recvbuf_xY,recvCount_xY,rank_xY,recvtag+8);
req1[9] = MPI_COMM_SCALBL.Isend(sendbuf_xY,sendCount_xY,rank_xY,sendtag+9);
req2[9] = MPI_COMM_SCALBL.Irecv(recvbuf_Xy,recvCount_Xy,rank_Xy,recvtag+9);
req1[10] = MPI_COMM_SCALBL.Isend(sendbuf_xz,sendCount_xz,rank_xz,sendtag+10);
req2[10] = MPI_COMM_SCALBL.Irecv(recvbuf_XZ,recvCount_XZ,rank_XZ,recvtag+10);
req1[11] = MPI_COMM_SCALBL.Isend(sendbuf_XZ,sendCount_XZ,rank_XZ,sendtag+11);
req2[11] = MPI_COMM_SCALBL.Irecv(recvbuf_xz,recvCount_xz,rank_xz,recvtag+11);
req1[12] = MPI_COMM_SCALBL.Isend(sendbuf_Xz,sendCount_Xz,rank_Xz,sendtag+12);
req2[12] = MPI_COMM_SCALBL.Irecv(recvbuf_xZ,recvCount_xZ,rank_xZ,recvtag+12);
req1[13] = MPI_COMM_SCALBL.Isend(sendbuf_xZ,sendCount_xZ,rank_xZ,sendtag+13);
req2[13] = MPI_COMM_SCALBL.Irecv(recvbuf_Xz,recvCount_Xz,rank_Xz,recvtag+13);
req1[14] = MPI_COMM_SCALBL.Isend(sendbuf_yz,sendCount_yz,rank_yz,sendtag+14);
req2[14] = MPI_COMM_SCALBL.Irecv(recvbuf_YZ,recvCount_YZ,rank_YZ,recvtag+14);
req1[15] = MPI_COMM_SCALBL.Isend(sendbuf_YZ,sendCount_YZ,rank_YZ,sendtag+15);
req2[15] = MPI_COMM_SCALBL.Irecv(recvbuf_yz,recvCount_yz,rank_yz,recvtag+15);
req1[16] = MPI_COMM_SCALBL.Isend(sendbuf_Yz,sendCount_Yz,rank_Yz,sendtag+16);
req2[16] = MPI_COMM_SCALBL.Irecv(recvbuf_yZ,recvCount_yZ,rank_yZ,recvtag+16);
req1[17] = MPI_COMM_SCALBL.Isend(sendbuf_yZ,sendCount_yZ,rank_yZ,sendtag+17);
req2[17] = MPI_COMM_SCALBL.Irecv(recvbuf_Yz,recvCount_Yz,rank_Yz,recvtag+17);
/* Corners */
req1[18] = MPI_COMM_SCALBL.Isend(sendbuf_xyz,sendCount_xyz,rank_xyz,sendtag+18);
req2[18] = MPI_COMM_SCALBL.Irecv(recvbuf_XYZ,recvCount_XYZ,rank_XYZ,recvtag+18);
req1[19] = MPI_COMM_SCALBL.Isend(sendbuf_XYz,sendCount_XYz,rank_XYz,sendtag+19);
req2[19] = MPI_COMM_SCALBL.Irecv(recvbuf_xyZ,recvCount_xyZ,rank_xyZ,recvtag+19);
req1[20] = MPI_COMM_SCALBL.Isend(sendbuf_Xyz,sendCount_Xyz,rank_Xyz,sendtag+20);
req2[20] = MPI_COMM_SCALBL.Irecv(recvbuf_xYZ,recvCount_xYZ,rank_xYZ,recvtag+20);
req1[21] = MPI_COMM_SCALBL.Isend(sendbuf_xYz,sendCount_xYz,rank_xYz,sendtag+21);
req2[21] = MPI_COMM_SCALBL.Irecv(recvbuf_XyZ,recvCount_XyZ,rank_XyZ,recvtag+21);
req1[22] = MPI_COMM_SCALBL.Isend(sendbuf_xyZ,sendCount_xyZ,rank_xyZ,sendtag+22);
req2[22] = MPI_COMM_SCALBL.Irecv(recvbuf_XYz,recvCount_XYz,rank_XYz,recvtag+22);
req1[23] = MPI_COMM_SCALBL.Isend(sendbuf_XYZ,sendCount_XYZ,rank_XYZ,sendtag+23);
req2[23] = MPI_COMM_SCALBL.Irecv(recvbuf_xyz,recvCount_xyz,rank_xyz,recvtag+23);
req1[24] = MPI_COMM_SCALBL.Isend(sendbuf_XyZ,sendCount_XyZ,rank_XyZ,sendtag+24);
req2[24] = MPI_COMM_SCALBL.Irecv(recvbuf_xYz,recvCount_xYz,rank_xYz,recvtag+24);
req1[25] = MPI_COMM_SCALBL.Isend(sendbuf_xYZ,sendCount_xYZ,rank_xYZ,sendtag+25);
req2[25] = MPI_COMM_SCALBL.Irecv(recvbuf_Xyz,recvCount_Xyz,rank_Xyz,recvtag+25);
//...................................................................................
}
ScaLBLWideHalo_Communicator::~ScaLBLWideHalo_Communicator()
{
}
void ScaLBLWideHalo_Communicator::Recv(double *data){
//...................................................................................
Utilities::MPI::waitAll(26,req1);
Utilities::MPI::waitAll(26,req2);
ScaLBL_DeviceBarrier();
//...................................................................................
//printf("Ready to unpack %i to x\n",recvCount_x);
//printf(" print first 10 values...\n");
//for (int idx=0; idx<10; idx++) printf(" recvBuf[%i]=%f \n",idx,recvbuf_x[idx]);
ScaLBL_Scalar_Unpack(dvcRecvList_x, recvCount_x,recvbuf_x, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_y, recvCount_y,recvbuf_y, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_X, recvCount_X,recvbuf_X, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Y, recvCount_Y,recvbuf_Y, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xy, recvCount_xy,recvbuf_xy, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xY, recvCount_xY,recvbuf_xY, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Xy, recvCount_Xy,recvbuf_Xy, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_XY, recvCount_XY,recvbuf_XY, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_z, recvCount_z,recvbuf_z, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xz, recvCount_xz,recvbuf_xz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Xz, recvCount_Xz,recvbuf_Xz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_yz, recvCount_yz,recvbuf_yz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Yz, recvCount_Yz,recvbuf_Yz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Z, recvCount_Z,recvbuf_Z, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xZ, recvCount_xZ,recvbuf_xZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_XZ, recvCount_XZ,recvbuf_XZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_yZ, recvCount_yZ,recvbuf_yZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_YZ, recvCount_YZ,recvbuf_YZ, data, Nh);
/* corners */
ScaLBL_Scalar_Unpack(dvcRecvList_xyz, recvCount_xyz,recvbuf_xyz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xYz, recvCount_xYz,recvbuf_xYz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xyZ, recvCount_xyZ,recvbuf_xyZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_xYZ, recvCount_xYZ,recvbuf_xYZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_Xyz, recvCount_Xyz,recvbuf_Xyz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_XYz, recvCount_XYz,recvbuf_XYz, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_XyZ, recvCount_XyZ,recvbuf_XyZ, data, Nh);
ScaLBL_Scalar_Unpack(dvcRecvList_XYZ, recvCount_XYZ,recvbuf_XYZ, data, Nh);
//...................................................................................
Lock=false; // unlock the communicator after communications complete
//...................................................................................
}

115
common/WideHalo.h Normal file
View File

@ -0,0 +1,115 @@
/*
This class implements support for halo widths larger than 1
*/
#ifndef WideHalo_H
#define WideHalo_H
#include "common/ScaLBL.h"
#include "common/MPI.h"
class ScaLBLWideHalo_Communicator{
public:
//......................................................................................
ScaLBLWideHalo_Communicator(std::shared_ptr <Domain> Dm, int width);
~ScaLBLWideHalo_Communicator();
//......................................................................................
//MPI_Comm MPI_COMM_SCALBL; // MPI Communicator
Utilities::MPI MPI_COMM_SCALBL;
unsigned long int CommunicationCount,SendCount,RecvCount;
int Nx,Ny,Nz,N; // original domain structure
int Nxh,Nyh,Nzh,Nh; // with wide halo
DoubleArray Map; // map to regular halo
int first_interior,last_interior;
//......................................................................................
// Set up for D3Q19 distributions -- all 27 neighbors are needed
//......................................................................................
// Buffers to store data sent and recieved by this MPI process
double *sendbuf_x, *sendbuf_y, *sendbuf_z, *sendbuf_X, *sendbuf_Y, *sendbuf_Z;
double *sendbuf_xy, *sendbuf_yz, *sendbuf_xz, *sendbuf_Xy, *sendbuf_Yz, *sendbuf_xZ;
double *sendbuf_xY, *sendbuf_yZ, *sendbuf_Xz, *sendbuf_XY, *sendbuf_YZ, *sendbuf_XZ;
double *sendbuf_xyz, *sendbuf_Xyz, *sendbuf_xYz, *sendbuf_XYz;
double *sendbuf_xyZ, *sendbuf_XyZ, *sendbuf_xYZ, *sendbuf_XYZ;
double *recvbuf_x, *recvbuf_y, *recvbuf_z, *recvbuf_X, *recvbuf_Y, *recvbuf_Z;
double *recvbuf_xy, *recvbuf_yz, *recvbuf_xz, *recvbuf_Xy, *recvbuf_Yz, *recvbuf_xZ;
double *recvbuf_xY, *recvbuf_yZ, *recvbuf_Xz, *recvbuf_XY, *recvbuf_YZ, *recvbuf_XZ;
double *recvbuf_xyz, *recvbuf_Xyz, *recvbuf_xYz, *recvbuf_XYz;
double *recvbuf_xyZ, *recvbuf_XyZ, *recvbuf_xYZ, *recvbuf_XYZ;
//......................................................................................
int LastExterior();
int FirstInterior();
int LastInterior();
void Send(double *data);
void Recv(double *data);
// Debugging and unit testing functions
void PrintDebug();
private:
bool Lock; // use Lock to make sure only one call at a time to protect data in transit
// only one set of Send requests can be active at any time (per instance)
int i,j,k,n;
int iproc,jproc,kproc;
int nprocx,nprocy,nprocz;
int sendtag,recvtag;
// Give the object it's own MPI communicator
RankInfoStruct rank_info;
MPI_Request req1[26],req2[26];
//......................................................................................
// MPI ranks for all 18 neighbors
//......................................................................................
// These variables are all private to prevent external things from modifying them!!
//......................................................................................
int rank;
int rank_x,rank_y,rank_z,rank_X,rank_Y,rank_Z;
int rank_xy,rank_XY,rank_xY,rank_Xy;
int rank_xz,rank_XZ,rank_xZ,rank_Xz;
int rank_yz,rank_YZ,rank_yZ,rank_Yz;
int rank_xyz,rank_Xyz,rank_xYz,rank_XYz;
int rank_xyZ,rank_XyZ,rank_xYZ,rank_XYZ;
//......................................................................................
//......................................................................................
int sendCount_x, sendCount_y, sendCount_z, sendCount_X, sendCount_Y, sendCount_Z;
int sendCount_xy, sendCount_yz, sendCount_xz, sendCount_Xy, sendCount_Yz, sendCount_xZ;
int sendCount_xY, sendCount_yZ, sendCount_Xz, sendCount_XY, sendCount_YZ, sendCount_XZ;
int sendCount_xyz,sendCount_Xyz,sendCount_xYz,sendCount_XYz;
int sendCount_xyZ,sendCount_XyZ,sendCount_xYZ,sendCount_XYZ;
//......................................................................................
int recvCount_x, recvCount_y, recvCount_z, recvCount_X, recvCount_Y, recvCount_Z;
int recvCount_xy, recvCount_yz, recvCount_xz, recvCount_Xy, recvCount_Yz, recvCount_xZ;
int recvCount_xY, recvCount_yZ, recvCount_Xz, recvCount_XY, recvCount_YZ, recvCount_XZ;
int recvCount_xyz,recvCount_Xyz,recvCount_xYz,recvCount_XYz;
int recvCount_xyZ,recvCount_XyZ,recvCount_xYZ,recvCount_XYZ;
//......................................................................................
// Send buffers that reside on the compute device
int *dvcSendList_x, *dvcSendList_y, *dvcSendList_z, *dvcSendList_X, *dvcSendList_Y, *dvcSendList_Z;
int *dvcSendList_xy, *dvcSendList_yz, *dvcSendList_xz, *dvcSendList_Xy, *dvcSendList_Yz, *dvcSendList_xZ;
int *dvcSendList_xY, *dvcSendList_yZ, *dvcSendList_Xz, *dvcSendList_XY, *dvcSendList_YZ, *dvcSendList_XZ;
int *dvcSendList_xyz,*dvcSendList_Xyz,*dvcSendList_xYz,*dvcSendList_XYz;
int *dvcSendList_xyZ,*dvcSendList_XyZ,*dvcSendList_xYZ,*dvcSendList_XYZ;
// Recieve buffers that reside on the compute device
int *dvcRecvList_x, *dvcRecvList_y, *dvcRecvList_z, *dvcRecvList_X, *dvcRecvList_Y, *dvcRecvList_Z;
int *dvcRecvList_xy, *dvcRecvList_yz, *dvcRecvList_xz, *dvcRecvList_Xy, *dvcRecvList_Yz, *dvcRecvList_xZ;
int *dvcRecvList_xY, *dvcRecvList_yZ, *dvcRecvList_Xz, *dvcRecvList_XY, *dvcRecvList_YZ, *dvcRecvList_XZ;
int *dvcRecvList_xyz,*dvcRecvList_Xyz,*dvcRecvList_xYz,*dvcRecvList_XYz;
int *dvcRecvList_xyZ,*dvcRecvList_XyZ,*dvcRecvList_xYZ,*dvcRecvList_XYZ;
//......................................................................................
inline int getHaloBlock(int imin, int imax, int jmin, int jmax, int kmin, int kmax, int *& dvcList){
int count = 0;
int *List;
List = new int [(imax-imin)*(jmax-jmin)*(kmax-kmin)];
for (k=kmin; k<kmax; k++){
for (j=jmin; j<jmax; j++){
for (i=imin; i<imax; i++){
List[count++] = k*Nxh*Nyh + j*Nxh + i;
}
}
}
size_t numbytes=count*sizeof(int);
ScaLBL_AllocateZeroCopy((void **) &dvcList, numbytes); // Allocate device memory
ScaLBL_CopyToZeroCopy(dvcList,List,numbytes);
return count;
}
};
#endif

194
cpu/Color.cpp
View File

@ -2505,10 +2505,200 @@ extern "C" void ScaLBL_D3Q19_AAodd_Color(int *neighborList, int *Map, double *di
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Color(int *neighborList, int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
int nr1,nr2,nr3,nr4,nr5,nr6;
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* neighbors */
nr1 = neighborList[n+0*Np];
nr2 = neighborList[n+1*Np];
nr3 = neighborList[n+2*Np];
nr4 = neighborList[n+3*Np];
nr5 = neighborList[n+4*Np];
nr6 = neighborList[n+5*Np];
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
// compute phase indicator field
phi=(nA-nB)/(nA+nB);
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
// q = 1
//nread = neighborList[n+Np];
Aq[nr2] = a1;
Bq[nr2] = b1;
// q=2
//nread = neighborList[n];
Aq[nr1] = a2;
Bq[nr1] = b2;
//...............................................
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
// q = 3
//nread = neighborList[n+3*Np];
Aq[nr4] = a1;
Bq[nr4] = b1;
// q = 4
//nread = neighborList[n+2*Np];
Aq[nr3] = a2;
Bq[nr3] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
// q = 5
//nread = neighborList[n+5*Np];
Aq[nr6] = a1;
Bq[nr6] = b1;
// q = 6
//nread = neighborList[n+4*Np];
Aq[nr5] = a2;
Bq[nr5] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Color(int *Map, double *Aq, double *Bq, double *Den,
double *Phi, double *ColorGrad, double *Vel, double rhoA, double rhoB, double beta, int start, int finish, int Np){
double nA,nB; // number density
double a1,b1,a2,b2,nAB,delta;
double C,nx,ny,nz; //color gradient magnitude and direction
double ux,uy,uz;
double phi;
// Instantiate mass transport distributions
// Stationary value - distribution 0
for (int n=start; n<finish; n++){
/* load velocity */
ux = Vel[n];
uy = Vel[Np+n];
uz = Vel[2*Np+n];
/* load color gradient */
nx = ColorGrad[n];
ny = ColorGrad[Np+n];
nz = ColorGrad[2*Np+n];
C = sqrt(nx*nx+ny*ny+nz*nz);
double ColorMag = C;
if (C==0.0) ColorMag=1.0;
nx = nx/ColorMag;
ny = ny/ColorMag;
nz = nz/ColorMag;
// read the component number densities
nA = Den[n];
nB = Den[Np + n];
nAB = 1.0/(nA+nB);
Aq[n] = 0.3333333333333333*nA;
Bq[n] = 0.3333333333333333*nB;
//...............................................
// q = 0,2,4
// Cq = {1,0,0}, {0,1,0}, {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nx;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*ux))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*ux))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*ux))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*ux))+delta;
Aq[1*Np+n] = a1;
Bq[1*Np+n] = b1;
Aq[2*Np+n] = a2;
Bq[2*Np+n] = b2;
//...............................................
// q = 2
// Cq = {0,1,0}
delta = beta*nA*nB*nAB*0.1111111111111111*ny;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uy))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uy))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uy))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uy))+delta;
Aq[3*Np+n] = a1;
Bq[3*Np+n] = b1;
Aq[4*Np+n] = a2;
Bq[4*Np+n] = b2;
//...............................................
// q = 4
// Cq = {0,0,1}
delta = beta*nA*nB*nAB*0.1111111111111111*nz;
if (!(nA*nB*nAB>0)) delta=0;
a1 = nA*(0.1111111111111111*(1+4.5*uz))+delta;
b1 = nB*(0.1111111111111111*(1+4.5*uz))-delta;
a2 = nA*(0.1111111111111111*(1-4.5*uz))-delta;
b2 = nB*(0.1111111111111111*(1-4.5*uz))+delta;
Aq[5*Np+n] = a1;
Bq[5*Np+n] = b1;
Aq[6*Np+n] = a2;
Bq[6*Np+n] = b2;
//...............................................
}
}
extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double *Aq, double *Bq,
double *Den, double *Phi, int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){
@ -2595,7 +2785,7 @@ extern "C" void ScaLBL_D3Q7_AAodd_PhaseField(int *neighborList, int *Map, double
extern "C" void ScaLBL_D3Q7_AAeven_PhaseField(int *Map, double *Aq, double *Bq, double *Den, double *Phi,
int start, int finish, int Np){
int idx,n,nread;
int idx,nread;
double fq,nA,nB;
for (int n=start; n<finish; n++){

347
cpu/D3Q7BC.cpp Normal file
View File

@ -0,0 +1,347 @@
// CPU Functions for D3Q7 Lattice Boltzmann Methods
// Boundary Conditions
extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
int idx;
int iq,ib;
double value_b,value_q;
for (idx=0; idx<N; idx++){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
}
}
extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist,double *BoundaryValue,int *BounceBackDist_list,int *BounceBackSolid_list,int N){
int idx;
int iq,ib;
double value_b,value_q;
for (idx=0; idx<N; idx++){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = value_q + value_b;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f6 = dist[5*Np+n];
//...................................................
double f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f5 = dist[6*Np+n];
//...................................................
double f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
int nread,nr5;
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
nread = d_neighborList[n];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
double f3 = dist[nread];
nread = d_neighborList[n+Np];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
double f4 = dist[nread];
nread = d_neighborList[n+5*Np];
double f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
double f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
int nread,nr6;
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
nread = d_neighborList[n];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
double f3 = dist[nread];
nread = d_neighborList[n+4*Np];
double f5 = dist[nread];
nread = d_neighborList[n+Np];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
double f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
double f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
{
int idx,n,nm;
for (idx=0; idx<count; idx++){
n = list[idx];
nm = Map[n];
Psi[nm] = Vin;
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
{
int idx,n,nm;
for (idx=0; idx<count; idx++){
n = list[idx];
nm = Map[n];
Psi[nm] = Vout;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f6 = dist[5*Np+n];
//...................................................
double f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
double f1 = dist[2*Np+n];
double f2 = dist[1*Np+n];
double f3 = dist[4*Np+n];
double f4 = dist[3*Np+n];
double f5 = dist[6*Np+n];
//...................................................
double f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
int nread,nr5;
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
nread = d_neighborList[n];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
double f3 = dist[nread];
nread = d_neighborList[n+Np];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
double f4 = dist[nread];
nread = d_neighborList[n+5*Np];
double f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
double f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
int nread,nr6;
for (int idx=0; idx<count; idx++){
int n = list[idx];
double f0 = dist[n];
nread = d_neighborList[n];
double f1 = dist[nread];
nread = d_neighborList[n+2*Np];
double f3 = dist[nread];
nread = d_neighborList[n+4*Np];
double f5 = dist[nread];
nread = d_neighborList[n+Np];
double f2 = dist[nread];
nread = d_neighborList[n+3*Np];
double f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
double f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
//NOTE: FluxIn is the inward flux
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
int n;
double uz;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
dist[6*Np+n] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
//NOTE: FluxIn is the inward flux
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
int n;
double uz;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
dist[5*Np+n] = f6;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
//NOTE: FluxIn is the inward flux
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
int n;
int nread,nr5;
double uz;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
dist[nr5] = f5;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
//NOTE: FluxIn is the inward flux
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
int n;
int nread,nr6;
double uz;
for (int idx=0; idx<count; idx++){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
// unknown distributions
nr6 = d_neighborList[n+5*Np];
dist[nr6] = f6;
}
}

2158
cpu/FreeLee.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

6
cpu/Greyscale.cpp
View File

@ -1974,7 +1974,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_Greyscale_MRT(int *neighborList, double *dist
}
//Calculate pressure for MRT model
pressure=rho/3.f/porosity;
//pressure=rho/3.f/porosity;
pressure=rho/3.f;
//-------------------- MRT collison where body force has NO higher-order terms -------------//
m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);
@ -2472,7 +2473,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_Greyscale_MRT(double *dist, int start, int f
}
//Calculate pressure for Incompressible-MRT model
pressure=rho/3.f/porosity;
//pressure=rho/3.f/porosity;
pressure=rho/3.f;
//-------------------- IMRT collison where body force has NO higher-order terms -------------//
m1 = m1 + rlx_setA*((19*(ux*ux+uy*uy+uz*uz)*rho0/porosity - 11*rho) - m1);

32
cpu/GreyscaleColor.cpp
View File

@ -494,7 +494,8 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, d
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
Pressure[n] = rho/3.f/porosity;
//Pressure[n] = rho/3.f/porosity;
Pressure[n] = rho/3.f;
//........................................................................
//..............carry out relaxation process..............................
@ -1149,7 +1150,8 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
Pressure[n] = rho/3.f/porosity;
//Pressure[n] = rho/3.f/porosity;
Pressure[n] = rho/3.f;
//........................................................................
//..............carry out relaxation process..............................
@ -1336,6 +1338,32 @@ extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, doubl
}
}
extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
int idx;
double nA,nB;
for (idx=start; idx<finish; idx++){
nA = Den[idx];
nB = Den[Np+idx];
Aq[idx]=0.3333333333333333*nA;
Aq[Np+idx]=0.1111111111111111*nA;
Aq[2*Np+idx]=0.1111111111111111*nA;
Aq[3*Np+idx]=0.1111111111111111*nA;
Aq[4*Np+idx]=0.1111111111111111*nA;
Aq[5*Np+idx]=0.1111111111111111*nA;
Aq[6*Np+idx]=0.1111111111111111*nA;
Bq[idx]=0.3333333333333333*nB;
Bq[Np+idx]=0.1111111111111111*nB;
Bq[2*Np+idx]=0.1111111111111111*nB;
Bq[3*Np+idx]=0.1111111111111111*nB;
Bq[4*Np+idx]=0.1111111111111111*nB;
Bq[5*Np+idx]=0.1111111111111111*nB;
Bq[6*Np+idx]=0.1111111111111111*nB;
}
}
//extern "C" void ScaLBL_D3Q19_GreyscaleColor_Init(double *dist, double *Porosity, int Np){
// int n;

254
cpu/Ion.cpp Normal file
View File

@ -0,0 +1,254 @@
#include <stdio.h>
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
int n,nread;
double fq,Ci;
for (n=start; n<finish; n++){
// q=0
fq = dist[n];
Ci = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
Ci += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
Ci += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
Ci += fq;
// q=4
nread = neighborList[n+3*Np];
fq = dist[nread];
Ci += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
Ci += fq;
// q=6
nread = neighborList[n+5*Np];
fq = dist[nread];
Ci += fq;
Den[n]=Ci;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
int n;
double fq,Ci;
for (n=start; n<finish; n++){
// q=0
fq = dist[n];
Ci = fq;
// q=1
fq = dist[2*Np+n];
Ci += fq;
// q=2
fq = dist[1*Np+n];
Ci += fq;
// q=3
fq = dist[4*Np+n];
Ci += fq;
// q=4
fq = dist[3*Np+n];
Ci += fq;
// q=5
fq = dist[6*Np+n];
Ci += fq;
// q=6
fq = dist[5*Np+n];
Ci += fq;
Den[n]=Ci;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
for (n=start; n<finish; n++){
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
// q=2
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q=4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q=6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
// q = 1
dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
// q=2
dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
// q = 3
dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
// q = 4
dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
// q = 5
dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
// q = 6
dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
for (n=start; n<finish; n++){
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
// q=2
dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
}
}
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np)
{
int n;
for (n=0; n<Np; n++){
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
Den[n] = DenInit;
}
}
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np)
{
int n;
double DenInit;
for (n=0; n<Np; n++){
DenInit = Den[n];
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
}
}
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
int n;
double Ci;//ion concentration of species i
double CD;//charge density
double CD_tmp;
double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
for (n=start; n<finish; n++){
Ci = Den[n+ion_component*Np];
CD = ChargeDensity[n];
CD_tmp = F*IonValence*Ci;
ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
}
}

48
cpu/MixedGradient.cpp Normal file
View File

@ -0,0 +1,48 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
for (int idx=start; idx<finish; idx++){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}

372
cpu/Poisson.cpp Normal file
View File

@ -0,0 +1,372 @@
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int nread;
int idx;
for (n=start; n<finish; n++){
// q=0
fq = dist[n];
psi = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
psi += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
psi += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
psi += fq;
// q = 4
nread = neighborList[n+3*Np];
fq = dist[nread];
psi += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
psi += fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int idx;
for (n=start; n<finish; n++){
// q=0
fq = dist[n];
psi = fq;
// q=1
fq = dist[2*Np+n];
psi += fq;
// q=2
fq = dist[1*Np+n];
psi += fq;
// q=3
fq = dist[4*Np+n];
psi += fq;
// q=4
fq = dist[3*Np+n];
psi += fq;
// q=5
fq = dist[6*Np+n];
psi += fq;
// q=6
fq = dist[5*Np+n];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
double rlx=1.0/tau;
int idx;
for (n=start; n<finish; n++){
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q = 4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q = 6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
double rlx=1.0/tau;
int idx;
for (n=start; n<finish; n++){
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np)
{
int n;
int ijk;
for (n=start; n<finish; n++){
ijk = Map[n];
dist[0*Np+n] = 0.25*Psi[ijk];
dist[1*Np+n] = 0.125*Psi[ijk];
dist[2*Np+n] = 0.125*Psi[ijk];
dist[3*Np+n] = 0.125*Psi[ijk];
dist[4*Np+n] = 0.125*Psi[ijk];
dist[5*Np+n] = 0.125*Psi[ijk];
dist[6*Np+n] = 0.125*Psi[ijk];
}
}
//extern "C" void ScaLBL_D3Q7_Poisson_ElectricField(int *neighborList, int *Map, signed char *ID, double *Psi, double *ElectricField, int SolidBC,
// int strideY, int strideZ,int start, int finish, int Np){
//
// int n,nn;
// int ijk;
// int id;
// // distributions
// double m1,m2,m3,m4,m5,m6,m7,m8,m9;
// double m10,m11,m12,m13,m14,m15,m16,m17,m18;
// double nx,ny,nz;
//
// for (n=start; n<finish; n++){
//
// // Get the 1D index based on regular data layout
// ijk = Map[n];
// // COMPUTE THE COLOR GRADIENT
// //........................................................................
// //.................Read Phase Indicator Values............................
// //........................................................................
// nn = ijk-1; // neighbor index (get convention)
// id = ID[nn];
// m1 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 1
// //........................................................................
// nn = ijk+1; // neighbor index (get convention)
// id = ID[nn];
// m2 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 2
// //........................................................................
// nn = ijk-strideY; // neighbor index (get convention)
// id = ID[nn];
// m3 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 3
// //........................................................................
// nn = ijk+strideY; // neighbor index (get convention)
// id = ID[nn];
// m4 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 4
// //........................................................................
// nn = ijk-strideZ; // neighbor index (get convention)
// id = ID[nn];
// m5 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 5
// //........................................................................
// nn = ijk+strideZ; // neighbor index (get convention)
// id = ID[nn];
// m6 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 6
// //........................................................................
// nn = ijk-strideY-1; // neighbor index (get convention)
// id = ID[nn];
// m7 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 7
// //........................................................................
// nn = ijk+strideY+1; // neighbor index (get convention)
// id = ID[nn];
// m8 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 8
// //........................................................................
// nn = ijk+strideY-1; // neighbor index (get convention)
// id = ID[nn];
// m9 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 9
// //........................................................................
// nn = ijk-strideY+1; // neighbor index (get convention)
// id = ID[nn];
// m10 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 10
// //........................................................................
// nn = ijk-strideZ-1; // neighbor index (get convention)
// id = ID[nn];
// m11 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 11
// //........................................................................
// nn = ijk+strideZ+1; // neighbor index (get convention)
// id = ID[nn];
// m12 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 12
// //........................................................................
// nn = ijk+strideZ-1; // neighbor index (get convention)
// id = ID[nn];
// m13 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 13
// //........................................................................
// nn = ijk-strideZ+1; // neighbor index (get convention)
// id = ID[nn];
// m14 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 14
// //........................................................................
// nn = ijk-strideZ-strideY; // neighbor index (get convention)
// id = ID[nn];
// m15 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 15
// //........................................................................
// nn = ijk+strideZ+strideY; // neighbor index (get convention)
// id = ID[nn];
// m16 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 16
// //........................................................................
// nn = ijk+strideZ-strideY; // neighbor index (get convention)
// id = ID[nn];
// m17 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 17
// //........................................................................
// nn = ijk-strideZ+strideY; // neighbor index (get convention)
// id = ID[nn];
// m18 = SolidBC==1 ? Psi[nn] : Psi[nn]*(id>0)+Psi[ijk]*(id<=0);// get neighbor for phi - 18
// //............Compute the Color Gradient...................................
// //nx = 1.f/6.f*(m1-m2+0.5*(m7-m8+m9-m10+m11-m12+m13-m14));
// //ny = 1.f/6.f*(m3-m4+0.5*(m7-m8-m9+m10+m15-m16+m17-m18));
// //nz = 1.f/6.f*(m5-m6+0.5*(m11-m12-m13+m14+m15-m16-m17+m18));
// nx = 1.f/6.f*(m1-m2);//but looks like it needs to multiply another factor of 3
// ny = 1.f/6.f*(m3-m4);
// nz = 1.f/6.f*(m5-m6);
//
// ElectricField[n] = nx;
// ElectricField[Np+n] = ny;
// ElectricField[2*Np+n] = nz;
// }
//}
//extern "C" void ScaLBL_D3Q7_Poisson_getElectricField(double *dist, double *ElectricField, double tau, int Np){
// int n;
// // distributions
// double f1,f2,f3,f4,f5,f6;
// double Ex,Ey,Ez;
// double rlx=1.0/tau;
//
// for (n=0; n<Np; n++){
// //........................................................................
// // Registers to store the distributions
// //........................................................................
// f1 = dist[Np+n];
// f2 = dist[2*Np+n];
// f3 = dist[3*Np+n];
// f4 = dist[4*Np+n];
// f5 = dist[5*Np+n];
// f6 = dist[6*Np+n];
// //.................Compute the Electric Field...................................
// //Ex = (f1-f2)*rlx*4.5;//NOTE the unit of electric field here is V/lu
// //Ey = (f3-f4)*rlx*4.5;
// //Ez = (f5-f6)*rlx*4.5;
// Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
// Ey = (f3-f4)*rlx*4.0;
// Ez = (f5-f6)*rlx*4.0;
// //..................Write the Electric Field.....................................
// ElectricField[0*Np+n] = Ex;
// ElectricField[1*Np+n] = Ey;
// ElectricField[2*Np+n] = Ez;
// //........................................................................
// }
//}

999
cpu/Stokes.cpp Normal file
View File

@ -0,0 +1,999 @@
#include <stdio.h>
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np)
{
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
constexpr double mrt_V1=0.05263157894736842;
constexpr double mrt_V2=0.012531328320802;
constexpr double mrt_V3=0.04761904761904762;
constexpr double mrt_V4=0.004594820384294068;
constexpr double mrt_V5=0.01587301587301587;
constexpr double mrt_V6=0.0555555555555555555555555;
constexpr double mrt_V7=0.02777777777777778;
constexpr double mrt_V8=0.08333333333333333;
constexpr double mrt_V9=0.003341687552213868;
constexpr double mrt_V10=0.003968253968253968;
constexpr double mrt_V11=0.01388888888888889;
constexpr double mrt_V12=0.04166666666666666;
for (int n=start; n<finish; n++){
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
fq = dist[2*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
fq = dist[1*Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
fq = dist[4*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
fq = dist[6*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
fq = dist[5*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
fq = dist[10*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
fq = dist[12*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
fq = dist[11*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
fq = dist[18*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//........................................................................
// READ THE DISTRIBUTIONS
// (read from opposite array due to previous swap operation)
//........................................................................
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
dist[1*Np+n] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
dist[2*Np+n] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
dist[3*Np+n] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
dist[4*Np+n] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
dist[5*Np+n] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
dist[6*Np+n] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
dist[7*Np+n] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
dist[8*Np+n] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
dist[9*Np+n] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
dist[10*Np+n] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
dist[11*Np+n] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
dist[12*Np+n] = fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
dist[13*Np+n] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
dist[14*Np+n] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
dist[15*Np+n] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
dist[16*Np+n] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
dist[17*Np+n] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
dist[18*Np+n] = fq;
//........................................................................
}
}
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np)
{
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
int nread;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
constexpr double mrt_V1=0.05263157894736842;
constexpr double mrt_V2=0.012531328320802;
constexpr double mrt_V3=0.04761904761904762;
constexpr double mrt_V4=0.004594820384294068;
constexpr double mrt_V5=0.01587301587301587;
constexpr double mrt_V6=0.0555555555555555555555555;
constexpr double mrt_V7=0.02777777777777778;
constexpr double mrt_V8=0.08333333333333333;
constexpr double mrt_V9=0.003341687552213868;
constexpr double mrt_V10=0.003968253968253968;
constexpr double mrt_V11=0.01388888888888889;
constexpr double mrt_V12=0.04166666666666666;
for (int n=start; n<finish; n++){
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
fq = dist[nread]; // reading the f1 data into register fq
//fp = dist[10*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
fq = dist[nread]; // reading the f2 data into register fq
//fq = dist[Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
nread = neighborList[n+2*Np]; // neighbor 4
fq = dist[nread];
//fq = dist[11*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
nread = neighborList[n+3*Np]; // neighbor 3
fq = dist[nread];
//fq = dist[2*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
//fq = dist[12*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
//fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
nread = neighborList[n+6*Np];
fq = dist[nread];
//fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
nread = neighborList[n+7*Np];
fq = dist[nread];
//fq = dist[4*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
nread = neighborList[n+8*Np];
fq = dist[nread];
//fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
nread = neighborList[n+9*Np];
fq = dist[nread];
//fq = dist[5*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
nread = neighborList[n+10*Np];
fq = dist[nread];
//fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
nread = neighborList[n+11*Np];
fq = dist[nread];
//fq = dist[6*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
nread = neighborList[n+12*Np];
fq = dist[nread];
//fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
nread = neighborList[n+13*Np];
fq = dist[nread];
//fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
nread = neighborList[n+14*Np];
fq = dist[nread];
//fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
nread = neighborList[n+15*Np];
fq = dist[nread];
//fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
//fq = dist[18*Np+n];
nread = neighborList[n+16*Np];
fq = dist[nread];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
nread = neighborList[n+17*Np];
fq = dist[nread];
//fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
nread = neighborList[n+Np];
dist[nread] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
nread = neighborList[n];
dist[nread] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
nread = neighborList[n+3*Np];
dist[nread] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
nread = neighborList[n+2*Np];
dist[nread] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
nread = neighborList[n+5*Np];
dist[nread] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
nread = neighborList[n+4*Np];
dist[nread] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
nread = neighborList[n+7*Np];
dist[nread] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
nread = neighborList[n+6*Np];
dist[nread] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
nread = neighborList[n+9*Np];
dist[nread] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
nread = neighborList[n+8*Np];
dist[nread] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
nread = neighborList[n+11*Np];
dist[nread] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
nread = neighborList[n+10*Np];
dist[nread]= fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
nread = neighborList[n+13*Np];
dist[nread] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
nread = neighborList[n+12*Np];
dist[nread] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
nread = neighborList[n+15*Np];
dist[nread] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
nread = neighborList[n+14*Np];
dist[nread] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
nread = neighborList[n+17*Np];
dist[nread] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
nread = neighborList[n+16*Np];
dist[nread] = fq;
}
}
//extern "C" void ScaLBL_D3Q19_Momentum_Phys(double *dist, double *vel, double h, double time_conv, int Np)
//{
// //h: resolution [um/lu]
// //time_conv: time conversion factor [sec/lt]
// int n;
// // distributions
// double f1,f2,f3,f4,f5,f6,f7,f8,f9;
// double f10,f11,f12,f13,f14,f15,f16,f17,f18;
// double vx,vy,vz;
//
// for (n=0; n<Np; n++){
// //........................................................................
// // Registers to store the distributions
// //........................................................................
// f2 = dist[2*Np+n];
// f4 = dist[4*Np+n];
// f6 = dist[6*Np+n];
// f8 = dist[8*Np+n];
// f10 = dist[10*Np+n];
// f12 = dist[12*Np+n];
// f14 = dist[14*Np+n];
// f16 = dist[16*Np+n];
// f18 = dist[18*Np+n];
// //........................................................................
// f1 = dist[Np+n];
// f3 = dist[3*Np+n];
// f5 = dist[5*Np+n];
// f7 = dist[7*Np+n];
// f9 = dist[9*Np+n];
// f11 = dist[11*Np+n];
// f13 = dist[13*Np+n];
// f15 = dist[15*Np+n];
// f17 = dist[17*Np+n];
// //.................Compute the velocity...................................
// vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
// vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
// vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
// //..................Write the velocity.....................................
// vel[0*Np+n] = vx*(h*1.0e-6)/time_conv;
// vel[1*Np+n] = vy*(h*1.0e-6)/time_conv;
// vel[2*Np+n] = vz*(h*1.0e-6)/time_conv;
// //........................................................................
// }
//}

2
ctest_script.cmake
View File

@ -164,7 +164,7 @@ SET( CTEST_OPTIONS )
SET( CTEST_OPTIONS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_C_COMPILER:PATH=${CC};-DCMAKE_C_FLAGS='${C_FLAGS}';" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCMAKE_CXX_COMPILER:PATH=${CXX};-DCMAKE_CXX_FLAGS='${CXX_FLAGS}'" )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPI_COMPILER:BOOL=true;-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DMPIEXEC=${MPIEXEC};-DUSE_EXT_MPI_FOR_SERIAL_TESTS:BOOL=true")
IF ( USE_CUDA )
SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DUSE_CUDA:BOOL=true;-DCUDA_NVCC_FLAGS='${CUDA_FLAGS}';-DCUDA_HOST_COMPILER=${CUDA_HOST_COMPILER}" )
ELSE()

311
cuda/BGK.cu Normal file
View File

@ -0,0 +1,311 @@
#include <stdio.h>
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
int n;
// conserved momemnts
double rho,ux,uy,uz,uu;
// non-conserved moments
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if ( n<finish ){
// q=0
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
f7 = dist[8*Np+n];
f8 = dist[7*Np+n];
f9 = dist[10*Np+n];
f10 = dist[9*Np+n];
f11 = dist[12*Np+n];
f12 = dist[11*Np+n];
f13 = dist[14*Np+n];
f14 = dist[13*Np+n];
f15 = dist[16*Np+n];
f16 = dist[15*Np+n];
f17 = dist[18*Np+n];
f18 = dist[17*Np+n];
rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
uu = 1.5*(ux*ux+uy*uy+uz*uz);
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu);
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx;
// q=2
dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx;
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) +
rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy;
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) +
rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy;
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) +
rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz;
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) +
rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz;
// q = 7
dist[7*Np+n] = f7*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy);
// q = 8
dist[8*Np+n] = f8*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy);
// q = 9
dist[9*Np+n] = f9*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy);
// q = 10
dist[10*Np+n] = f10*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy);
// q = 11
dist[11*Np+n] = f11*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz);
// q = 12
dist[12*Np+n] = f12*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) - 0.08333333333*(Fx+Fz);
// q = 13
dist[13*Np+n] = f13*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz);
// q= 14
dist[14*Np+n] = f14*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz);
// q = 15
dist[15*Np+n] = f15*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz);
// q = 16
dist[16*Np+n] = f16*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz);
// q = 17
dist[17*Np+n] = f17*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz);
// q = 18
dist[18*Np+n] = f18*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
int n;
// conserved momemnts
double rho,ux,uy,uz,uu;
// non-conserved moments
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18;
int nr1,nr2,nr3,nr4,nr5,nr6,nr7,nr8,nr9,nr10,nr11,nr12,nr13,nr14,nr15,nr16,nr17,nr18;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if ( n<finish ){
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q = 4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q = 6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
// q=7
nr7 = neighborList[n+6*Np];
f7 = dist[nr7];
// q = 8
nr8 = neighborList[n+7*Np];
f8 = dist[nr8];
// q=9
nr9 = neighborList[n+8*Np];
f9 = dist[nr9];
// q = 10
nr10 = neighborList[n+9*Np];
f10 = dist[nr10];
// q=11
nr11 = neighborList[n+10*Np];
f11 = dist[nr11];
// q=12
nr12 = neighborList[n+11*Np];
f12 = dist[nr12];
// q=13
nr13 = neighborList[n+12*Np];
f13 = dist[nr13];
// q=14
nr14 = neighborList[n+13*Np];
f14 = dist[nr14];
// q=15
nr15 = neighborList[n+14*Np];
f15 = dist[nr15];
// q=16
nr16 = neighborList[n+15*Np];
f16 = dist[nr16];
// q=17
//fq = dist[18*Np+n];
nr17 = neighborList[n+16*Np];
f17 = dist[nr17];
// q=18
nr18 = neighborList[n+17*Np];
f18 = dist[nr18];
rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
ux = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
uy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
uz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
uu = 1.5*(ux*ux+uy*uy+uz*uz);
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.3333333333333333*(1.0-uu);
// q = 1
dist[nr2] = f1*(1.0-rlx) + rlx*0.05555555555555555*(rho + 3.0*ux + 4.5*ux*ux - uu) + 0.16666666*Fx;
// q=2
dist[nr1] = f2*(1.0-rlx) + rlx*0.05555555555555555*(rho - 3.0*ux + 4.5*ux*ux - uu)- 0.16666666*Fx;
// q = 3
dist[nr4] = f3*(1.0-rlx) +
rlx*0.05555555555555555*(rho + 3.0*uy + 4.5*uy*uy - uu) + 0.16666666*Fy;
// q = 4
dist[nr3] = f4*(1.0-rlx) +
rlx*0.05555555555555555*(rho - 3.0*uy + 4.5*uy*uy - uu)- 0.16666666*Fy;
// q = 5
dist[nr6] = f5*(1.0-rlx) +
rlx*0.05555555555555555*(rho + 3.0*uz + 4.5*uz*uz - uu) + 0.16666666*Fz;
// q = 6
dist[nr5] = f6*(1.0-rlx) +
rlx*0.05555555555555555*(rho - 3.0*uz + 4.5*uz*uz - uu) - 0.16666666*Fz;
// q = 7
dist[nr8] = f7*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) + 0.08333333333*(Fx+Fy);
// q = 8
dist[nr7] = f8*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux+uy) + 4.5*(ux+uy)*(ux+uy) - uu) - 0.08333333333*(Fx+Fy);
// q = 9
dist[nr10] = f9*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) + 0.08333333333*(Fx-Fy);
// q = 10
dist[nr9] = f10*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux-uy) + 4.5*(ux-uy)*(ux-uy) - uu) - 0.08333333333*(Fx-Fy);
// q = 11
dist[nr12] = f11*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) + 0.08333333333*(Fx+Fz);
// q = 12
dist[nr11] = f12*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux+uz) + 4.5*(ux+uz)*(ux+uz) - uu) - 0.08333333333*(Fx+Fz);
// q = 13
dist[nr14] = f13*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu) + 0.08333333333*(Fx-Fz);
// q= 14
dist[nr13] = f14*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(ux-uz) + 4.5*(ux-uz)*(ux-uz) - uu)- 0.08333333333*(Fx-Fz);
// q = 15
dist[nr16] = f15*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) + 0.08333333333*(Fy+Fz);
// q = 16
dist[nr15] = f16*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(uy+uz) + 4.5*(uy+uz)*(uy+uz) - uu) - 0.08333333333*(Fy+Fz);
// q = 17
dist[nr18] = f17*(1.0-rlx) +
rlx*0.02777777777777778*(rho + 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) + 0.08333333333*(Fy-Fz);
// q = 18
dist[nr17] = f18*(1.0-rlx) +
rlx*0.02777777777777778*(rho - 3.0*(uy-uz) + 4.5*(uy-uz)*(uy-uz) - uu) - 0.08333333333*(Fy-Fz);
}
}
}
extern "C" void ScaLBL_D3Q19_AAeven_BGK(double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
dvc_ScaLBL_D3Q19_AAeven_BGK<<<NBLOCKS,NTHREADS >>>(dist,start,finish,Np,rlx,Fx,Fy,Fz);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_AAeven_BGK: %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q19_AAodd_BGK(int *neighborList, double *dist, int start, int finish, int Np, double rlx, double Fx, double Fy, double Fz){
dvc_ScaLBL_D3Q19_AAodd_BGK<<<NBLOCKS,NTHREADS >>>(neighborList,dist,start,finish,Np,rlx,Fx,Fy,Fz);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_AAeven_BGK: %s \n",cudaGetErrorString(err));
}
}

16
gpu/Color.cu → cuda/Color.cu
View File

@ -1,19 +1,3 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <math.h>
#include <stdio.h>
#include <cuda_profiler_api.h>

16
gpu/CudaExtras.cu → cuda/CudaExtras.cu
View File

@ -1,19 +1,3 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
// Basic cuda functions callable from C/C++ code
#include <cuda.h>

16
gpu/D3Q19.cu → cuda/D3Q19.cu
View File

@ -1,19 +1,3 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <cooperative_groups.h>

16
gpu/D3Q7.cu → cuda/D3Q7.cu
View File

@ -1,19 +1,3 @@
/*
Copyright 2013--2018 James E. McClure, Virginia Polytechnic & State University
Copyright Equnior ASA
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
*/
// GPU Functions for D3Q7 Lattice Boltzmann Methods
#include <stdio.h>

537
cuda/D3Q7BC.cu Normal file
View File

@ -0,0 +1,537 @@
#include <math.h>
#include <stdio.h>
#include <cuda_profiler_api.h>
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = -1.0*value_q + value_b*0.25;//NOTE 0.25 is the speed of sound for D3Q7 lattice
}
}
__global__ void dvc_ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count)
{
int idx;
int iq,ib;
double value_b,value_q;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
iq = BounceBackDist_list[idx];
ib = BounceBackSolid_list[idx];
value_b = BoundaryValue[ib];//get boundary value from a solid site
value_q = dist[iq];
dist[iq] = value_q + value_b;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Vin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Vout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vin;
}
}
__global__ void dvc_ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count)
{
int idx,n,nm;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
nm = Map[n];
Psi[nm] = Vout;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
//...................................................
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np)
{
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
//...................................................
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np)
{
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
f5 = Cin - (f0+f1+f2+f3+f4+f6);
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np)
{
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
// unknown distributions
nr6 = d_neighborList[n+5*Np];
f6 = Cout - (f0+f1+f2+f3+f4+f5);
dist[nr6] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f6 = dist[5*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
dist[6*Np+n] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx,n;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
dist[5*Np+n] = f6;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr5;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
nread = d_neighborList[n+5*Np];
f6 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f6;
uz = VelocityZ[n];
//...................................................
f5 =(FluxIn+(1.0-0.5/tau)*f6-0.5*uz*fsum_partial/tau)/(1.0-0.5/tau+0.5*uz/tau);
// Unknown distributions
nr5 = d_neighborList[n+4*Np];
dist[nr5] = f5;
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np)
{
//NOTE: FluxIn is the inward flux
int idx, n;
int nread,nr6;
double f0,f1,f2,f3,f4,f5,f6;
double fsum_partial;
double uz;
idx = blockIdx.x*blockDim.x + threadIdx.x;
if (idx < count){
n = list[idx];
f0 = dist[n];
nread = d_neighborList[n];
f1 = dist[nread];
nread = d_neighborList[n+2*Np];
f3 = dist[nread];
nread = d_neighborList[n+4*Np];
f5 = dist[nread];
nread = d_neighborList[n+Np];
f2 = dist[nread];
nread = d_neighborList[n+3*Np];
f4 = dist[nread];
fsum_partial = f0+f1+f2+f3+f4+f5;
uz = VelocityZ[n];
//...................................................
f6 =(FluxIn+(1.0-0.5/tau)*f5+0.5*uz*fsum_partial/tau)/(1.0-0.5/tau-0.5*uz/tau);
// unknown distributions
nr6 = d_neighborList[n+5*Np];
dist[nr6] = f6;
}
}
//*************************************************************************
extern "C" void ScaLBL_Solid_Dirichlet_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Dirichlet_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_Solid_Dirichlet_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_Solid_Neumann_D3Q7(double *dist, double *BoundaryValue, int *BounceBackDist_list, int *BounceBackSolid_list, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Solid_Neumann_D3Q7<<<GRID,512>>>(dist, BoundaryValue, BounceBackDist_list, BounceBackSolid_list, count);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_Solid_Neumann_D3Q7 (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z(int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z<<<GRID,512>>>(list, dist, Vin, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z(int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z<<<GRID,512>>>(list, dist, Vout, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z(int *d_neighborList, int *list, double *dist, double Vin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Vin, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z(int *d_neighborList, int *list, double *dist, double Vout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Vout, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_Potential_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_z(int *list, int *Map, double *Psi, double Vin, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_z<<<GRID,512>>>(list, Map, Psi, Vin, count);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_Poisson_D3Q7_BC_Z(int *list, int *Map, double *Psi, double Vout, int count){
int GRID = count / 512 + 1;
dvc_ScaLBL_Poisson_D3Q7_BC_Z<<<GRID,512>>>(list, Map, Psi, Vout, count);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_Poisson_D3Q7_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z(int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z<<<GRID,512>>>(list, dist, Cin, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z(int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z<<<GRID,512>>>(list, dist, Cout, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z(int *d_neighborList, int *list, double *dist, double Cin, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z<<<GRID,512>>>(d_neighborList, list, dist, Cin, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z(int *d_neighborList, int *list, double *dist, double Cout, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, Cout, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Concentration_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z(int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z<<<GRID,512>>>(list, dist, FluxIn, tau, VelocityZ, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_z (kernel): %s \n",cudaGetErrorString(err));
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z(int *d_neighborList, int *list, double *dist, double FluxIn, double tau, double *VelocityZ, int count, int Np){
int GRID = count / 512 + 1;
dvc_ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z<<<GRID,512>>>(d_neighborList, list, dist, FluxIn, tau, VelocityZ, count, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion_Flux_BC_Z (kernel): %s \n",cudaGetErrorString(err));
}
}

0
gpu/Extras.cu → cuda/Extras.cu
View File

2122
cuda/FreeLee.cu Normal file
View File

File diff suppressed because it is too large Load Diff

2743
cuda/Greyscale.cu Normal file
View File

File diff suppressed because it is too large Load Diff

44
gpu/GreyscaleColor.cu → cuda/GreyscaleColor.cu
View File

@ -512,7 +512,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Ma
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
Pressure[n] = rho/3.f/porosity;
//Pressure[n] = rho/3.f/porosity;
Pressure[n] = rho/3.f;
//........................................................................
//..............carry out relaxation process..............................
@ -1218,7 +1219,8 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist,
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
Pressure[n] = rho/3.f/porosity;
//Pressure[n] = rho/3.f/porosity;
Pressure[n] = rho/3.f;
//........................................................................
//..............carry out relaxation process..............................
@ -1445,6 +1447,37 @@ __global__ void dvc_ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist,
}
}
__global__ void dvc_ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
int idx;
double nA,nB;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (idx<finish) {
nA = Den[idx];
nB = Den[Np+idx];
Aq[idx]=0.3333333333333333*nA;
Aq[Np+idx]=0.1111111111111111*nA;
Aq[2*Np+idx]=0.1111111111111111*nA;
Aq[3*Np+idx]=0.1111111111111111*nA;
Aq[4*Np+idx]=0.1111111111111111*nA;
Aq[5*Np+idx]=0.1111111111111111*nA;
Aq[6*Np+idx]=0.1111111111111111*nA;
Bq[idx]=0.3333333333333333*nB;
Bq[Np+idx]=0.1111111111111111*nB;
Bq[2*Np+idx]=0.1111111111111111*nB;
Bq[3*Np+idx]=0.1111111111111111*nB;
Bq[4*Np+idx]=0.1111111111111111*nB;
Bq[5*Np+idx]=0.1111111111111111*nB;
Bq[6*Np+idx]=0.1111111111111111*nB;
}
}
}
////Model-2&3
//__global__ void dvc_ScaLBL_D3Q19_AAodd_GreyscaleColor(int *neighborList, int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *Phi, double *GreySolidGrad, double *Poros,double *Perm, double *Velocity,
@ -2957,6 +2990,13 @@ extern "C" void ScaLBL_D3Q19_AAodd_GreyscaleColor(int *d_neighborList, int *Map,
//cudaProfilerStop();
}
extern "C" void ScaLBL_PhaseField_InitFromRestart(double *Den, double *Aq, double *Bq, int start, int finish, int Np){
dvc_ScaLBL_PhaseField_InitFromRestart<<<NBLOCKS,NTHREADS >>>(Den, Aq, Bq, start, finish, Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_PhaseField_InitFromRestart: %s \n",cudaGetErrorString(err));
}
}
////Model-2&3
//extern "C" void ScaLBL_D3Q19_AAeven_GreyscaleColor(int *Map, double *dist, double *Aq, double *Bq, double *Den,
// double *Phi,double *GreySolidGrad, double *Poros,double *Perm,double *Vel,

391
cuda/Ion.cu Normal file
View File

@ -0,0 +1,391 @@
#include <stdio.h>
#include <math.h>
//#include <cuda_profiler_api.h>
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
int n,nread;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
Ci += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
Ci += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
Ci += fq;
// q=4
nread = neighborList[n+3*Np];
fq = dist[nread];
Ci += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
Ci += fq;
// q=6
nread = neighborList[n+5*Np];
fq = dist[nread];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
int n;
double fq,Ci;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
Ci = fq;
// q=1
fq = dist[2*Np+n];
Ci += fq;
// q=2
fq = dist[1*Np+n];
Ci += fq;
// q=3
fq = dist[4*Np+n];
Ci += fq;
// q=4
fq = dist[3*Np+n];
Ci += fq;
// q=5
fq = dist[6*Np+n];
Ci += fq;
// q=6
fq = dist[5*Np+n];
Ci += fq;
Den[n]=Ci;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
// q=2
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q=4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q=6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[nr2] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[nr1] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[nr4] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[nr3] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[nr6] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[nr5] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
int n;
double Ci;
double ux,uy,uz;
double uEPx,uEPy,uEPz;//electrochemical induced velocity
double Ex,Ey,Ez;//electrical field
double f0,f1,f2,f3,f4,f5,f6;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
Ci=Den[n];
Ex=ElectricField[n+0*Np];
Ey=ElectricField[n+1*Np];
Ez=ElectricField[n+2*Np];
ux=Velocity[n+0*Np];
uy=Velocity[n+1*Np];
uz=Velocity[n+2*Np];
uEPx=zi*Di/Vt*Ex;
uEPy=zi*Di/Vt*Ey;
uEPz=zi*Di/Vt*Ez;
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
// q=0
dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci;
//dist[n] = f0*(1.0-rlx)+rlx*0.25*Ci*(1.0 - 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx));
//dist[1*Np+n] = f1*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q=2
dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx));
//dist[2*Np+n] = f2*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(ux+uEPx)+8.0*(ux+uEPx)*(ux+uEPx)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy));
//dist[3*Np+n] = f3*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy));
//dist[4*Np+n] = f4*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uy+uEPy)+8.0*(uy+uEPy)*(uy+uEPy)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz));
//dist[5*Np+n] = f5*(1.0-rlx) + rlx*0.125*Ci*(1.0+4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz));
//dist[6*Np+n] = f6*(1.0-rlx) + rlx*0.125*Ci*(1.0-4.0*(uz+uEPz)+8.0*(uz+uEPz)*(uz+uEPz)- 2.0*((ux+uEPx)*(ux+uEPx) + (uy+uEPy)*(uy+uEPy) + (uz+uEPz)*(uz+uEPz)));
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
int n;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
Den[n] = DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
int n;
double DenInit;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<Np) {
DenInit = Den[n];
dist[0*Np+n] = 0.25*DenInit;
dist[1*Np+n] = 0.125*DenInit;
dist[2*Np+n] = 0.125*DenInit;
dist[3*Np+n] = 0.125*DenInit;
dist[4*Np+n] = 0.125*DenInit;
dist[5*Np+n] = 0.125*DenInit;
dist[6*Np+n] = 0.125*DenInit;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
int n;
double Ci;//ion concentration of species i
double CD;//charge density
double CD_tmp;
double F = 96485.0;//Faraday's constant; unit[C/mol]; F=e*Na, where Na is the Avogadro constant
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
Ci = Den[n+ion_component*Np];
CD = ChargeDensity[n];
CD_tmp = F*IonValence*Ci;
ChargeDensity[n] = CD*(ion_component>0) + CD_tmp;
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_IonConcentration(int *neighborList, double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_IonConcentration<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_IonConcentration: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_IonConcentration(double *dist, double *Den, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_IonConcentration<<<NBLOCKS,NTHREADS >>>(dist,Den,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_IonConcentration: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Ion(int *neighborList, double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Ion<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Ion: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Ion(double *dist, double *Den, double *Velocity, double *ElectricField,
double Di, int zi, double rlx, double Vt, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Ion<<<NBLOCKS,NTHREADS >>>(dist,Den,Velocity,ElectricField,Di,zi,rlx,Vt,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Ion: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init(double *dist, double *Den, double DenInit, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init<<<NBLOCKS,NTHREADS >>>(dist,Den,DenInit,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Ion_Init: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_Init_FromFile(double *dist, double *Den, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_Init_FromFile<<<NBLOCKS,NTHREADS >>>(dist,Den,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Ion_Init_FromFile: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Ion_ChargeDensity(double *Den, double *ChargeDensity, int IonValence, int ion_component, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Ion_ChargeDensity<<<NBLOCKS,NTHREADS >>>(Den,ChargeDensity,IonValence,ion_component,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Ion_ChargeDensity: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}

310
cuda/MRT.cu Normal file
View File

@ -0,0 +1,310 @@
//*************************************************************************
// CUDA kernels for single-phase ScaLBL_D3Q19_MRT code
// James McClure
//*************************************************************************
#include <cuda.h>
#define NBLOCKS 560
#define NTHREADS 128
__global__ void INITIALIZE(char *ID, double *f_even, double *f_odd, int Nx, int Ny, int Nz)
{
int n,N;
N = Nx*Ny*Nz;
int S = N/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<N){
if (ID[n] > 0){
f_even[n] = 0.3333333333333333;
f_odd[n] = 0.055555555555555555; //double(100*n)+1.f;
f_even[N+n] = 0.055555555555555555; //double(100*n)+2.f;
f_odd[N+n] = 0.055555555555555555; //double(100*n)+3.f;
f_even[2*N+n] = 0.055555555555555555; //double(100*n)+4.f;
f_odd[2*N+n] = 0.055555555555555555; //double(100*n)+5.f;
f_even[3*N+n] = 0.055555555555555555; //double(100*n)+6.f;
f_odd[3*N+n] = 0.0277777777777778; //double(100*n)+7.f;
f_even[4*N+n] = 0.0277777777777778; //double(100*n)+8.f;
f_odd[4*N+n] = 0.0277777777777778; //double(100*n)+9.f;
f_even[5*N+n] = 0.0277777777777778; //double(100*n)+10.f;
f_odd[5*N+n] = 0.0277777777777778; //double(100*n)+11.f;
f_even[6*N+n] = 0.0277777777777778; //double(100*n)+12.f;
f_odd[6*N+n] = 0.0277777777777778; //double(100*n)+13.f;
f_even[7*N+n] = 0.0277777777777778; //double(100*n)+14.f;
f_odd[7*N+n] = 0.0277777777777778; //double(100*n)+15.f;
f_even[8*N+n] = 0.0277777777777778; //double(100*n)+16.f;
f_odd[8*N+n] = 0.0277777777777778; //double(100*n)+17.f;
f_even[9*N+n] = 0.0277777777777778; //double(100*n)+18.f;
}
else{
for(int q=0; q<9; q++){
f_even[q*N+n] = -1.0;
f_odd[q*N+n] = -1.0;
}
f_even[9*N+n] = -1.0;
}
}
}
}
__global__ void Compute_VELOCITY(char *ID, double *disteven, double *distodd, double *vel, int Nx, int Ny, int Nz)
{
int n,N;
// distributions
double f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
double vx,vy,vz;
N = Nx*Ny*Nz;
int S = N/NBLOCKS/NTHREADS + 1;
// S - number of threadblocks per grid block
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (n<N){
if (ID[n] > 0){
//........................................................................
// Registers to store the distributions
//........................................................................
f2 = disteven[N+n];
f4 = disteven[2*N+n];
f6 = disteven[3*N+n];
f8 = disteven[4*N+n];
f10 = disteven[5*N+n];
f12 = disteven[6*N+n];
f14 = disteven[7*N+n];
f16 = disteven[8*N+n];
f18 = disteven[9*N+n];
//........................................................................
f1 = distodd[n];
f3 = distodd[1*N+n];
f5 = distodd[2*N+n];
f7 = distodd[3*N+n];
f9 = distodd[4*N+n];
f11 = distodd[5*N+n];
f13 = distodd[6*N+n];
f15 = distodd[7*N+n];
f17 = distodd[8*N+n];
//.................Compute the velocity...................................
vx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
vy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
vz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
//..................Write the velocity.....................................
vel[n] = vx;
vel[N+n] = vy;
vel[2*N+n] = vz;
//........................................................................
}
}
}
}
//*************************************************************************
__global__ void
__launch_bounds__(512,2)
D3Q19_MRT(char *ID, double *disteven, double *distodd, int Nx, int Ny, int Nz,
double rlx_setA, double rlx_setB, double Fx, double Fy, double Fz)
{
int n,N;
// distributions
double f0,f1,f2,f3,f4,f5,f6,f7,f8,f9;
double f10,f11,f12,f13,f14,f15,f16,f17,f18;
// conserved momemnts
double rho,jx,jy,jz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
N = Nx*Ny*Nz;
char id;
int S = N/NBLOCKS/NTHREADS + 1;
// S - number of threadblocks per grid block
for (int s=0; s<S; s++){
// for (int n=0; n<N; n++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
id = ID[n];
if (n<N){
if (id > 0){
//........................................................................
// Registers to store the distributions - read based on swap convention
//........................................................................
f2 = distodd[n];
f4 = distodd[N+n];
f6 = distodd[2*N+n];
f8 = distodd[3*N+n];
f10 = distodd[4*N+n];
f12 = distodd[5*N+n];
f14 = distodd[6*N+n];
f16 = distodd[7*N+n];
f18 = distodd[8*N+n];
//........................................................................
f0 = disteven[n];
f1 = disteven[N+n];
f3 = disteven[2*N+n];
f5 = disteven[3*N+n];
f7 = disteven[4*N+n];
f9 = disteven[5*N+n];
f11 = disteven[6*N+n];
f13 = disteven[7*N+n];
f15 = disteven[8*N+n];
f17 = disteven[9*N+n];
//........................................................................
//....................compute the moments...............................................
rho = f0+f2+f1+f4+f3+f6+f5+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
m1 = -30*f0-11*(f2+f1+f4+f3+f6+f5)+8*(f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18 +f17);
m2 = 12*f0-4*(f2+f1 +f4+f3+f6 +f5)+f8+f7+f10+f9+f12+f11+f14+f13+f16+f15+f18+f17;
jx = f1-f2+f7-f8+f9-f10+f11-f12+f13-f14;
m4 = 4*(-f1+f2)+f7-f8+f9-f10+f11-f12+f13-f14;
jy = f3-f4+f7-f8-f9+f10+f15-f16+f17-f18;
m6 = -4*(f3-f4)+f7-f8-f9+f10+f15-f16+f17-f18;
jz = f5-f6+f11-f12-f13+f14+f15-f16-f17+f18;
m8 = -4*(f5-f6)+f11-f12-f13+f14+f15-f16-f17+f18;
m9 = 2*(f1+f2)-f3-f4-f5-f6+f7+f8+f9+f10+f11+f12+f13+f14-2*(f15+f16+f17+f18);
m10 = -4*(f1+f2)+2*(f4+f3+f6+f5)+f8+f7+f10+f9+f12+f11+f14+f13-2*(f16+f15+f18+f17);
m11 = f4+f3-f6-f5+f8+f7+f10+f9-f12-f11-f14-f13;
m12 = -2*(f4+f3-f6-f5)+f8+f7+f10+f9-f12-f11-f14-f13;
m13 = f8+f7-f10-f9;
m14 = f16+f15-f18-f17;
m15 = f12+f11-f14-f13;
m16 = f7-f8+f9-f10-f11+f12-f13+f14;
m17 = -f7+f8+f9-f10+f15-f16+f17-f18;
m18 = f11-f12-f13+f14-f15+f16+f17-f18;
//..............incorporate external force................................................
//jx += 0.5*Fx;
//jy += 0.5*Fy;
//jz += 0.5*Fz;
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.................inverse transformation......................................................
f0 = 0.05263157894736842*rho-0.012531328320802*m1+0.04761904761904762*m2;
f1 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(jx-m4)+0.05555555555555555*(m9-m10);
f2 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(m4-jx)+0.05555555555555555*(m9-m10);
f3 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(jy-m6)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
f4 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(m6-jy)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m11-m12);
f5 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(jz-m8)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
f6 = 0.05263157894736842*rho-0.004594820384294068*m1-0.01587301587301587*m2
+0.1*(m8-jz)+0.02777777777777778*(m10-m9)+0.08333333333333333*(m12-m11);
f7 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx+jy)+0.025*(m4+m6)
+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+0.04166666666666666*m12+0.25*m13+0.125*(m16-m17);
f8 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2-0.1*(jx+jy)-0.025*(m4+m6)
+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+0.04166666666666666*m12+0.25*m13+0.125*(m17-m16);
f9 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jx-jy)+0.025*(m4-m6)
+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+0.04166666666666666*m12-0.25*m13+0.125*(m16+m17);
f10 = 0.05263157894736842*rho+0.003341687552213868*m1+0.003968253968253968*m2+0.1*(jy-jx)+0.025*(m6-m4)
+0.02777777777777778*m9+0.01388888888888889*m10+0.08333333333333333*m11
+0.04166666666666666*m12-0.25*m13-0.125*(m16+m17);
f11 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jx+jz)+0.025*(m4+m8)
+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-0.04166666666666666*m12+0.25*m15+0.125*(m18-m16);
f12 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2-0.1*(jx+jz)-0.025*(m4+m8)
+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-0.04166666666666666*m12+0.25*m15+0.125*(m16-m18);
f13 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jx-jz)+0.025*(m4-m8)
+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-0.04166666666666666*m12-0.25*m15-0.125*(m16+m18);
f14 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jz-jx)+0.025*(m8-m4)
+0.02777777777777778*m9+0.01388888888888889*m10-0.08333333333333333*m11
-0.04166666666666666*m12-0.25*m15+0.125*(m16+m18);
f15 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jy+jz)+0.025*(m6+m8)
-0.05555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m17-m18);
f16 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2-0.1*(jy+jz)-0.025*(m6+m8)
-0.05555555555555555*m9-0.02777777777777778*m10+0.25*m14+0.125*(m18-m17);
f17 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jy-jz)+0.025*(m6-m8)
-0.05555555555555555*m9-0.02777777777777778*m10-0.25*m14+0.125*(m17+m18);
f18 = 0.05263157894736842*rho+0.003341687552213868*m1
+0.003968253968253968*m2+0.1*(jz-jy)+0.025*(m8-m6)
-0.05555555555555555*m9-0.02777777777777778*m10-0.25*m14-0.125*(m17+m18);
//.......................................................................................................
// incorporate external force
f1 += 0.16666666*Fx;
f2 -= 0.16666666*Fx;
f3 += 0.16666666*Fy;
f4 -= 0.16666666*Fy;
f5 += 0.16666666*Fz;
f6 -= 0.16666666*Fz;
f7 += 0.08333333333*(Fx+Fy);
f8 -= 0.08333333333*(Fx+Fy);
f9 += 0.08333333333*(Fx-Fy);
f10 -= 0.08333333333*(Fx-Fy);
f11 += 0.08333333333*(Fx+Fz);
f12 -= 0.08333333333*(Fx+Fz);
f13 += 0.08333333333*(Fx-Fz);
f14 -= 0.08333333333*(Fx-Fz);
f15 += 0.08333333333*(Fy+Fz);
f16 -= 0.08333333333*(Fy+Fz);
f17 += 0.08333333333*(Fy-Fz);
f18 -= 0.08333333333*(Fy-Fz);
//.......................................................................................................
// Write data based on un-swapped convention
disteven[n] = f0;
disteven[N+n] = f2;
disteven[2*N+n] = f4;
disteven[3*N+n] = f6;
disteven[4*N+n] = f8;
disteven[5*N+n] = f10;
disteven[6*N+n] = f12;
disteven[7*N+n] = f14;
disteven[8*N+n] = f16;
disteven[9*N+n] = f18;
distodd[n] = f1;
distodd[N+n] = f3;
distodd[2*N+n] = f5;
distodd[3*N+n] = f7;
distodd[4*N+n] = f9;
distodd[5*N+n] = f11;
distodd[6*N+n] = f13;
distodd[7*N+n] = f15;
distodd[8*N+n] = f17;
//.......................................................................................................
}
}
}
}
extern "C" void ScaLBL_D3Q19_MRT(char *ID, double *f_even, double *f_odd, double rlxA, double rlxB, double Fx, double Fy, double Fz,int Nx, int Ny, int Nz)
{
D3Q19_MRT <<< NBLOCKS,NTHREADS>>> (ID, f_even, f_odd, Nx, Ny, Nz, rlxA, rlxB, Fx, Fy, Fz);
}

76
cuda/MixedGradient.cu Normal file
View File

@ -0,0 +1,76 @@
/* Implement Mixed Gradient (Lee et al. JCP 2016)*/
#include <cuda.h>
#include <stdio.h>
#include <cuda_profiler_api.h>
#define NBLOCKS 560
#define NTHREADS 128
__global__ void dvc_ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
static int D3Q19[18][3]={{1,0,0},{-1,0,0},{0,1,0},{0,-1,0},{0,0,1},{0,0,-1},
{1,1,0},{-1,-1,0},{1,-1,0},{-1,1,0},
{1,0,1},{-1,0,-1},{1,0,-1},{-1,0,1},
{0,1,1},{0,-1,-1},{0,1,-1},{0,-1,1}};
int i,j,k,n,N,idx;
int np,np2,nm; // neighbors
double v,vp,vp2,vm; // values at neighbors
double grad;
N = Nx*Ny*Nz;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
idx = start + S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x;
if (idx<finish){
n = Map[idx]; // layout in regular array
//.......Back out the 3-D indices for node n..............
k = n/(Nx*Ny);
j = (n-Nx*Ny*k)/Nx;
i = n-Nx*Ny*k-Nx*j;
v = Phi[n];
grad = 0.0;
for (int q=0; q<6; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.25*(5.0*vp-vp2-3.0*v-vm);
}
for (int q=6; q<18; q++){
int iqx = D3Q19[q][0];
int iqy = D3Q19[q][1];
int iqz = D3Q19[q][2];
np = (k+iqz)*Nx*Ny + (j+iqy)*Nx + i + iqx;
np2 = (k+2*iqz)*Nx*Ny + (j+2*iqy)*Nx + i + 2*iqx;
nm = (k-iqz)*Nx*Ny + (j-iqy)*Nx + i - iqx;
vp = Phi[np];
vp2 = Phi[np2];
vm = Phi[nm];
grad += 0.125*(5.0*vp-vp2-3.0*v-vm);
}
Gradient[n] = grad;
}
}
}
extern "C" void ScaLBL_D3Q19_MixedGradient(int *Map, double *Phi, double *Gradient, int start, int finish, int Np, int Nx, int Ny, int Nz)
{
cudaProfilerStart();
dvc_ScaLBL_D3Q19_MixedGradient<<<NBLOCKS,NTHREADS >>>(Map, Phi, Gradient, start, finish, Np, Nx, Ny, Nz);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_MixedGradient: %s \n",cudaGetErrorString(err));
}
cudaProfilerStop();
}

330
cuda/Poisson.cu Normal file
View File

@ -0,0 +1,330 @@
#include <stdio.h>
#include <math.h>
//#include <cuda_profiler_api.h>
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int nread;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
nread = neighborList[n];
fq = dist[nread];
psi += fq;
// q=2
nread = neighborList[n+Np];
fq = dist[nread];
psi += fq;
// q=3
nread = neighborList[n+2*Np];
fq = dist[nread];
psi += fq;
// q = 4
nread = neighborList[n+3*Np];
fq = dist[nread];
psi += fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
psi += fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
double psi;//electric potential
double fq;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
// q=0
fq = dist[n];
psi = fq;
// q=1
fq = dist[2*Np+n];
psi += fq;
// q=2
fq = dist[1*Np+n];
psi += fq;
// q=3
fq = dist[4*Np+n];
psi += fq;
// q=4
fq = dist[3*Np+n];
psi += fq;
// q=5
fq = dist[6*Np+n];
psi += fq;
// q=6
fq = dist[5*Np+n];
psi += fq;
idx=Map[n];
Psi[idx] = psi;
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
int nr1,nr2,nr3,nr4,nr5,nr6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
// q=0
f0 = dist[n];
// q=1
nr1 = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
f1 = dist[nr1]; // reading the f1 data into register fq
nr2 = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
f2 = dist[nr2]; // reading the f2 data into register fq
// q=3
nr3 = neighborList[n+2*Np]; // neighbor 4
f3 = dist[nr3];
// q = 4
nr4 = neighborList[n+3*Np]; // neighbor 3
f4 = dist[nr4];
// q=5
nr5 = neighborList[n+4*Np];
f5 = dist[nr5];
// q = 6
nr6 = neighborList[n+5*Np];
f6 = dist[nr6];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[nr2] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[nr1] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[nr4] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[nr3] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[nr6] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[nr5] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
int n;
double psi;//electric potential
double Ex,Ey,Ez;//electric field
double rho_e;//local charge density
double f0,f1,f2,f3,f4,f5,f6;
double rlx=1.0/tau;
int idx;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rho_e = Den_charge[n];
rho_e = rho_e/epsilon_LB;
idx=Map[n];
psi = Psi[idx];
f0 = dist[n];
f1 = dist[2*Np+n];
f2 = dist[1*Np+n];
f3 = dist[4*Np+n];
f4 = dist[3*Np+n];
f5 = dist[6*Np+n];
f6 = dist[5*Np+n];
Ex = (f1-f2)*rlx*4.0;//NOTE the unit of electric field here is V/lu
Ey = (f3-f4)*rlx*4.0;//factor 4.0 is D3Q7 lattice speed of sound
Ez = (f5-f6)*rlx*4.0;
ElectricField[n+0*Np] = Ex;
ElectricField[n+1*Np] = Ey;
ElectricField[n+2*Np] = Ez;
// q = 0
dist[n] = f0*(1.0-rlx) + 0.25*(rlx*psi+rho_e);
// q = 1
dist[1*Np+n] = f1*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 2
dist[2*Np+n] = f2*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 3
dist[3*Np+n] = f3*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 4
dist[4*Np+n] = f4*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 5
dist[5*Np+n] = f5*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
// q = 6
dist[6*Np+n] = f6*(1.0-rlx) + 0.125*(rlx*psi+rho_e);
//........................................................................
}
}
}
__global__ void dvc_ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
int n;
int ijk;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
ijk = Map[n];
dist[0*Np+n] = 0.25*Psi[ijk];
dist[1*Np+n] = 0.125*Psi[ijk];
dist[2*Np+n] = 0.125*Psi[ijk];
dist[3*Np+n] = 0.125*Psi[ijk];
dist[4*Np+n] = 0.125*Psi[ijk];
dist[5*Np+n] = 0.125*Psi[ijk];
dist[6*Np+n] = 0.125*Psi[ijk];
}
}
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential(int *neighborList,int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Psi,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson_ElectricPotential: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAodd_Poisson(int *neighborList, int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAodd_Poisson<<<NBLOCKS,NTHREADS >>>(neighborList,Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAodd_Poisson: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_AAeven_Poisson(int *Map, double *dist, double *Den_charge, double *Psi, double *ElectricField, double tau, double epsilon_LB,int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_AAeven_Poisson<<<NBLOCKS,NTHREADS >>>(Map,dist,Den_charge,Psi,ElectricField,tau,epsilon_LB,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_AAeven_Poisson: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q7_Poisson_Init(int *Map, double *dist, double *Psi, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q7_Poisson_Init<<<NBLOCKS,NTHREADS >>>(Map,dist,Psi,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q7_Poisson_Init: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}

995
cuda/Stokes.cu Normal file
View File

@ -0,0 +1,995 @@
#include <stdio.h>
#include <math.h>
//#include <cuda_profiler_api.h>
#define NBLOCKS 1024
#define NTHREADS 256
__global__ void dvc_ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz, double rho0, double den_scale, double h, double time_conv,int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
int nread;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
nread = neighborList[n]; // neighbor 2 ( > 10Np => odd part of dist)
fq = dist[nread]; // reading the f1 data into register fq
//fp = dist[10*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
nread = neighborList[n+Np]; // neighbor 1 ( < 10Np => even part of dist)
fq = dist[nread]; // reading the f2 data into register fq
//fq = dist[Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
nread = neighborList[n+2*Np]; // neighbor 4
fq = dist[nread];
//fq = dist[11*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
nread = neighborList[n+3*Np]; // neighbor 3
fq = dist[nread];
//fq = dist[2*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
nread = neighborList[n+4*Np];
fq = dist[nread];
//fq = dist[12*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
nread = neighborList[n+5*Np];
fq = dist[nread];
//fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
nread = neighborList[n+6*Np];
fq = dist[nread];
//fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
nread = neighborList[n+7*Np];
fq = dist[nread];
//fq = dist[4*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
nread = neighborList[n+8*Np];
fq = dist[nread];
//fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
nread = neighborList[n+9*Np];
fq = dist[nread];
//fq = dist[5*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
nread = neighborList[n+10*Np];
fq = dist[nread];
//fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
nread = neighborList[n+11*Np];
fq = dist[nread];
//fq = dist[6*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
nread = neighborList[n+12*Np];
fq = dist[nread];
//fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
nread = neighborList[n+13*Np];
fq = dist[nread];
//fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
nread = neighborList[n+14*Np];
fq = dist[nread];
//fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
nread = neighborList[n+15*Np];
fq = dist[nread];
//fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
//fq = dist[18*Np+n];
nread = neighborList[n+16*Np];
fq = dist[nread];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
nread = neighborList[n+17*Np];
fq = dist[nread];
//fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10)+0.16666666*Fx;
nread = neighborList[n+Np];
dist[nread] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
nread = neighborList[n];
dist[nread] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
nread = neighborList[n+3*Np];
dist[nread] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
nread = neighborList[n+2*Np];
dist[nread] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
nread = neighborList[n+5*Np];
dist[nread] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
nread = neighborList[n+4*Np];
dist[nread] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
nread = neighborList[n+7*Np];
dist[nread] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
nread = neighborList[n+6*Np];
dist[nread] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
nread = neighborList[n+9*Np];
dist[nread] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
nread = neighborList[n+8*Np];
dist[nread] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
nread = neighborList[n+11*Np];
dist[nread] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
nread = neighborList[n+10*Np];
dist[nread]= fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
nread = neighborList[n+13*Np];
dist[nread] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
nread = neighborList[n+12*Np];
dist[nread] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
nread = neighborList[n+15*Np];
dist[nread] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
nread = neighborList[n+14*Np];
dist[nread] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
nread = neighborList[n+17*Np];
dist[nread] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
nread = neighborList[n+16*Np];
dist[nread] = fq;
}
}
}
__global__ void dvc_ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
int n;
double fq;
// conserved momemnts
double rho,jx,jy,jz;
double ux,uy,uz;
// non-conserved moments
double m1,m2,m4,m6,m8,m9,m10,m11,m12,m13,m14,m15,m16,m17,m18;
// body force due to electric field
double rhoE;//charge density
double Ex,Ey,Ez;
// total body force
double Fx,Fy,Fz;
const double mrt_V1=0.05263157894736842;
const double mrt_V2=0.012531328320802;
const double mrt_V3=0.04761904761904762;
const double mrt_V4=0.004594820384294068;
const double mrt_V5=0.01587301587301587;
const double mrt_V6=0.0555555555555555555555555;
const double mrt_V7=0.02777777777777778;
const double mrt_V8=0.08333333333333333;
const double mrt_V9=0.003341687552213868;
const double mrt_V10=0.003968253968253968;
const double mrt_V11=0.01388888888888889;
const double mrt_V12=0.04166666666666666;
int S = Np/NBLOCKS/NTHREADS + 1;
for (int s=0; s<S; s++){
//........Get 1-D index for this thread....................
n = S*blockIdx.x*blockDim.x + s*blockDim.x + threadIdx.x + start;
if (n<finish) {
//Load data
rhoE = ChargeDensity[n];
Ex = ElectricField[n+0*Np];
Ey = ElectricField[n+1*Np];
Ez = ElectricField[n+2*Np];
//compute total body force, including input body force (Gx,Gy,Gz)
Fx = Gx + rhoE*Ex*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;//the extra factors at the end necessarily convert unit from phys to LB
Fy = Gy + rhoE*Ey*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
Fz = Gz + rhoE*Ez*(time_conv*time_conv)/(h*h*1.0e-12)/den_scale;
// q=0
fq = dist[n];
rho = fq;
m1 = -30.0*fq;
m2 = 12.0*fq;
// q=1
fq = dist[2*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jx = fq;
m4 = -4.0*fq;
m9 = 2.0*fq;
m10 = -4.0*fq;
// f2 = dist[10*Np+n];
fq = dist[1*Np+n];
rho += fq;
m1 -= 11.0*(fq);
m2 -= 4.0*(fq);
jx -= fq;
m4 += 4.0*(fq);
m9 += 2.0*(fq);
m10 -= 4.0*(fq);
// q=3
fq = dist[4*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy = fq;
m6 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 = fq;
m12 = -2.0*fq;
// q = 4
fq = dist[3*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jy -= fq;
m6 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 += fq;
m12 -= 2.0*fq;
// q=5
fq = dist[6*Np+n];
rho += fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz = fq;
m8 = -4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q = 6
fq = dist[5*Np+n];
rho+= fq;
m1 -= 11.0*fq;
m2 -= 4.0*fq;
jz -= fq;
m8 += 4.0*fq;
m9 -= fq;
m10 += 2.0*fq;
m11 -= fq;
m12 += 2.0*fq;
// q=7
fq = dist[8*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 = fq;
m16 = fq;
m17 = -fq;
// q = 8
fq = dist[7*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 += fq;
m16 -= fq;
m17 += fq;
// q=9
fq = dist[10*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jy -= fq;
m6 -= fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 += fq;
m17 += fq;
// q = 10
fq = dist[9*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jy += fq;
m6 += fq;
m9 += fq;
m10 += fq;
m11 += fq;
m12 += fq;
m13 -= fq;
m16 -= fq;
m17 -= fq;
// q=11
fq = dist[12*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 = fq;
m16 -= fq;
m18 = fq;
// q=12
fq = dist[11*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 += fq;
m16 += fq;
m18 -= fq;
// q=13
fq = dist[14*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx += fq;
m4 += fq;
jz -= fq;
m8 -= fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 -= fq;
m18 -= fq;
// q=14
fq = dist[13*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jx -= fq;
m4 -= fq;
jz += fq;
m8 += fq;
m9 += fq;
m10 += fq;
m11 -= fq;
m12 -= fq;
m15 -= fq;
m16 += fq;
m18 += fq;
// q=15
fq = dist[16*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 = fq;
m17 += fq;
m18 -= fq;
// q=16
fq = dist[15*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 += fq;
m17 -= fq;
m18 += fq;
// q=17
fq = dist[18*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy += fq;
m6 += fq;
jz -= fq;
m8 -= fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 += fq;
m18 += fq;
// q=18
fq = dist[17*Np+n];
rho += fq;
m1 += 8.0*fq;
m2 += fq;
jy -= fq;
m6 -= fq;
jz += fq;
m8 += fq;
m9 -= 2.0*fq;
m10 -= 2.0*fq;
m14 -= fq;
m17 -= fq;
m18 -= fq;
// write the velocity
ux = jx / rho0;
uy = jy / rho0;
uz = jz / rho0;
Velocity[n] = ux;
Velocity[Np+n] = uy;
Velocity[2*Np+n] = uz;
//........................................................................
// READ THE DISTRIBUTIONS
// (read from opposite array due to previous swap operation)
//........................................................................
//..............incorporate external force................................................
//..............carry out relaxation process...............................................
m1 = m1 + rlx_setA*((19*(jx*jx+jy*jy+jz*jz)/rho0 - 11*rho) - m1);
m2 = m2 + rlx_setA*((3*rho - 5.5*(jx*jx+jy*jy+jz*jz)/rho0) - m2);
m4 = m4 + rlx_setB*((-0.6666666666666666*jx) - m4);
m6 = m6 + rlx_setB*((-0.6666666666666666*jy) - m6);
m8 = m8 + rlx_setB*((-0.6666666666666666*jz) - m8);
m9 = m9 + rlx_setA*(((2*jx*jx-jy*jy-jz*jz)/rho0) - m9);
m10 = m10 + rlx_setA*(-0.5*((2*jx*jx-jy*jy-jz*jz)/rho) - m10);
m11 = m11 + rlx_setA*(((jy*jy-jz*jz)/rho0) - m11);
m12 = m12 + rlx_setA*(-0.5*((jy*jy-jz*jz)/rho0) - m12);
m13 = m13 + rlx_setA*((jx*jy/rho0) - m13);
m14 = m14 + rlx_setA*((jy*jz/rho0) - m14);
m15 = m15 + rlx_setA*((jx*jz/rho0) - m15);
m16 = m16 + rlx_setB*( - m16);
m17 = m17 + rlx_setB*( - m17);
m18 = m18 + rlx_setB*( - m18);
//.......................................................................................................
//.................inverse transformation......................................................
// q=0
fq = mrt_V1*rho-mrt_V2*m1+mrt_V3*m2;
dist[n] = fq;
// q = 1
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jx-m4)+mrt_V6*(m9-m10) + 0.16666666*Fx;
dist[1*Np+n] = fq;
// q=2
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m4-jx)+mrt_V6*(m9-m10) - 0.16666666*Fx;
dist[2*Np+n] = fq;
// q = 3
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jy-m6)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) + 0.16666666*Fy;
dist[3*Np+n] = fq;
// q = 4
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m6-jy)+mrt_V7*(m10-m9)+mrt_V8*(m11-m12) - 0.16666666*Fy;
dist[4*Np+n] = fq;
// q = 5
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(jz-m8)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) + 0.16666666*Fz;
dist[5*Np+n] = fq;
// q = 6
fq = mrt_V1*rho-mrt_V4*m1-mrt_V5*m2+0.1*(m8-jz)+mrt_V7*(m10-m9)+mrt_V8*(m12-m11) - 0.16666666*Fz;
dist[6*Np+n] = fq;
// q = 7
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx+jy)+0.025*(m4+m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m16-m17) + 0.08333333333*(Fx+Fy);
dist[7*Np+n] = fq;
// q = 8
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jy)-0.025*(m4+m6) +mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12+0.25*m13+0.125*(m17-m16) - 0.08333333333*(Fx+Fy);
dist[8*Np+n] = fq;
// q = 9
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jx-jy)+0.025*(m4-m6)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13+0.125*(m16+m17) + 0.08333333333*(Fx-Fy);
dist[9*Np+n] = fq;
// q = 10
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2+0.1*(jy-jx)+0.025*(m6-m4)
+mrt_V7*m9+mrt_V11*m10+mrt_V8*m11
+mrt_V12*m12-0.25*m13-0.125*(m16+m17)- 0.08333333333*(Fx-Fy);
dist[10*Np+n] = fq;
// q = 11
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx+jz)+0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m18-m16) + 0.08333333333*(Fx+Fz);
dist[11*Np+n] = fq;
// q = 12
fq = mrt_V1*rho+mrt_V9*m1+mrt_V10*m2-0.1*(jx+jz)-0.025*(m4+m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12+0.25*m15+0.125*(m16-m18) - 0.08333333333*(Fx+Fz);
dist[12*Np+n] = fq;
// q = 13
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jx-jz)+0.025*(m4-m8)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15-0.125*(m16+m18) + 0.08333333333*(Fx-Fz);
dist[13*Np+n] = fq;
// q= 14
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jx)+0.025*(m8-m4)
+mrt_V7*m9+mrt_V11*m10-mrt_V8*m11
-mrt_V12*m12-0.25*m15+0.125*(m16+m18) - 0.08333333333*(Fx-Fz);
dist[14*Np+n] = fq;
// q = 15
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy+jz)+0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m17-m18) + 0.08333333333*(Fy+Fz);
dist[15*Np+n] = fq;
// q = 16
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2-0.1*(jy+jz)-0.025*(m6+m8)
-mrt_V6*m9-mrt_V7*m10+0.25*m14+0.125*(m18-m17)- 0.08333333333*(Fy+Fz);
dist[16*Np+n] = fq;
// q = 17
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jy-jz)+0.025*(m6-m8)
-mrt_V6*m9-mrt_V7*m10-0.25*m14+0.125*(m17+m18) + 0.08333333333*(Fy-Fz);
dist[17*Np+n] = fq;
// q = 18
fq = mrt_V1*rho+mrt_V9*m1
+mrt_V10*m2+0.1*(jz-jy)+0.025*(m8-m6)
-mrt_V6*m9-mrt_V7*m10-0.25*m14-0.125*(m17+m18) - 0.08333333333*(Fy-Fz);
dist[18*Np+n] = fq;
//........................................................................
}
}
}
extern "C" void ScaLBL_D3Q19_AAodd_StokesMRT(int *neighborList, double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAodd_StokesMRT<<<NBLOCKS,NTHREADS >>>(neighborList,dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_AAodd_StokesMRT: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}
extern "C" void ScaLBL_D3Q19_AAeven_StokesMRT(double *dist, double *Velocity, double *ChargeDensity, double *ElectricField, double rlx_setA, double rlx_setB, double Gx, double Gy, double Gz,double rho0, double den_scale, double h, double time_conv, int start, int finish, int Np){
//cudaProfilerStart();
dvc_ScaLBL_D3Q19_AAeven_StokesMRT<<<NBLOCKS,NTHREADS >>>(dist,Velocity,ChargeDensity,ElectricField,rlx_setA,rlx_setB,Gx,Gy,Gz,rho0,den_scale,h,time_conv,start,finish,Np);
cudaError_t err = cudaGetLastError();
if (cudaSuccess != err){
printf("CUDA error in ScaLBL_D3Q19_AAeven_StokesMRT: %s \n",cudaGetErrorString(err));
}
//cudaProfilerStop();
}

1516
cuda/dfh.cu Normal file
View File

File diff suppressed because it is too large Load Diff

0
gpu/exe/CMakeLists.txt → cuda/exe/CMakeLists.txt
View File

0
gpu/exe/lb1_MRT-swap.cu → cuda/exe/lb1_MRT-swap.cu
View File

0
gpu/exe/lb1_MRT.cu → cuda/exe/lb1_MRT.cu
View File

0
gpu/exe/lb1_MRT_mpi.cpp → cuda/exe/lb1_MRT_mpi.cpp
View File

Some files were not shown because too many files have changed in this diff Show More